From f2a6696362f2249ed607a65f2d48289f67aeb51b Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 15 Oct 2020 02:39:31 +0400 Subject: [PATCH 001/284] Implement verification_cooldown LDAP server connection param --- programs/server/config.xml | 4 ++++ src/Access/Authentication.cpp | 31 ++++++++++++++++++++++++++- src/Access/Authentication.h | 5 +++++ src/Access/ExternalAuthenticators.cpp | 4 ++++ src/Access/LDAPParams.h | 18 ++++++++++++++++ 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index 5bdec5377fd..f295221452a 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -222,6 +222,9 @@ auth_dn_prefix, auth_dn_suffix - prefix and suffix used to construct the DN to bind to. Effectively, the resulting DN will be constructed as auth_dn_prefix + escape(user_name) + auth_dn_suffix string. Note, that this implies that auth_dn_suffix should usually have comma ',' as its first non-space character. + verification_cooldown - a period of time, in seconds, after a successful bind attempt, during which a user will be assumed + to be successfully authenticated for all consecutive requests without contacting the LDAP server. + Specify 0 (the default) to disable caching and force contacting the LDAP server for each authentication request. enable_tls - flag to trigger use of secure connection to the LDAP server. Specify 'no' for plain text (ldap://) protocol (not recommended). Specify 'yes' for LDAP over SSL/TLS (ldaps://) protocol (recommended, the default). @@ -241,6 +244,7 @@ 636 uid= ,ou=users,dc=example,dc=com + 300 yes tls1.2 demand diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index d29e2f897e8..7476e6eb2ea 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -86,8 +86,37 @@ bool Authentication::isCorrectPassword(const String & password_, const String & ldap_server_params.user = user_; ldap_server_params.password = password_; + const auto current_params_hash = ldap_server_params.getCoreHash(); + const auto last_check_period = std::chrono::steady_clock::now() - last_successful_password_check_timestamp; + + if ( + // Forbid the initial values explicitly. + last_successful_password_check_params_hash != 0 && + last_successful_password_check_timestamp != std::chrono::steady_clock::time_point{} && + + // Check if we can "reuse" the result of the previous successful password verification. + current_params_hash == last_successful_password_check_params_hash && + last_check_period <= ldap_server_params.verification_cooldown + ) + { + return true; + } + LDAPSimpleAuthClient ldap_client(ldap_server_params); - return ldap_client.check(); + const auto result = ldap_client.check(); + + if (result) + { + last_successful_password_check_params_hash = current_params_hash; + last_successful_password_check_timestamp = std::chrono::steady_clock::now(); + } + else + { + last_successful_password_check_params_hash = 0; + last_successful_password_check_timestamp = std::chrono::steady_clock::time_point{}; + } + + return result; } case MAX_TYPE: diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h index 38714339221..8584c9f7325 100644 --- a/src/Access/Authentication.h +++ b/src/Access/Authentication.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -103,7 +104,11 @@ private: Type type = Type::NO_PASSWORD; Digest password_hash; + + // Used and maintained only for LDAP. String server_name; + mutable std::size_t last_successful_password_check_params_hash = 0; + mutable std::chrono::steady_clock::time_point last_successful_password_check_timestamp; }; diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index 3ed1b21c3c2..9b908841576 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -29,6 +29,7 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str const bool has_port = config.has(ldap_server_config + ".port"); const bool has_auth_dn_prefix = config.has(ldap_server_config + ".auth_dn_prefix"); const bool has_auth_dn_suffix = config.has(ldap_server_config + ".auth_dn_suffix"); + const bool has_verification_cooldown = config.has(ldap_server_config + ".verification_cooldown"); const bool has_enable_tls = config.has(ldap_server_config + ".enable_tls"); const bool has_tls_minimum_protocol_version = config.has(ldap_server_config + ".tls_minimum_protocol_version"); const bool has_tls_require_cert = config.has(ldap_server_config + ".tls_require_cert"); @@ -52,6 +53,9 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str if (has_auth_dn_suffix) params.auth_dn_suffix = config.getString(ldap_server_config + ".auth_dn_suffix"); + if (has_verification_cooldown) + params.verification_cooldown = std::chrono::seconds{config.getUInt64(ldap_server_config + ".verification_cooldown")}; + if (has_enable_tls) { String enable_tls_lc_str = config.getString(ldap_server_config + ".enable_tls"); diff --git a/src/Access/LDAPParams.h b/src/Access/LDAPParams.h index eeadba6bc01..28dcc5fe50f 100644 --- a/src/Access/LDAPParams.h +++ b/src/Access/LDAPParams.h @@ -2,6 +2,8 @@ #include +#include + #include @@ -68,10 +70,26 @@ struct LDAPServerParams String user; String password; + std::chrono::seconds verification_cooldown{0}; + std::chrono::seconds operation_timeout{40}; std::chrono::seconds network_timeout{30}; std::chrono::seconds search_timeout{20}; std::uint32_t search_limit = 100; + + std::size_t getCoreHash() const + { + std::size_t seed = 0; + + boost::hash_combine(seed, host); + boost::hash_combine(seed, port); + boost::hash_combine(seed, auth_dn_prefix); + boost::hash_combine(seed, auth_dn_suffix); + boost::hash_combine(seed, user); + boost::hash_combine(seed, password); + + return seed; + } }; } From 1eb7c31011ed25738da65fc169816dd5720e721a Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 15 Oct 2020 03:05:33 +0400 Subject: [PATCH 002/284] Add "verification_cooldown enabled" check --- src/Access/Authentication.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 7476e6eb2ea..b8328057161 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -94,8 +94,12 @@ bool Authentication::isCorrectPassword(const String & password_, const String & last_successful_password_check_params_hash != 0 && last_successful_password_check_timestamp != std::chrono::steady_clock::time_point{} && + // Check if the caching is enabled at all. + ldap_server_params.verification_cooldown > std::chrono::seconds{0} && + // Check if we can "reuse" the result of the previous successful password verification. current_params_hash == last_successful_password_check_params_hash && + last_check_period >= std::chrono::seconds{0} && last_check_period <= ldap_server_params.verification_cooldown ) { From 92840dd7171bace7f41f57578a84c5c68e2e20f2 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 20 Oct 2020 15:36:22 +0400 Subject: [PATCH 003/284] Trigger CI From 1567a3976b9ca7893c8e730f8f6db8de3057c519 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 26 Oct 2020 09:28:46 -0400 Subject: [PATCH 004/284] Adding requirements related to the LDAP login cache feature. Updated syntax tests. Linked specifications to ldap/authentication and ldap/external_user_directory features. --- docker/test/testflows/runner/Dockerfile | 2 +- .../ldap/authentication/regression.py | 5 +- .../requirements/requirements.md | 83 +- .../requirements/requirements.py | 958 +++++++++++++++--- .../authentication/tests/server_config.py | 4 +- .../external_user_directory/regression.py | 5 +- .../requirements/requirements.md | 80 +- .../requirements/requirements.py | 893 +++++++++++++++- .../tests/server_config.py | 4 +- 9 files changed, 1859 insertions(+), 175 deletions(-) diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 9565e39598c..06241d6d497 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN pip3 install urllib3 testflows==1.6.57 docker-compose docker dicttoxml kazoo tzlocal +RUN pip3 install urllib3 testflows==1.6.59 docker-compose docker dicttoxml kazoo tzlocal ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 17.09.1-ce diff --git a/tests/testflows/ldap/authentication/regression.py b/tests/testflows/ldap/authentication/regression.py index 9d0a5ca743f..97b63dddd33 100755 --- a/tests/testflows/ldap/authentication/regression.py +++ b/tests/testflows/ldap/authentication/regression.py @@ -29,9 +29,8 @@ xfails = { @TestFeature @Name("authentication") @ArgumentParser(argparser) -@Requirements( - RQ_SRS_007_LDAP_Authentication("1.0") -) +@Specifications(SRS_007_ClickHouse_Authentication_of_Users_via_LDAP) +@Requirements(RQ_SRS_007_LDAP_Authentication("1.0")) @XFails(xfails) def regression(self, local, clickhouse_binary_path): """ClickHouse integration with LDAP regression module. diff --git a/tests/testflows/ldap/authentication/requirements/requirements.md b/tests/testflows/ldap/authentication/requirements/requirements.md index d322db70330..683dae4ead4 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.md +++ b/tests/testflows/ldap/authentication/requirements/requirements.md @@ -57,22 +57,27 @@ * 4.2.25 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir](#rqsrs-007ldapconfigurationservertlscacertdir) * 4.2.26 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile](#rqsrs-007ldapconfigurationservertlscacertfile) * 4.2.27 [RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite](#rqsrs-007ldapconfigurationservertlsciphersuite) - * 4.2.28 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) - * 4.2.29 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) - * 4.2.30 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) - * 4.2.31 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) - * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) - * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) - * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) - * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) - * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) - * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) - * 4.2.38 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) - * 4.2.39 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) - * 4.2.40 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) - * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) - * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) - * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) + * 4.2.28 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown](#rqsrs-007ldapconfigurationserververificationcooldown) + * 4.2.29 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default](#rqsrs-007ldapconfigurationserververificationcooldowndefault) + * 4.2.30 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) + * 4.2.31 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) + * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) + * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) + * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) + * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) + * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) + * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) + * 4.2.38 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) + * 4.2.39 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) + * 4.2.40 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) + * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) + * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) + * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) + * 4.2.44 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) + * 4.2.45 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) + * 4.2.46 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance](#rqsrs-007ldapauthenticationverificationcooldownperformance) + * 4.2.47 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters](#rqsrs-007ldapauthenticationverificationcooldownresetchangeincoreserverparameters) + * 4.2.48 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword](#rqsrs-007ldapauthenticationverificationcooldownresetinvalidpassword) * 5 [References](#references) ## Revision History @@ -393,9 +398,25 @@ For example, The available suites SHALL depend on the [OpenSSL] library version and variant used to build [ClickHouse] and therefore might change. -#### RQ.SRS-007.LDAP.Configuration.Server.Syntax +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown version: 1.0 +[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section +that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed +to be successfully authenticated for all consecutive requests without contacting the [LDAP] server. +After period of time since the last successful attempt expires then on the authentication attempt +SHALL result in contacting the [LDAP] server to verify the username and password. + +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default +version: 1.0 + +[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section +SHALL have a default value of `0` that disables caching and forces contacting +the [LDAP] server for each authentication request. + +#### RQ.SRS-007.LDAP.Configuration.Server.Syntax +version: 2.0 + [ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml` configuration file or of any configuration file inside the `config.d` directory. @@ -406,6 +427,7 @@ configuration file or of any configuration file inside the `config.d` directory. 636 cn= , ou=users, dc=example, dc=com + 0 yes tls1.2 demand @@ -521,6 +543,33 @@ version: 1.0 [ClickHouse] SHALL support [UTF-8] characters in passwords used to authenticate users using an [LDAP] server. +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance +version: 1.0 + +[ClickHouse] SHALL provide better login performance of [LDAP] authenticated users +when `verification_cooldown` parameter is set to a positive value when comparing +to the the case when `verification_cooldown` is turned off either for a single user or multiple users +making a large number of repeated requests. + +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters +version: 1.0 + +[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the +`verification_cooldown` parameter in the [LDAP] server configuration section +if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values +change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user +to result in contacting the [LDAP] server to verify user's username and password. + +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword +version: 1.0 + +[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the +`verification_cooldown` parameter in the [LDAP] server configuration section +for the user if the password provided in the current authentication attempt does not match +the valid password provided during the first successful authentication request that was cached +for this exact user. The reset SHALL cause the next authentication attempt for this user +to result in contacting the [LDAP] server to verify user's username and password. + ## References * **ClickHouse:** https://clickhouse.tech diff --git a/tests/testflows/ldap/authentication/requirements/requirements.py b/tests/testflows/ldap/authentication/requirements/requirements.py index 967e097d1fa..afdc497e75c 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.py +++ b/tests/testflows/ldap/authentication/requirements/requirements.py @@ -1,10 +1,620 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.200811.1124123. +# document by TestFlows v1.6.201021.1163815. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. +from testflows.core import Specification from testflows.core import Requirement +SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( + name='SRS-007 ClickHouse Authentication of Users via LDAP', + description=None, + author=None, + date=None, + status=None, + approved_by=None, + approved_date=None, + approved_version=None, + version=None, + group=None, + type=None, + link=None, + uid=None, + parent=None, + children=None, + content=''' +# SRS-007 ClickHouse Authentication of Users via LDAP + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [RQ.SRS-007.LDAP.Authentication](#rqsrs-007ldapauthentication) + * 4.1.2 [RQ.SRS-007.LDAP.Authentication.MultipleServers](#rqsrs-007ldapauthenticationmultipleservers) + * 4.1.3 [RQ.SRS-007.LDAP.Authentication.Protocol.PlainText](#rqsrs-007ldapauthenticationprotocolplaintext) + * 4.1.4 [RQ.SRS-007.LDAP.Authentication.Protocol.TLS](#rqsrs-007ldapauthenticationprotocoltls) + * 4.1.5 [RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS](#rqsrs-007ldapauthenticationprotocolstarttls) + * 4.1.6 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation](#rqsrs-007ldapauthenticationtlscertificatevalidation) + * 4.1.7 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned](#rqsrs-007ldapauthenticationtlscertificateselfsigned) + * 4.1.8 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority](#rqsrs-007ldapauthenticationtlscertificatespecificcertificationauthority) + * 4.1.9 [RQ.SRS-007.LDAP.Server.Configuration.Invalid](#rqsrs-007ldapserverconfigurationinvalid) + * 4.1.10 [RQ.SRS-007.LDAP.User.Configuration.Invalid](#rqsrs-007ldapuserconfigurationinvalid) + * 4.1.11 [RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous](#rqsrs-007ldapauthenticationmechanismanonymous) + * 4.1.12 [RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated](#rqsrs-007ldapauthenticationmechanismunauthenticated) + * 4.1.13 [RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword](#rqsrs-007ldapauthenticationmechanismnamepassword) + * 4.1.14 [RQ.SRS-007.LDAP.Authentication.Valid](#rqsrs-007ldapauthenticationvalid) + * 4.1.15 [RQ.SRS-007.LDAP.Authentication.Invalid](#rqsrs-007ldapauthenticationinvalid) + * 4.1.16 [RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser](#rqsrs-007ldapauthenticationinvaliddeleteduser) + * 4.1.17 [RQ.SRS-007.LDAP.Authentication.UsernameChanged](#rqsrs-007ldapauthenticationusernamechanged) + * 4.1.18 [RQ.SRS-007.LDAP.Authentication.PasswordChanged](#rqsrs-007ldapauthenticationpasswordchanged) + * 4.1.19 [RQ.SRS-007.LDAP.Authentication.LDAPServerRestart](#rqsrs-007ldapauthenticationldapserverrestart) + * 4.1.20 [RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart](#rqsrs-007ldapauthenticationclickhouseserverrestart) + * 4.1.21 [RQ.SRS-007.LDAP.Authentication.Parallel](#rqsrs-007ldapauthenticationparallel) + * 4.1.22 [RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid](#rqsrs-007ldapauthenticationparallelvalidandinvalid) + * 4.2 [Specific](#specific) + * 4.2.1 [RQ.SRS-007.LDAP.UnreachableServer](#rqsrs-007ldapunreachableserver) + * 4.2.2 [RQ.SRS-007.LDAP.Configuration.Server.Name](#rqsrs-007ldapconfigurationservername) + * 4.2.3 [RQ.SRS-007.LDAP.Configuration.Server.Host](#rqsrs-007ldapconfigurationserverhost) + * 4.2.4 [RQ.SRS-007.LDAP.Configuration.Server.Port](#rqsrs-007ldapconfigurationserverport) + * 4.2.5 [RQ.SRS-007.LDAP.Configuration.Server.Port.Default](#rqsrs-007ldapconfigurationserverportdefault) + * 4.2.6 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix](#rqsrs-007ldapconfigurationserverauthdnprefix) + * 4.2.7 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix](#rqsrs-007ldapconfigurationserverauthdnsuffix) + * 4.2.8 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value](#rqsrs-007ldapconfigurationserverauthdnvalue) + * 4.2.9 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS](#rqsrs-007ldapconfigurationserverenabletls) + * 4.2.10 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default](#rqsrs-007ldapconfigurationserverenabletlsoptionsdefault) + * 4.2.11 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No](#rqsrs-007ldapconfigurationserverenabletlsoptionsno) + * 4.2.12 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes](#rqsrs-007ldapconfigurationserverenabletlsoptionsyes) + * 4.2.13 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS](#rqsrs-007ldapconfigurationserverenabletlsoptionsstarttls) + * 4.2.14 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion](#rqsrs-007ldapconfigurationservertlsminimumprotocolversion) + * 4.2.15 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values](#rqsrs-007ldapconfigurationservertlsminimumprotocolversionvalues) + * 4.2.16 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default](#rqsrs-007ldapconfigurationservertlsminimumprotocolversiondefault) + * 4.2.17 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert](#rqsrs-007ldapconfigurationservertlsrequirecert) + * 4.2.18 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdefault) + * 4.2.19 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdemand) + * 4.2.20 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsallow) + * 4.2.21 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try](#rqsrs-007ldapconfigurationservertlsrequirecertoptionstry) + * 4.2.22 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsnever) + * 4.2.23 [RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile](#rqsrs-007ldapconfigurationservertlscertfile) + * 4.2.24 [RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile](#rqsrs-007ldapconfigurationservertlskeyfile) + * 4.2.25 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir](#rqsrs-007ldapconfigurationservertlscacertdir) + * 4.2.26 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile](#rqsrs-007ldapconfigurationservertlscacertfile) + * 4.2.27 [RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite](#rqsrs-007ldapconfigurationservertlsciphersuite) + * 4.2.28 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown](#rqsrs-007ldapconfigurationserververificationcooldown) + * 4.2.29 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default](#rqsrs-007ldapconfigurationserververificationcooldowndefault) + * 4.2.30 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) + * 4.2.31 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) + * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) + * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) + * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) + * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) + * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) + * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) + * 4.2.38 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) + * 4.2.39 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) + * 4.2.40 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) + * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) + * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) + * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) + * 4.2.44 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) + * 4.2.45 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) + * 4.2.46 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance](#rqsrs-007ldapauthenticationverificationcooldownperformance) + * 4.2.47 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters](#rqsrs-007ldapauthenticationverificationcooldownresetchangeincoreserverparameters) + * 4.2.48 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword](#rqsrs-007ldapauthenticationverificationcooldownresetinvalidpassword) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Git]'s [Revision History]. + +## Introduction + +[ClickHouse] currently does not have any integration with [LDAP]. +As the initial step in integrating with [LDAP] this software requirements specification covers +only the requirements to enable authentication of users using an [LDAP] server. + +## Terminology + +* **CA** - + Certificate Authority ([CA]) + +* **LDAP** - + Lightweight Directory Access Protocol ([LDAP]) + +## Requirements + +### Generic + +#### RQ.SRS-007.LDAP.Authentication +version: 1.0 + +[ClickHouse] SHALL support user authentication via an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.MultipleServers +version: 1.0 + +[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate +users. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.PlainText +version: 1.0 + +[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.TLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a +plain text `ldap://` protocol that is upgraded to [TLS]. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation +version: 1.0 + +[ClickHouse] SHALL support certificate validation used for [TLS] connections. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned +version: 1.0 + +[ClickHouse] SHALL support self-signed certificates for [TLS] connections. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority +version: 1.0 + +[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections. + +#### RQ.SRS-007.LDAP.Server.Configuration.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid. + +#### RQ.SRS-007.LDAP.User.Configuration.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword +version: 1.0 + +[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Valid +version: 1.0 + +[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if +user name and password match [LDAP] server records for the user. + +#### RQ.SRS-007.LDAP.Authentication.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if either user name or password +do not match [LDAP] server records for the user. + +#### RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the user +has been deleted from the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.UsernameChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the username is changed +on the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.PasswordChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the password +for the user is changed on the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.LDAPServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted. + +#### RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after server is restarted. + +#### RQ.SRS-007.LDAP.Authentication.Parallel +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users using [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid +version: 1.0 + +[ClickHouse] SHALL support authentication of valid users and +prohibit authentication of invalid users using [LDAP] server +in parallel without having invalid attempts affecting valid authentications. + +### Specific + +#### RQ.SRS-007.LDAP.UnreachableServer +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable. + +#### RQ.SRS-007.LDAP.Configuration.Server.Name +version: 1.0 + +[ClickHouse] SHALL not support empty string as a server name. + +#### RQ.SRS-007.LDAP.Configuration.Server.Host +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] +server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty. + +#### RQ.SRS-007.LDAP.Configuration.Server.Port +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] server port. + +#### RQ.SRS-007.LDAP.Configuration.Server.Port.Default +version: 1.0 + +[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the prefix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the suffix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value +version: 1.0 + +[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string. + +> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS +version: 1.0 + +[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `yes` value as the default for `` parameter +to enable SSL/TLS `ldaps://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No +version: 1.0 + +[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable +plain text `ldap://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes +version: 1.0 + +[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable +SSL/TLS `ldaps://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS +version: 1.0 + +[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable +legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS]. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify +the minimum protocol version of SSL/TLS. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values +version: 1.0 + +[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2` +as a value of the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default +version: 1.0 + +[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [TLS] peer +certificate verification behavior. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `demand` value as the default for the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand +version: 1.0 + +[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, or a bad certificate is +provided, the session SHALL be immediately terminated. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow +version: 1.0 + +[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to +enable requesting of client certificate. If no +certificate is provided, the session SHALL proceed normally. +If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try +version: 1.0 + +[ClickHouse] SHALL support specifying `try` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, the session +SHALL proceed normally. If a bad certificate is provided, the session SHALL be +immediately terminated. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never +version: 1.0 + +[ClickHouse] SHALL support specifying `never` as the value of `` parameter to +disable requesting of client certificate. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to certificate file used by +[ClickHouse] to establish connection with the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to key file for the certificate +specified by the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify to a path to +the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify a path to a specific +[CA] certificate file used to verify certificates provided by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite +version: 1.0 + +[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites. +The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers]. + +For example, + +```xml +ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 +``` + +The available suites SHALL depend on the [OpenSSL] library version and variant used to build +[ClickHouse] and therefore might change. + +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown +version: 1.0 + +[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section +that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed +to be successfully authenticated for all consecutive requests without contacting the [LDAP] server. +After period of time since the last successful attempt expires then on the authentication attempt +SHALL result in contacting the [LDAP] server to verify the username and password. + +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default +version: 1.0 + +[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section +SHALL have a default value of `0` that disables caching and forces contacting +the [LDAP] server for each authentication request. + +#### RQ.SRS-007.LDAP.Configuration.Server.Syntax +version: 2.0 + +[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml` +configuration file or of any configuration file inside the `config.d` directory. + +```xml + + + localhost + 636 + cn= + , ou=users, dc=example, dc=com + 0 + yes + tls1.2 + demand + /path/to/tls_cert_file + /path/to/tls_key_file + /path/to/tls_ca_cert_file + /path/to/tls_ca_cert_dir + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + +``` + +#### RQ.SRS-007.LDAP.Configuration.User.RBAC +version: 1.0 + +[ClickHouse] SHALL support creating users identified using an [LDAP] server using +the following RBAC command + +```sql +CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name' +``` + +#### RQ.SRS-007.LDAP.Configuration.User.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following example syntax to create a user that is authenticated using +an [LDAP] server inside the `users.xml` file or any configuration file inside the `users.d` directory. + +```xml + + + + + my_ldap_server + + + + +``` + +#### RQ.SRS-007.LDAP.Configuration.User.Name.Empty +version: 1.0 + +[ClickHouse] SHALL not support empty string as a user name. + +#### RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP +version: 1.0 + +[ClickHouse] SHALL throw an error if `` is specified for the user and at the same +time user configuration contains any of the `` entries. + +#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined +version: 1.0 + +[ClickHouse] SHALL throw an error during any authentification attempt +if the name of the [LDAP] server used inside the `` entry +is not defined in the `` section. + +#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty +version: 1.0 + +[ClickHouse] SHALL throw an error during any authentification attempt +if the name of the [LDAP] server used inside the `` entry +is empty. + +#### RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer +version: 1.0 + +[ClickHouse] SHALL support specifying only one [LDAP] server for a given user. + +#### RQ.SRS-007.LDAP.Configuration.User.Name.Long +version: 1.0 + +[ClickHouse] SHALL support long user names of at least 256 bytes +to specify users that can be authenticated using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.User.Name.UTF8 +version: 1.0 + +[ClickHouse] SHALL support user names that contain [UTF-8] characters. + +#### RQ.SRS-007.LDAP.Authentication.Username.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty username. + +#### RQ.SRS-007.LDAP.Authentication.Username.Long +version: 1.0 + +[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes. + +#### RQ.SRS-007.LDAP.Authentication.Username.UTF8 +version: 1.0 + +[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters. + +#### RQ.SRS-007.LDAP.Authentication.Password.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty passwords +even if an empty password is valid for the user and +is allowed by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Password.Long +version: 1.0 + +[ClickHouse] SHALL support long password of at least 256 bytes +that can be used to authenticate users using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Password.UTF8 +version: 1.0 + +[ClickHouse] SHALL support [UTF-8] characters in passwords +used to authenticate users using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance +version: 1.0 + +[ClickHouse] SHALL provide better login performance of [LDAP] authenticated users +when `verification_cooldown` parameter is set to a positive value when comparing +to the the case when `verification_cooldown` is turned off either for a single user or multiple users +making a large number of repeated requests. + +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters +version: 1.0 + +[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the +`verification_cooldown` parameter in the [LDAP] server configuration section +if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values +change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user +to result in contacting the [LDAP] server to verify user's username and password. + +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword +version: 1.0 + +[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the +`verification_cooldown` parameter in the [LDAP] server configuration section +for the user if the password provided in the current authentication attempt does not match +the valid password provided during the first successful authentication request that was cached +for this exact user. The reset SHALL cause the next authentication attempt for this user +to result in contacting the [LDAP] server to verify user's username and password. + +## References + +* **ClickHouse:** https://clickhouse.tech + +[Anonymous Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-AnonymousAuthenticationMechanismOfSimpleBind +[Unauthenticated Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-UnauthenticatedAuthenticationMechanismOfSimpleBind +[Name/Password Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-NamePasswordAuthenticationMechanismOfSimpleBind +[UTF-8]: https://en.wikipedia.org/wiki/UTF-8 +[OpenSSL]: https://www.openssl.org/ +[OpenSSL Ciphers]: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html +[CA]: https://en.wikipedia.org/wiki/Certificate_authority +[TLS]: https://en.wikipedia.org/wiki/Transport_Layer_Security +[LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol +[ClickHouse]: https://clickhouse.tech +[GitHub]: https://github.com +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/authentication/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/authentication/requirements/requirements.md +[Git]: https://git-scm.com/ +''') + RQ_SRS_007_LDAP_Authentication = Requirement( name='RQ.SRS-007.LDAP.Authentication', version='1.0', @@ -14,9 +624,9 @@ RQ_SRS_007_LDAP_Authentication = Requirement( uid=None, description=( '[ClickHouse] SHALL support user authentication via an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_MultipleServers = Requirement( name='RQ.SRS-007.LDAP.Authentication.MultipleServers', @@ -28,9 +638,9 @@ RQ_SRS_007_LDAP_Authentication_MultipleServers = Requirement( description=( '[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate\n' 'users.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Protocol_PlainText = Requirement( name='RQ.SRS-007.LDAP.Authentication.Protocol.PlainText', @@ -41,9 +651,9 @@ RQ_SRS_007_LDAP_Authentication_Protocol_PlainText = Requirement( uid=None, description=( '[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Protocol_TLS = Requirement( name='RQ.SRS-007.LDAP.Authentication.Protocol.TLS', @@ -54,9 +664,9 @@ RQ_SRS_007_LDAP_Authentication_Protocol_TLS = Requirement( uid=None, description=( '[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Protocol_StartTLS = Requirement( name='RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS', @@ -68,9 +678,9 @@ RQ_SRS_007_LDAP_Authentication_Protocol_StartTLS = Requirement( description=( '[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a\n' 'plain text `ldap://` protocol that is upgraded to [TLS].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_Validation = Requirement( name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation', @@ -81,9 +691,9 @@ RQ_SRS_007_LDAP_Authentication_TLS_Certificate_Validation = Requirement( uid=None, description=( '[ClickHouse] SHALL support certificate validation used for [TLS] connections.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SelfSigned = Requirement( name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned', @@ -94,9 +704,9 @@ RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SelfSigned = Requirement( uid=None, description=( '[ClickHouse] SHALL support self-signed certificates for [TLS] connections.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SpecificCertificationAuthority = Requirement( name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority', @@ -107,9 +717,9 @@ RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SpecificCertificationAuthority = uid=None, description=( '[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Server_Configuration_Invalid = Requirement( name='RQ.SRS-007.LDAP.Server.Configuration.Invalid', @@ -120,9 +730,9 @@ RQ_SRS_007_LDAP_Server_Configuration_Invalid = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_User_Configuration_Invalid = Requirement( name='RQ.SRS-007.LDAP.User.Configuration.Invalid', @@ -133,9 +743,9 @@ RQ_SRS_007_LDAP_User_Configuration_Invalid = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Mechanism_Anonymous = Requirement( name='RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous', @@ -147,9 +757,9 @@ RQ_SRS_007_LDAP_Authentication_Mechanism_Anonymous = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind]\n' 'authentication mechanism.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Mechanism_Unauthenticated = Requirement( name='RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated', @@ -161,9 +771,9 @@ RQ_SRS_007_LDAP_Authentication_Mechanism_Unauthenticated = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind]\n' 'authentication mechanism.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword = Requirement( name='RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword', @@ -175,9 +785,9 @@ RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword = Requirement( description=( '[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind]\n' 'authentication mechanism.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Valid = Requirement( name='RQ.SRS-007.LDAP.Authentication.Valid', @@ -189,9 +799,9 @@ RQ_SRS_007_LDAP_Authentication_Valid = Requirement( description=( '[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if\n' 'user name and password match [LDAP] server records for the user.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Invalid = Requirement( name='RQ.SRS-007.LDAP.Authentication.Invalid', @@ -203,9 +813,9 @@ RQ_SRS_007_LDAP_Authentication_Invalid = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if either user name or password\n' 'do not match [LDAP] server records for the user.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser = Requirement( name='RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser', @@ -217,9 +827,9 @@ RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if the user\n' 'has been deleted from the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_UsernameChanged = Requirement( name='RQ.SRS-007.LDAP.Authentication.UsernameChanged', @@ -231,9 +841,9 @@ RQ_SRS_007_LDAP_Authentication_UsernameChanged = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if the username is changed\n' 'on the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_PasswordChanged = Requirement( name='RQ.SRS-007.LDAP.Authentication.PasswordChanged', @@ -245,9 +855,9 @@ RQ_SRS_007_LDAP_Authentication_PasswordChanged = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if the password\n' 'for the user is changed on the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_LDAPServerRestart = Requirement( name='RQ.SRS-007.LDAP.Authentication.LDAPServerRestart', @@ -258,9 +868,9 @@ RQ_SRS_007_LDAP_Authentication_LDAPServerRestart = Requirement( uid=None, description=( '[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart = Requirement( name='RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart', @@ -271,9 +881,9 @@ RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart = Requirement( uid=None, description=( '[ClickHouse] SHALL support authenticating users after server is restarted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Parallel = Requirement( name='RQ.SRS-007.LDAP.Authentication.Parallel', @@ -284,9 +894,9 @@ RQ_SRS_007_LDAP_Authentication_Parallel = Requirement( uid=None, description=( '[ClickHouse] SHALL support parallel authentication of users using [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid = Requirement( name='RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid', @@ -299,9 +909,9 @@ RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid = Requirement( '[ClickHouse] SHALL support authentication of valid users and\n' 'prohibit authentication of invalid users using [LDAP] server\n' 'in parallel without having invalid attempts affecting valid authentications.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_UnreachableServer = Requirement( name='RQ.SRS-007.LDAP.UnreachableServer', @@ -312,9 +922,9 @@ RQ_SRS_007_LDAP_UnreachableServer = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Name = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Name', @@ -325,9 +935,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Name = Requirement( uid=None, description=( '[ClickHouse] SHALL not support empty string as a server name.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Host = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Host', @@ -339,9 +949,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Host = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify [LDAP]\n' 'server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Port = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Port', @@ -352,9 +962,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Port = Requirement( uid=None, description=( '[ClickHouse] SHALL support `` parameter to specify [LDAP] server port.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Port_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Port.Default', @@ -365,9 +975,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Port_Default = Requirement( uid=None, description=( '[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Prefix = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix', @@ -379,9 +989,9 @@ RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Prefix = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify the prefix\n' 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix', @@ -393,9 +1003,9 @@ RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify the suffix\n' 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Value = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value', @@ -408,9 +1018,9 @@ RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Value = Requirement( '[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string.\n' '\n' "> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character.\n" + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS', @@ -421,9 +1031,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS = Requirement( uid=None, description=( '[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default', @@ -435,9 +1045,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Default = Requirement( description=( '[ClickHouse] SHALL use `yes` value as the default for `` parameter\n' 'to enable SSL/TLS `ldaps://` protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_No = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No', @@ -449,9 +1059,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_No = Requirement( description=( '[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable\n' 'plain text `ldap://` protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Yes = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes', @@ -463,9 +1073,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Yes = Requirement( description=( '[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable\n' 'SSL/TLS `ldaps://` protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_StartTLS = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS', @@ -477,9 +1087,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_StartTLS = Requirement( description=( '[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable\n' 'legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion', @@ -491,9 +1101,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify\n' 'the minimum protocol version of SSL/TLS.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Values = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values', @@ -505,9 +1115,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Values = Requirem description=( '[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2`\n' 'as a value of the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default', @@ -518,9 +1128,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Default = Require uid=None, description=( '[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert', @@ -532,9 +1142,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify [TLS] peer\n' 'certificate verification behavior.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default', @@ -545,9 +1155,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Default = Requiremen uid=None, description=( '[ClickHouse] SHALL use `demand` value as the default for the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Demand = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand', @@ -560,9 +1170,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Demand = Requirement '[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to\n' 'enable requesting of client certificate. If no certificate is provided, or a bad certificate is\n' 'provided, the session SHALL be immediately terminated.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Allow = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow', @@ -576,9 +1186,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Allow = Requirement( 'enable requesting of client certificate. If no\n' 'certificate is provided, the session SHALL proceed normally.\n' 'If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Try = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try', @@ -592,9 +1202,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Try = Requirement( 'enable requesting of client certificate. If no certificate is provided, the session\n' 'SHALL proceed normally. If a bad certificate is provided, the session SHALL be\n' 'immediately terminated.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Never = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never', @@ -606,9 +1216,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Never = Requirement( description=( '[ClickHouse] SHALL support specifying `never` as the value of `` parameter to\n' 'disable requesting of client certificate.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCertFile = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile', @@ -620,9 +1230,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCertFile = Requirement( description=( '[ClickHouse] SHALL support `` to specify the path to certificate file used by\n' '[ClickHouse] to establish connection with the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSKeyFile = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile', @@ -634,9 +1244,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSKeyFile = Requirement( description=( '[ClickHouse] SHALL support `` to specify the path to key file for the certificate\n' 'specified by the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCACertDir = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir', @@ -648,9 +1258,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCACertDir = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify to a path to\n' 'the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCACertFile = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile', @@ -662,9 +1272,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCACertFile = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify a path to a specific\n' '[CA] certificate file used to verify certificates provided by the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite', @@ -685,13 +1295,45 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite = Requirement( '\n' 'The available suites SHALL depend on the [OpenSSL] library version and variant used to build\n' '[ClickHouse] and therefore might change.\n' + '\n' ), - link=None - ) + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed\n' + 'to be successfully authenticated for all consecutive requests without contacting the [LDAP] server.\n' + 'After period of time since the last successful attempt expires then on the authentication attempt\n' + 'SHALL result in contacting the [LDAP] server to verify the username and password. \n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Default = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'SHALL have a default value of `0` that disables caching and forces contacting\n' + 'the [LDAP] server for each authentication request.\n' + '\n' + ), + link=None) RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Syntax', - version='1.0', + version='2.0', priority=None, group=None, type=None, @@ -707,6 +1349,7 @@ RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( ' 636\n' ' cn=\n' ' , ou=users, dc=example, dc=com\n' + ' 0\n' ' yes\n' ' tls1.2\n' ' demand\n' @@ -718,9 +1361,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( ' \n' '\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_RBAC = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.RBAC', @@ -736,9 +1379,9 @@ RQ_SRS_007_LDAP_Configuration_User_RBAC = Requirement( '```sql\n' "CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name'\n" '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Syntax = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Syntax', @@ -762,9 +1405,9 @@ RQ_SRS_007_LDAP_Configuration_User_Syntax = Requirement( ' \n' '\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Name_Empty = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.Empty', @@ -775,9 +1418,9 @@ RQ_SRS_007_LDAP_Configuration_User_Name_Empty = Requirement( uid=None, description=( '[ClickHouse] SHALL not support empty string as a user name.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP', @@ -789,9 +1432,9 @@ RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP = Requirement( description=( '[ClickHouse] SHALL throw an error if `` is specified for the user and at the same\n' 'time user configuration contains any of the `` entries.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined', @@ -804,9 +1447,9 @@ RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined = Requireme '[ClickHouse] SHALL throw an error during any authentification attempt\n' 'if the name of the [LDAP] server used inside the `` entry\n' 'is not defined in the `` section.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty', @@ -819,9 +1462,9 @@ RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty = Requirement( '[ClickHouse] SHALL throw an error during any authentification attempt\n' 'if the name of the [LDAP] server used inside the `` entry\n' 'is empty.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer', @@ -832,9 +1475,9 @@ RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer = Requirement( uid=None, description=( '[ClickHouse] SHALL support specifying only one [LDAP] server for a given user.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Name_Long = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.Long', @@ -846,9 +1489,9 @@ RQ_SRS_007_LDAP_Configuration_User_Name_Long = Requirement( description=( '[ClickHouse] SHALL support long user names of at least 256 bytes\n' 'to specify users that can be authenticated using an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Name_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.UTF8', @@ -859,9 +1502,9 @@ RQ_SRS_007_LDAP_Configuration_User_Name_UTF8 = Requirement( uid=None, description=( '[ClickHouse] SHALL support user names that contain [UTF-8] characters.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Username_Empty = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.Empty', @@ -872,9 +1515,9 @@ RQ_SRS_007_LDAP_Authentication_Username_Empty = Requirement( uid=None, description=( '[ClickHouse] SHALL not support authenticating users with empty username.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Username_Long = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.Long', @@ -885,9 +1528,9 @@ RQ_SRS_007_LDAP_Authentication_Username_Long = Requirement( uid=None, description=( '[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Username_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.UTF8', @@ -898,9 +1541,9 @@ RQ_SRS_007_LDAP_Authentication_Username_UTF8 = Requirement( uid=None, description=( '[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Password_Empty = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.Empty', @@ -913,9 +1556,9 @@ RQ_SRS_007_LDAP_Authentication_Password_Empty = Requirement( '[ClickHouse] SHALL not support authenticating users with empty passwords\n' 'even if an empty password is valid for the user and\n' 'is allowed by the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Password_Long = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.Long', @@ -927,9 +1570,9 @@ RQ_SRS_007_LDAP_Authentication_Password_Long = Requirement( description=( '[ClickHouse] SHALL support long password of at least 256 bytes\n' 'that can be used to authenticate users using an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Password_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.UTF8', @@ -941,6 +1584,57 @@ RQ_SRS_007_LDAP_Authentication_Password_UTF8 = Requirement( description=( '[ClickHouse] SHALL support [UTF-8] characters in passwords\n' 'used to authenticate users using an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) + +RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Performance = Requirement( + name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL provide better login performance of [LDAP] authenticated users\n' + 'when `verification_cooldown` parameter is set to a positive value when comparing\n' + 'to the the case when `verification_cooldown` is turned off either for a single user or multiple users\n' + 'making a large number of repeated requests.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters = Requirement( + name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the\n' + '`verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values\n' + 'change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user\n' + "to result in contacting the [LDAP] server to verify user's username and password.\n" + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_InvalidPassword = Requirement( + name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the\n' + '`verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'for the user if the password provided in the current authentication attempt does not match\n' + 'the valid password provided during the first successful authentication request that was cached\n' + 'for this exact user. The reset SHALL cause the next authentication attempt for this user\n' + "to result in contacting the [LDAP] server to verify user's username and password.\n" + '\n' + ), + link=None) diff --git a/tests/testflows/ldap/authentication/tests/server_config.py b/tests/testflows/ldap/authentication/tests/server_config.py index f62fda0bbf7..5b3a96caa9c 100644 --- a/tests/testflows/ldap/authentication/tests/server_config.py +++ b/tests/testflows/ldap/authentication/tests/server_config.py @@ -219,7 +219,7 @@ def auth_dn_value(self): @TestScenario @Requirements( - RQ_SRS_007_LDAP_Configuration_Server_Syntax("1.0") + RQ_SRS_007_LDAP_Configuration_Server_Syntax("2.0") ) def syntax(self): """Check that server configuration with valid syntax can be loaded. @@ -230,6 +230,7 @@ def syntax(self): 636 cn= , ou=users, dc=example, dc=com + 0 yes tls1.2 demand @@ -248,6 +249,7 @@ def syntax(self): "port": "389", "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "0", "enable_tls": "yes", "tls_minimum_protocol_version": "tls1.2" , "tls_require_cert": "demand", diff --git a/tests/testflows/ldap/external_user_directory/regression.py b/tests/testflows/ldap/external_user_directory/regression.py index 6ce860a6fd2..2ad2bd7b1d2 100755 --- a/tests/testflows/ldap/external_user_directory/regression.py +++ b/tests/testflows/ldap/external_user_directory/regression.py @@ -29,9 +29,8 @@ xfails = { @TestFeature @Name("external user directory") @ArgumentParser(argparser) -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication("1.0") -) +@Specifications(SRS_009_ClickHouse_LDAP_External_User_Directory) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication("1.0")) @XFails(xfails) def regression(self, local, clickhouse_binary_path): """ClickHouse LDAP external user directory regression module. diff --git a/tests/testflows/ldap/external_user_directory/requirements/requirements.md b/tests/testflows/ldap/external_user_directory/requirements/requirements.md index 46532c3945d..74248196998 100644 --- a/tests/testflows/ldap/external_user_directory/requirements/requirements.md +++ b/tests/testflows/ldap/external_user_directory/requirements/requirements.md @@ -80,20 +80,22 @@ * 4.2.3.26 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertDir](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlscacertdir) * 4.2.3.27 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertFile](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlscacertfile) * 4.2.3.28 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsciphersuite) - * 4.2.3.29 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationserversyntax) - * 4.2.3.30 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectory) - * 4.2.3.31 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectorymorethanone) - * 4.2.3.32 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationuserssyntax) - * 4.2.3.33 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserver) - * 4.2.3.34 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverempty) - * 4.2.3.35 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermissing) - * 4.2.3.36 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermorethanone) - * 4.2.3.37 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverinvalid) - * 4.2.3.38 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersroles) - * 4.2.3.39 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmorethanone) - * 4.2.3.40 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesinvalid) - * 4.2.3.41 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesempty) - * 4.2.3.42 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmissing) + * 4.2.3.29 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown](#rqsrs-009ldapexternaluserdirectoryconfigurationserververificationcooldown) + * 4.2.3.30 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationserververificationcooldowndefault) + * 4.2.3.31 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationserversyntax) + * 4.2.3.32 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectory) + * 4.2.3.33 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectorymorethanone) + * 4.2.3.34 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationuserssyntax) + * 4.2.3.35 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserver) + * 4.2.3.36 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverempty) + * 4.2.3.37 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermissing) + * 4.2.3.38 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermorethanone) + * 4.2.3.39 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverinvalid) + * 4.2.3.40 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersroles) + * 4.2.3.41 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmorethanone) + * 4.2.3.42 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesinvalid) + * 4.2.3.43 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesempty) + * 4.2.3.44 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmissing) * 4.2.4 [Authentication](#authentication) * 4.2.4.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty](#rqsrs-009ldapexternaluserdirectoryauthenticationusernameempty) * 4.2.4.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long](#rqsrs-009ldapexternaluserdirectoryauthenticationusernamelong) @@ -101,6 +103,9 @@ * 4.2.4.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Empty](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordempty) * 4.2.4.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Long](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordlong) * 4.2.4.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.UTF8](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordutf8) + * 4.2.4.7 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Performance](#rqsrs-009ldapexternaluserdirectoryauthenticationverificationcooldownperformance) + * 4.2.4.8 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters](#rqsrs-009ldapexternaluserdirectoryauthenticationverificationcooldownresetchangeincoreserverparameters) + * 4.2.4.9 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.InvalidPassword](#rqsrs-009ldapexternaluserdirectoryauthenticationverificationcooldownresetinvalidpassword) * 5 [References](#references) ## Revision History @@ -556,9 +561,25 @@ For example, The available suites SHALL depend on the [OpenSSL] library version and variant used to build [ClickHouse] and therefore might change. -##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown version: 1.0 +[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section +that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed +to be successfully authenticated for all consecutive requests without contacting the [LDAP] server. +After period of time since the last successful attempt expires then on the authentication attempt +SHALL result in contacting the [LDAP] server to verify the username and password. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Default +version: 1.0 + +[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section +SHALL have a default value of `0` that disables caching and forces contacting +the [LDAP] server for each authentication request. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax +version: 2.0 + [ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml` configuration file or of any configuration file inside the `config.d` directory. @@ -569,6 +590,7 @@ configuration file or of any configuration file inside the `config.d` directory. 636 cn= , ou=users, dc=example, dc=com + 0 yes tls1.2 demand @@ -717,6 +739,34 @@ version: 1.0 [ClickHouse] SHALL support [UTF-8] characters in passwords used to authenticate users when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Performance +version: 1.0 + +[ClickHouse] SHALL provide better login performance of users authenticated using [LDAP] external user directory +when `verification_cooldown` parameter is set to a positive value when comparing +to the the case when `verification_cooldown` is turned off either for a single user or multiple users +making a large number of repeated requests. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters +version: 1.0 + +[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the +`verification_cooldown` parameter in the [LDAP] server configuration section +if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values +change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user +to result in contacting the [LDAP] server to verify user's username and password. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.InvalidPassword +version: 1.0 + +[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the +`verification_cooldown` parameter in the [LDAP] server configuration section +for the user if the password provided in the current authentication attempt does not match +the valid password provided during the first successful authentication request that was cached +for this exact user. The reset SHALL cause the next authentication attempt for this user +to result in contacting the [LDAP] server to verify user's username and password. + ## References * **Access Control and Account Management**: https://clickhouse.tech/docs/en/operations/access-rights/ diff --git a/tests/testflows/ldap/external_user_directory/requirements/requirements.py b/tests/testflows/ldap/external_user_directory/requirements/requirements.py index 4c4b17d01dc..3354d2b5dd7 100644 --- a/tests/testflows/ldap/external_user_directory/requirements/requirements.py +++ b/tests/testflows/ldap/external_user_directory/requirements/requirements.py @@ -1,10 +1,814 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.201009.1190249. +# document by TestFlows v1.6.201025.1200805. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. +from testflows.core import Specification from testflows.core import Requirement +SRS_009_ClickHouse_LDAP_External_User_Directory = Specification( + name='SRS-009 ClickHouse LDAP External User Directory', + description=None, + author=None, + date=None, + status=None, + approved_by=None, + approved_date=None, + approved_version=None, + version=None, + group=None, + type=None, + link=None, + uid=None, + parent=None, + children=None, + content=''' +# SRS-009 ClickHouse LDAP External User Directory +# Software Requirements Specification + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) + * 3.1 [LDAP](#ldap) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [User Authentication](#user-authentication) + * 4.1.1.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication](#rqsrs-009ldapexternaluserdirectoryauthentication) + * 4.1.1.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories](#rqsrs-009ldapexternaluserdirectorymultipleuserdirectories) + * 4.1.1.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories.Lookup](#rqsrs-009ldapexternaluserdirectorymultipleuserdirectorieslookup) + * 4.1.1.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Authentication.NewUsers](#rqsrs-009ldapexternaluserdirectoryusersauthenticationnewusers) + * 4.1.1.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.DeletedUsers](#rqsrs-009ldapexternaluserdirectoryauthenticationdeletedusers) + * 4.1.1.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Valid](#rqsrs-009ldapexternaluserdirectoryauthenticationvalid) + * 4.1.1.7 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Invalid](#rqsrs-009ldapexternaluserdirectoryauthenticationinvalid) + * 4.1.1.8 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.UsernameChanged](#rqsrs-009ldapexternaluserdirectoryauthenticationusernamechanged) + * 4.1.1.9 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.PasswordChanged](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordchanged) + * 4.1.1.10 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.LDAPServerRestart](#rqsrs-009ldapexternaluserdirectoryauthenticationldapserverrestart) + * 4.1.1.11 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.ClickHouseServerRestart](#rqsrs-009ldapexternaluserdirectoryauthenticationclickhouseserverrestart) + * 4.1.1.12 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel](#rqsrs-009ldapexternaluserdirectoryauthenticationparallel) + * 4.1.1.13 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.ValidAndInvalid](#rqsrs-009ldapexternaluserdirectoryauthenticationparallelvalidandinvalid) + * 4.1.1.14 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.MultipleServers](#rqsrs-009ldapexternaluserdirectoryauthenticationparallelmultipleservers) + * 4.1.1.15 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalOnly](#rqsrs-009ldapexternaluserdirectoryauthenticationparallellocalonly) + * 4.1.1.16 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalAndMultipleLDAP](#rqsrs-009ldapexternaluserdirectoryauthenticationparallellocalandmultipleldap) + * 4.1.1.17 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.SameUser](#rqsrs-009ldapexternaluserdirectoryauthenticationparallelsameuser) + * 4.1.1.18 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.DynamicallyAddedAndRemovedUsers](#rqsrs-009ldapexternaluserdirectoryauthenticationparalleldynamicallyaddedandremovedusers) + * 4.1.2 [Connection](#connection) + * 4.1.2.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.PlainText](#rqsrs-009ldapexternaluserdirectoryconnectionprotocolplaintext) + * 4.1.2.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS](#rqsrs-009ldapexternaluserdirectoryconnectionprotocoltls) + * 4.1.2.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.StartTLS](#rqsrs-009ldapexternaluserdirectoryconnectionprotocolstarttls) + * 4.1.2.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.Validation](#rqsrs-009ldapexternaluserdirectoryconnectionprotocoltlscertificatevalidation) + * 4.1.2.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SelfSigned](#rqsrs-009ldapexternaluserdirectoryconnectionprotocoltlscertificateselfsigned) + * 4.1.2.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SpecificCertificationAuthority](#rqsrs-009ldapexternaluserdirectoryconnectionprotocoltlscertificatespecificcertificationauthority) + * 4.1.2.7 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Anonymous](#rqsrs-009ldapexternaluserdirectoryconnectionauthenticationmechanismanonymous) + * 4.1.2.8 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Unauthenticated](#rqsrs-009ldapexternaluserdirectoryconnectionauthenticationmechanismunauthenticated) + * 4.1.2.9 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.NamePassword](#rqsrs-009ldapexternaluserdirectoryconnectionauthenticationmechanismnamepassword) + * 4.1.2.10 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.UnreachableServer](#rqsrs-009ldapexternaluserdirectoryconnectionauthenticationunreachableserver) + * 4.2 [Specific](#specific) + * 4.2.1 [User Discovery](#user-discovery) + * 4.2.1.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Lookup.Priority](#rqsrs-009ldapexternaluserdirectoryuserslookuppriority) + * 4.2.1.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Restart.Server](#rqsrs-009ldapexternaluserdirectoryrestartserver) + * 4.2.1.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.Restart.Server.ParallelLogins](#rqsrs-009ldapexternaluserdirectoryrestartserverparallellogins) + * 4.2.2 [Roles](#roles) + * 4.2.2.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed](#rqsrs-009ldapexternaluserdirectoryroleremoved) + * 4.2.2.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed.Privileges](#rqsrs-009ldapexternaluserdirectoryroleremovedprivileges) + * 4.2.2.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Readded.Privileges](#rqsrs-009ldapexternaluserdirectoryrolereaddedprivileges) + * 4.2.2.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.New](#rqsrs-009ldapexternaluserdirectoryrolenew) + * 4.2.2.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NewPrivilege](#rqsrs-009ldapexternaluserdirectoryrolenewprivilege) + * 4.2.2.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.RemovedPrivilege](#rqsrs-009ldapexternaluserdirectoryroleremovedprivilege) + * 4.2.3 [Configuration](#configuration) + * 4.2.3.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationserverinvalid) + * 4.2.3.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Definition](#rqsrs-009ldapexternaluserdirectoryconfigurationserverdefinition) + * 4.2.3.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Name](#rqsrs-009ldapexternaluserdirectoryconfigurationservername) + * 4.2.3.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Host](#rqsrs-009ldapexternaluserdirectoryconfigurationserverhost) + * 4.2.3.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port](#rqsrs-009ldapexternaluserdirectoryconfigurationserverport) + * 4.2.3.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationserverportdefault) + * 4.2.3.7 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Prefix](#rqsrs-009ldapexternaluserdirectoryconfigurationserverauthdnprefix) + * 4.2.3.8 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Suffix](#rqsrs-009ldapexternaluserdirectoryconfigurationserverauthdnsuffix) + * 4.2.3.9 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Value](#rqsrs-009ldapexternaluserdirectoryconfigurationserverauthdnvalue) + * 4.2.3.10 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletls) + * 4.2.3.11 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletlsoptionsdefault) + * 4.2.3.12 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.No](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletlsoptionsno) + * 4.2.3.13 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Yes](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletlsoptionsyes) + * 4.2.3.14 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.StartTLS](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletlsoptionsstarttls) + * 4.2.3.15 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsminimumprotocolversion) + * 4.2.3.16 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Values](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsminimumprotocolversionvalues) + * 4.2.3.17 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsminimumprotocolversiondefault) + * 4.2.3.18 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecert) + * 4.2.3.19 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionsdefault) + * 4.2.3.20 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Demand](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionsdemand) + * 4.2.3.21 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Allow](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionsallow) + * 4.2.3.22 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Try](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionstry) + * 4.2.3.23 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Never](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionsnever) + * 4.2.3.24 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCertFile](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlscertfile) + * 4.2.3.25 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSKeyFile](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlskeyfile) + * 4.2.3.26 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertDir](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlscacertdir) + * 4.2.3.27 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertFile](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlscacertfile) + * 4.2.3.28 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsciphersuite) + * 4.2.3.29 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown](#rqsrs-009ldapexternaluserdirectoryconfigurationserververificationcooldown) + * 4.2.3.30 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationserververificationcooldowndefault) + * 4.2.3.31 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationserversyntax) + * 4.2.3.32 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectory) + * 4.2.3.33 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectorymorethanone) + * 4.2.3.34 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationuserssyntax) + * 4.2.3.35 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserver) + * 4.2.3.36 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverempty) + * 4.2.3.37 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermissing) + * 4.2.3.38 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermorethanone) + * 4.2.3.39 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverinvalid) + * 4.2.3.40 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersroles) + * 4.2.3.41 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmorethanone) + * 4.2.3.42 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesinvalid) + * 4.2.3.43 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesempty) + * 4.2.3.44 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmissing) + * 4.2.4 [Authentication](#authentication) + * 4.2.4.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty](#rqsrs-009ldapexternaluserdirectoryauthenticationusernameempty) + * 4.2.4.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long](#rqsrs-009ldapexternaluserdirectoryauthenticationusernamelong) + * 4.2.4.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.UTF8](#rqsrs-009ldapexternaluserdirectoryauthenticationusernameutf8) + * 4.2.4.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Empty](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordempty) + * 4.2.4.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Long](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordlong) + * 4.2.4.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.UTF8](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordutf8) + * 4.2.4.7 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Performance](#rqsrs-009ldapexternaluserdirectoryauthenticationverificationcooldownperformance) + * 4.2.4.8 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters](#rqsrs-009ldapexternaluserdirectoryauthenticationverificationcooldownresetchangeincoreserverparameters) + * 4.2.4.9 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.InvalidPassword](#rqsrs-009ldapexternaluserdirectoryauthenticationverificationcooldownresetinvalidpassword) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Revision History]. + +## Introduction + +The [QA-SRS007 ClickHouse Authentication of Users via LDAP] enables support for authenticating +users using an [LDAP] server. This requirements specifications add addition functionality +for integrating [LDAP] with [ClickHouse]. + +This document will cover requirements to allow authenticatoin of users stored in the +external user discovery using an [LDAP] server without having to explicitly define users in [ClickHouse]'s +`users.xml` configuration file. + +## Terminology + +### LDAP + +* Lightweight Directory Access Protocol + +## Requirements + +### Generic + +#### User Authentication + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication +version: 1.0 + +[ClickHouse] SHALL support authenticating users that are defined only on the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories +version: 1.0 + +[ClickHouse] SHALL support authenticating users using multiple [LDAP] external user directories. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories.Lookup +version: 1.0 + +[ClickHouse] SHALL attempt to authenticate external [LDAP] user +using [LDAP] external user directory in the same order +in which user directories are specified in the `config.xml` file. +If a user cannot be authenticated using the first [LDAP] external user directory +then the next user directory in the list SHALL be used. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Authentication.NewUsers +version: 1.0 + +[ClickHouse] SHALL support authenticating users that are defined only on the [LDAP] server +as soon as they are added to the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.DeletedUsers +version: 1.0 + +[ClickHouse] SHALL not allow authentication of users that +were previously defined only on the [LDAP] server but were removed +from the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Valid +version: 1.0 + +[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if +user name and password match [LDAP] server records for the user +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if either user name or password +do not match [LDAP] server records for the user +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.UsernameChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the username is changed +on the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.PasswordChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the password +for the user is changed on the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.LDAPServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.ClickHouseServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after server is restarted +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users using [LDAP] server +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.ValidAndInvalid +version: 1.0 + +[ClickHouse] SHALL support authentication of valid users and +prohibit authentication of invalid users using [LDAP] server +in parallel without having invalid attempts affecting valid authentications +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.MultipleServers +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of external [LDAP] users +authenticated using multiple [LDAP] external user directories. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalOnly +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users defined only locally +when one or more [LDAP] external user directories are specified in the configuration file. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalAndMultipleLDAP +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of local and external [LDAP] users +authenticated using multiple [LDAP] external user directories. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.SameUser +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of the same external [LDAP] user +authenticated using the same [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.DynamicallyAddedAndRemovedUsers +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users using +[LDAP] external user directory when [LDAP] users are dynamically added and +removed. + +#### Connection + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.PlainText +version: 1.0 + +[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol +while connecting to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol +while connecting to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.StartTLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a +plain text `ldap://` protocol that is upgraded to [TLS] when connecting to the [LDAP] server +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.Validation +version: 1.0 + +[ClickHouse] SHALL support certificate validation used for [TLS] connections +to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SelfSigned +version: 1.0 + +[ClickHouse] SHALL support self-signed certificates for [TLS] connections +to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SpecificCertificationAuthority +version: 1.0 + +[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections +to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Anonymous +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind] +authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Unauthenticated +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind] +authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.NamePassword +version: 1.0 + +[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind] +authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.UnreachableServer +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable +when using [LDAP] external user directory. + +### Specific + +#### User Discovery + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Lookup.Priority +version: 2.0 + +[ClickHouse] SHALL lookup user presence in the same order +as user directories are defined in the `config.xml`. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Restart.Server +version: 1.0 + +[ClickHouse] SHALL support restarting server when one or more LDAP external directories +are configured. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Restart.Server.ParallelLogins +version: 1.0 + +[ClickHouse] SHALL support restarting server when one or more LDAP external directories +are configured during parallel [LDAP] user logins. + +#### Roles + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed +version: 1.0 + +[ClickHouse] SHALL reject authentication attempt if any of the roles that are specified in the configuration +of the external user directory are not defined at the time of the authentication attempt +with an exception that if a user was able to authenticate in past and its internal user object was created and cached +then the user SHALL be able to authenticate again, even if one of the roles is missing. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed.Privileges +version: 1.0 + +[ClickHouse] SHALL remove the privileges provided by the role from all the LDAP +users authenticated using external user directory if it is removed +including currently cached users that are still able to authenticated where the removed +role is specified in the configuration of the external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Readded.Privileges +version: 1.0 + +[ClickHouse] SHALL reassign the role and add the privileges provided by the role +when it is re-added after removal for all LDAP users authenticated using external user directory +including any cached users where the re-added role was specified in the configuration of the external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.New +version: 1.0 + +[ClickHouse] SHALL not allow any new roles to be assigned to any LDAP +users authenticated using external user directory unless the role is specified +in the configuration of the external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NewPrivilege +version: 1.0 + +[ClickHouse] SHALL add new privilege to all the LDAP users authenticated using external user directory +including cached users when new privilege is added to one of the roles specified +in the configuration of the external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.RemovedPrivilege +version: 1.0 + +[ClickHouse] SHALL remove privilege from all the LDAP users authenticated using external user directory +including cached users when privilege is removed from all the roles specified +in the configuration of the external user directory. + +#### Configuration + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Definition +version: 1.0 + +[ClickHouse] SHALL support using the [LDAP] servers defined in the +`ldap_servers` section of the `config.xml` as the server to be used +for a external user directory that uses an [LDAP] server as a source of user definitions. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Name +version: 1.0 + +[ClickHouse] SHALL not support empty string as a server name. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Host +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] +server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] server port. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port.Default +version: 1.0 + +[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Prefix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the prefix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Suffix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the suffix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Value +version: 1.0 + +[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string. + +> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS +version: 1.0 + +[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `yes` value as the default for `` parameter +to enable SSL/TLS `ldaps://` protocol. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.No +version: 1.0 + +[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable +plain text `ldap://` protocol. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Yes +version: 1.0 + +[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable +SSL/TLS `ldaps://` protocol. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.StartTLS +version: 1.0 + +[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable +legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS]. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify +the minimum protocol version of SSL/TLS. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Values +version: 1.0 + +[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2` +as a value of the `` parameter. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Default +version: 1.0 + +[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [TLS] peer +certificate verification behavior. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `demand` value as the default for the `` parameter. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Demand +version: 1.0 + +[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, or a bad certificate is +provided, the session SHALL be immediately terminated. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Allow +version: 1.0 + +[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to +enable requesting of client certificate. If no +certificate is provided, the session SHALL proceed normally. +If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Try +version: 1.0 + +[ClickHouse] SHALL support specifying `try` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, the session +SHALL proceed normally. If a bad certificate is provided, the session SHALL be +immediately terminated. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Never +version: 1.0 + +[ClickHouse] SHALL support specifying `never` as the value of `` parameter to +disable requesting of client certificate. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCertFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to certificate file used by +[ClickHouse] to establish connection with the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSKeyFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to key file for the certificate +specified by the `` parameter. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertDir +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify to a path to +the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertFile +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify a path to a specific +[CA] certificate file used to verify certificates provided by the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite +version: 1.0 + +[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites. +The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers]. + +For example, + +```xml +ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 +``` + +The available suites SHALL depend on the [OpenSSL] library version and variant used to build +[ClickHouse] and therefore might change. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown +version: 1.0 + +[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section +that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed +to be successfully authenticated for all consecutive requests without contacting the [LDAP] server. +After period of time since the last successful attempt expires then on the authentication attempt +SHALL result in contacting the [LDAP] server to verify the username and password. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Default +version: 1.0 + +[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section +SHALL have a default value of `0` that disables caching and forces contacting +the [LDAP] server for each authentication request. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax +version: 2.0 + +[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml` +configuration file or of any configuration file inside the `config.d` directory. + +```xml + + + localhost + 636 + cn= + , ou=users, dc=example, dc=com + 0 + yes + tls1.2 + demand + /path/to/tls_cert_file + /path/to/tls_key_file + /path/to/tls_ca_cert_file + /path/to/tls_ca_cert_dir + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + +``` + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory +version: 1.0 + +[ClickHouse] SHALL support `` sub-section in the `` section of the `config.xml` +that SHALL define a external user directory that uses an [LDAP] server as a source of user definitions. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne +version: 2.0 + +[ClickHouse] SHALL support more than one `` sub-sections in the `` section of the `config.xml` +that SHALL allow to define more than one external user directory that use an [LDAP] server as a source +of user definitions. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax +version: 1.0 + +[ClickHouse] SHALL support `` section with the following syntax + +```xml + + + + my_ldap_server + + + + + + + +``` + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server +version: 1.0 + +[ClickHouse] SHALL support `server` parameter in the `` sub-section in the `` +section of the `config.xml` that SHALL specify one of LDAP server names +defined in `` section. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty +version: 1.0 + +[ClickHouse] SHALL return an error if the `server` parameter in the `` sub-section in the `` +is empty. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing +version: 1.0 + +[ClickHouse] SHALL return an error if the `server` parameter in the `` sub-section in the `` +is missing. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne +version: 1.0 + +[ClickHouse] SHALL only use the first definitition of the `server` parameter in the `` sub-section in the `` +if more than one `server` parameter is defined in the configuration. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the server specified as the value of the `` +parameter is not defined. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles +version: 1.0 + +[ClickHouse] SHALL support `roles` parameter in the `` sub-section in the `` +section of the `config.xml` that SHALL specify the names of a locally defined roles that SHALL +be assigned to all users retrieved from the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne +version: 1.0 + +[ClickHouse] SHALL only use the first definitition of the `roles` parameter +in the `` sub-section in the `` +if more than one `roles` parameter is defined in the configuration. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the role specified in the `` +parameter does not exist locally. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty +version: 1.0 + +[ClickHouse] SHALL not allow users authenticated using LDAP external user directory +to perform any action if the `roles` parameter in the `` sub-section in the `` +section is empty. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing +version: 1.0 + +[ClickHouse] SHALL not allow users authenticated using LDAP external user directory +to perform any action if the `roles` parameter in the `` sub-section in the `` +section is missing. + +#### Authentication + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty username +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long +version: 1.0 + +[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.UTF8 +version: 1.0 + +[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty passwords +even if an empty password is valid for the user and +is allowed by the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Long +version: 1.0 + +[ClickHouse] SHALL support long password of at least 256 bytes +that can be used to authenticate users when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.UTF8 +version: 1.0 + +[ClickHouse] SHALL support [UTF-8] characters in passwords +used to authenticate users when using [LDAP] external user directory. + + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Performance +version: 1.0 + +[ClickHouse] SHALL provide better login performance of users authenticated using [LDAP] external user directory +when `verification_cooldown` parameter is set to a positive value when comparing +to the the case when `verification_cooldown` is turned off either for a single user or multiple users +making a large number of repeated requests. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters +version: 1.0 + +[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the +`verification_cooldown` parameter in the [LDAP] server configuration section +if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values +change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user +to result in contacting the [LDAP] server to verify user's username and password. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.InvalidPassword +version: 1.0 + +[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the +`verification_cooldown` parameter in the [LDAP] server configuration section +for the user if the password provided in the current authentication attempt does not match +the valid password provided during the first successful authentication request that was cached +for this exact user. The reset SHALL cause the next authentication attempt for this user +to result in contacting the [LDAP] server to verify user's username and password. + +## References + +* **Access Control and Account Management**: https://clickhouse.tech/docs/en/operations/access-rights/ +* **LDAP**: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol +* **ClickHouse:** https://clickhouse.tech + +[SRS]: #srs +[Access Control and Account Management]: https://clickhouse.tech/docs/en/operations/access-rights/ +[SRS-007 ClickHouse Authentication of Users via LDAP]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/authentication/requirements/requirements.md +[LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol +[ClickHouse]: https://clickhouse.tech +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/external_user_directory/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/external_user_directory/requirements/requirements.md +[Git]: https://git-scm.com/ +[GitHub]: https://github.com +''') + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication = Requirement( name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication', version='1.0', @@ -940,9 +1744,41 @@ RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCipherSuite = Requ ), link=None) +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed\n' + 'to be successfully authenticated for all consecutive requests without contacting the [LDAP] server.\n' + 'After period of time since the last successful attempt expires then on the authentication attempt\n' + 'SHALL result in contacting the [LDAP] server to verify the username and password. \n' + '\n' + ), + link=None) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown_Default = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'SHALL have a default value of `0` that disables caching and forces contacting\n' + 'the [LDAP] server for each authentication request.\n' + '\n' + ), + link=None) + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Syntax = Requirement( name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax', - version='1.0', + version='2.0', priority=None, group=None, type=None, @@ -958,6 +1794,7 @@ RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Syntax = Requirement( ' 636\n' ' cn=\n' ' , ou=users, dc=example, dc=com\n' + ' 0\n' ' yes\n' ' tls1.2\n' ' demand\n' @@ -1256,5 +2093,57 @@ RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_UTF8 = Requirement '[ClickHouse] SHALL support [UTF-8] characters in passwords\n' 'used to authenticate users when using [LDAP] external user directory.\n' '\n' + '\n' + ), + link=None) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Performance = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Performance', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL provide better login performance of users authenticated using [LDAP] external user directory\n' + 'when `verification_cooldown` parameter is set to a positive value when comparing\n' + 'to the the case when `verification_cooldown` is turned off either for a single user or multiple users\n' + 'making a large number of repeated requests.\n' + '\n' + ), + link=None) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the\n' + '`verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values\n' + 'change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user\n' + "to result in contacting the [LDAP] server to verify user's username and password.\n" + '\n' + ), + link=None) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_InvalidPassword = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.InvalidPassword', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the\n' + '`verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'for the user if the password provided in the current authentication attempt does not match\n' + 'the valid password provided during the first successful authentication request that was cached\n' + 'for this exact user. The reset SHALL cause the next authentication attempt for this user\n' + "to result in contacting the [LDAP] server to verify user's username and password.\n" + '\n' ), link=None) diff --git a/tests/testflows/ldap/external_user_directory/tests/server_config.py b/tests/testflows/ldap/external_user_directory/tests/server_config.py index 5df343b53df..2512a4d88de 100644 --- a/tests/testflows/ldap/external_user_directory/tests/server_config.py +++ b/tests/testflows/ldap/external_user_directory/tests/server_config.py @@ -234,7 +234,7 @@ def auth_dn_value(self): @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Syntax("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Syntax("2.0") ) def syntax(self): """Check that server configuration with valid syntax can be loaded. @@ -245,6 +245,7 @@ def syntax(self): 636 cn= , ou=users, dc=example, dc=com + 0 yes tls1.2 demand @@ -263,6 +264,7 @@ def syntax(self): "port": "389", "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "0", "enable_tls": "yes", "tls_minimum_protocol_version": "tls1.2" , "tls_require_cert": "demand", From a35088d681f8df4071e3427e8565bc928c48f222 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 10 Nov 2020 00:20:34 +0400 Subject: [PATCH 005/284] Add ldap_ prefix to var names --- src/Access/Authentication.cpp | 16 ++++++++-------- src/Access/Authentication.h | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index b8328057161..b55ffad2dfa 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -87,18 +87,18 @@ bool Authentication::isCorrectPassword(const String & password_, const String & ldap_server_params.password = password_; const auto current_params_hash = ldap_server_params.getCoreHash(); - const auto last_check_period = std::chrono::steady_clock::now() - last_successful_password_check_timestamp; + const auto last_check_period = std::chrono::steady_clock::now() - ldap_last_successful_password_check_timestamp; if ( // Forbid the initial values explicitly. - last_successful_password_check_params_hash != 0 && - last_successful_password_check_timestamp != std::chrono::steady_clock::time_point{} && + ldap_last_successful_password_check_params_hash != 0 && + ldap_last_successful_password_check_timestamp != std::chrono::steady_clock::time_point{} && // Check if the caching is enabled at all. ldap_server_params.verification_cooldown > std::chrono::seconds{0} && // Check if we can "reuse" the result of the previous successful password verification. - current_params_hash == last_successful_password_check_params_hash && + current_params_hash == ldap_last_successful_password_check_params_hash && last_check_period >= std::chrono::seconds{0} && last_check_period <= ldap_server_params.verification_cooldown ) @@ -111,13 +111,13 @@ bool Authentication::isCorrectPassword(const String & password_, const String & if (result) { - last_successful_password_check_params_hash = current_params_hash; - last_successful_password_check_timestamp = std::chrono::steady_clock::now(); + ldap_last_successful_password_check_params_hash = current_params_hash; + ldap_last_successful_password_check_timestamp = std::chrono::steady_clock::now(); } else { - last_successful_password_check_params_hash = 0; - last_successful_password_check_timestamp = std::chrono::steady_clock::time_point{}; + ldap_last_successful_password_check_params_hash = 0; + ldap_last_successful_password_check_timestamp = std::chrono::steady_clock::time_point{}; } return result; diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h index 8584c9f7325..65bbcd94720 100644 --- a/src/Access/Authentication.h +++ b/src/Access/Authentication.h @@ -107,8 +107,8 @@ private: // Used and maintained only for LDAP. String server_name; - mutable std::size_t last_successful_password_check_params_hash = 0; - mutable std::chrono::steady_clock::time_point last_successful_password_check_timestamp; + mutable std::size_t ldap_last_successful_password_check_params_hash = 0; + mutable std::chrono::steady_clock::time_point ldap_last_successful_password_check_timestamp; }; From 29c86da543b24dae06cb5c83e012bcdb73ac3147 Mon Sep 17 00:00:00 2001 From: Tai White Date: Tue, 17 Nov 2020 17:32:51 +0100 Subject: [PATCH 006/284] Updated requirements (markdown and python objects), verification_cooldown parameter tests written in authentications.py and server_config.py, helper functions written in common.py --- .../requirements/requirements.md | 610 ++++++ .../requirements/requirements.py | 1687 +++++++++++++++++ ldap/authentication/tests/authentications.py | 969 ++++++++++ ldap/authentication/tests/common.py | 466 +++++ ldap/authentication/tests/server_config.py | 304 +++ 5 files changed, 4036 insertions(+) create mode 100644 ldap/authentication/requirements/requirements.md create mode 100644 ldap/authentication/requirements/requirements.py create mode 100644 ldap/authentication/tests/authentications.py create mode 100644 ldap/authentication/tests/common.py create mode 100644 ldap/authentication/tests/server_config.py diff --git a/ldap/authentication/requirements/requirements.md b/ldap/authentication/requirements/requirements.md new file mode 100644 index 00000000000..17d46584772 --- /dev/null +++ b/ldap/authentication/requirements/requirements.md @@ -0,0 +1,610 @@ +# SRS-007 ClickHouse Authentication of Users via LDAP + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [RQ.SRS-007.LDAP.Authentication](#rqsrs-007ldapauthentication) + * 4.1.2 [RQ.SRS-007.LDAP.Authentication.MultipleServers](#rqsrs-007ldapauthenticationmultipleservers) + * 4.1.3 [RQ.SRS-007.LDAP.Authentication.Protocol.PlainText](#rqsrs-007ldapauthenticationprotocolplaintext) + * 4.1.4 [RQ.SRS-007.LDAP.Authentication.Protocol.TLS](#rqsrs-007ldapauthenticationprotocoltls) + * 4.1.5 [RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS](#rqsrs-007ldapauthenticationprotocolstarttls) + * 4.1.6 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation](#rqsrs-007ldapauthenticationtlscertificatevalidation) + * 4.1.7 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned](#rqsrs-007ldapauthenticationtlscertificateselfsigned) + * 4.1.8 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority](#rqsrs-007ldapauthenticationtlscertificatespecificcertificationauthority) + * 4.1.9 [RQ.SRS-007.LDAP.Server.Configuration.Invalid](#rqsrs-007ldapserverconfigurationinvalid) + * 4.1.10 [RQ.SRS-007.LDAP.User.Configuration.Invalid](#rqsrs-007ldapuserconfigurationinvalid) + * 4.1.11 [RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous](#rqsrs-007ldapauthenticationmechanismanonymous) + * 4.1.12 [RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated](#rqsrs-007ldapauthenticationmechanismunauthenticated) + * 4.1.13 [RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword](#rqsrs-007ldapauthenticationmechanismnamepassword) + * 4.1.14 [RQ.SRS-007.LDAP.Authentication.Valid](#rqsrs-007ldapauthenticationvalid) + * 4.1.15 [RQ.SRS-007.LDAP.Authentication.Invalid](#rqsrs-007ldapauthenticationinvalid) + * 4.1.16 [RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser](#rqsrs-007ldapauthenticationinvaliddeleteduser) + * 4.1.17 [RQ.SRS-007.LDAP.Authentication.UsernameChanged](#rqsrs-007ldapauthenticationusernamechanged) + * 4.1.18 [RQ.SRS-007.LDAP.Authentication.PasswordChanged](#rqsrs-007ldapauthenticationpasswordchanged) + * 4.1.19 [RQ.SRS-007.LDAP.Authentication.LDAPServerRestart](#rqsrs-007ldapauthenticationldapserverrestart) + * 4.1.20 [RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart](#rqsrs-007ldapauthenticationclickhouseserverrestart) + * 4.1.21 [RQ.SRS-007.LDAP.Authentication.Parallel](#rqsrs-007ldapauthenticationparallel) + * 4.1.22 [RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid](#rqsrs-007ldapauthenticationparallelvalidandinvalid) + * 4.2 [Specific](#specific) + * 4.2.1 [RQ.SRS-007.LDAP.UnreachableServer](#rqsrs-007ldapunreachableserver) + * 4.2.2 [RQ.SRS-007.LDAP.Configuration.Server.Name](#rqsrs-007ldapconfigurationservername) + * 4.2.3 [RQ.SRS-007.LDAP.Configuration.Server.Host](#rqsrs-007ldapconfigurationserverhost) + * 4.2.4 [RQ.SRS-007.LDAP.Configuration.Server.Port](#rqsrs-007ldapconfigurationserverport) + * 4.2.5 [RQ.SRS-007.LDAP.Configuration.Server.Port.Default](#rqsrs-007ldapconfigurationserverportdefault) + * 4.2.6 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix](#rqsrs-007ldapconfigurationserverauthdnprefix) + * 4.2.7 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix](#rqsrs-007ldapconfigurationserverauthdnsuffix) + * 4.2.8 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value](#rqsrs-007ldapconfigurationserverauthdnvalue) + * 4.2.9 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS](#rqsrs-007ldapconfigurationserverenabletls) + * 4.2.10 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default](#rqsrs-007ldapconfigurationserverenabletlsoptionsdefault) + * 4.2.11 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No](#rqsrs-007ldapconfigurationserverenabletlsoptionsno) + * 4.2.12 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes](#rqsrs-007ldapconfigurationserverenabletlsoptionsyes) + * 4.2.13 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS](#rqsrs-007ldapconfigurationserverenabletlsoptionsstarttls) + * 4.2.14 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion](#rqsrs-007ldapconfigurationservertlsminimumprotocolversion) + * 4.2.15 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values](#rqsrs-007ldapconfigurationservertlsminimumprotocolversionvalues) + * 4.2.16 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default](#rqsrs-007ldapconfigurationservertlsminimumprotocolversiondefault) + * 4.2.17 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert](#rqsrs-007ldapconfigurationservertlsrequirecert) + * 4.2.18 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdefault) + * 4.2.19 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdemand) + * 4.2.20 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsallow) + * 4.2.21 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try](#rqsrs-007ldapconfigurationservertlsrequirecertoptionstry) + * 4.2.22 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsnever) + * 4.2.23 [RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile](#rqsrs-007ldapconfigurationservertlscertfile) + * 4.2.24 [RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile](#rqsrs-007ldapconfigurationservertlskeyfile) + * 4.2.25 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir](#rqsrs-007ldapconfigurationservertlscacertdir) + * 4.2.26 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile](#rqsrs-007ldapconfigurationservertlscacertfile) + * 4.2.27 [RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite](#rqsrs-007ldapconfigurationservertlsciphersuite) + * 4.2.28 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown](#rqsrs-007ldapconfigurationserververificationcooldown) + * 4.2.29 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default](#rqsrs-007ldapconfigurationserververificationcooldowndefault) + * 4.2.30 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid](#rqsrs-007ldapconfigurationserververificationcooldowninvalid) + * 4.2.31 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) + * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) + * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) + * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) + * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) + * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) + * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) + * 4.2.38 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) + * 4.2.39 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) + * 4.2.40 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) + * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) + * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) + * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) + * 4.2.44 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) + * 4.2.45 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) + * 4.2.46 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) + * 4.2.47 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance](#rqsrs-007ldapauthenticationverificationcooldownperformance) + * 4.2.48 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters](#rqsrs-007ldapauthenticationverificationcooldownresetchangeincoreserverparameters) + * 4.2.49 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword](#rqsrs-007ldapauthenticationverificationcooldownresetinvalidpassword) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Git]'s [Revision History]. + +## Introduction + +[ClickHouse] currently does not have any integration with [LDAP]. +As the initial step in integrating with [LDAP] this software requirements specification covers +only the requirements to enable authentication of users using an [LDAP] server. + +## Terminology + +* **CA** - + Certificate Authority ([CA]) + +* **LDAP** - + Lightweight Directory Access Protocol ([LDAP]) + +## Requirements + +### Generic + +#### RQ.SRS-007.LDAP.Authentication +version: 1.0 + +[ClickHouse] SHALL support user authentication via an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.MultipleServers +version: 1.0 + +[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate +users. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.PlainText +version: 1.0 + +[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.TLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a +plain text `ldap://` protocol that is upgraded to [TLS]. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation +version: 1.0 + +[ClickHouse] SHALL support certificate validation used for [TLS] connections. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned +version: 1.0 + +[ClickHouse] SHALL support self-signed certificates for [TLS] connections. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority +version: 1.0 + +[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections. + +#### RQ.SRS-007.LDAP.Server.Configuration.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid. + +#### RQ.SRS-007.LDAP.User.Configuration.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword +version: 1.0 + +[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Valid +version: 1.0 + +[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if +user name and password match [LDAP] server records for the user. + +#### RQ.SRS-007.LDAP.Authentication.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if either user name or password +do not match [LDAP] server records for the user. + +#### RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the user +has been deleted from the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.UsernameChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the username is changed +on the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.PasswordChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the password +for the user is changed on the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.LDAPServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted. + +#### RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after server is restarted. + +#### RQ.SRS-007.LDAP.Authentication.Parallel +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users using [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid +version: 1.0 + +[ClickHouse] SHALL support authentication of valid users and +prohibit authentication of invalid users using [LDAP] server +in parallel without having invalid attempts affecting valid authentications. + +### Specific + +#### RQ.SRS-007.LDAP.UnreachableServer +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable. + +#### RQ.SRS-007.LDAP.Configuration.Server.Name +version: 1.0 + +[ClickHouse] SHALL not support empty string as a server name. + +#### RQ.SRS-007.LDAP.Configuration.Server.Host +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] +server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty. + +#### RQ.SRS-007.LDAP.Configuration.Server.Port +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] server port. + +#### RQ.SRS-007.LDAP.Configuration.Server.Port.Default +version: 1.0 + +[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the prefix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the suffix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value +version: 1.0 + +[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string. + +> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS +version: 1.0 + +[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `yes` value as the default for `` parameter +to enable SSL/TLS `ldaps://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No +version: 1.0 + +[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable +plain text `ldap://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes +version: 1.0 + +[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable +SSL/TLS `ldaps://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS +version: 1.0 + +[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable +legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS]. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify +the minimum protocol version of SSL/TLS. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values +version: 1.0 + +[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2` +as a value of the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default +version: 1.0 + +[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [TLS] peer +certificate verification behavior. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `demand` value as the default for the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand +version: 1.0 + +[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, or a bad certificate is +provided, the session SHALL be immediately terminated. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow +version: 1.0 + +[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to +enable requesting of client certificate. If no +certificate is provided, the session SHALL proceed normally. +If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try +version: 1.0 + +[ClickHouse] SHALL support specifying `try` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, the session +SHALL proceed normally. If a bad certificate is provided, the session SHALL be +immediately terminated. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never +version: 1.0 + +[ClickHouse] SHALL support specifying `never` as the value of `` parameter to +disable requesting of client certificate. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to certificate file used by +[ClickHouse] to establish connection with the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to key file for the certificate +specified by the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify to a path to +the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify a path to a specific +[CA] certificate file used to verify certificates provided by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite +version: 1.0 + +[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites. +The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers]. + +For example, + +```xml +ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 +``` + +The available suites SHALL depend on the [OpenSSL] library version and variant used to build +[ClickHouse] and therefore might change. + +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown +version: 1.0 + +[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section +that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed +to be successfully authenticated for all consecutive requests without contacting the [LDAP] server. +After period of time since the last successful attempt expires then on the authentication attempt +SHALL result in contacting the [LDAP] server to verify the username and password. + +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default +version: 1.0 + +[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section +SHALL have a default value of `0` that disables caching and forces contacting +the [LDAP] server for each authentication request. + +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid +version: 1.0 + +[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer. + +For example: + +* negative integer +* string +* empty value +* extremely large positive value (overflow) +* extremely large negative value (overflow) + +The error SHALL appear in the log and SHALL be similar to the following: + +```bash + Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value* +``` + +#### RQ.SRS-007.LDAP.Configuration.Server.Syntax +version: 2.0 + +[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml` +configuration file or of any configuration file inside the `config.d` directory. + +```xml + + + localhost + 636 + cn= + , ou=users, dc=example, dc=com + 0 + yes + tls1.2 + demand + /path/to/tls_cert_file + /path/to/tls_key_file + /path/to/tls_ca_cert_file + /path/to/tls_ca_cert_dir + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + +``` + +#### RQ.SRS-007.LDAP.Configuration.User.RBAC +version: 1.0 + +[ClickHouse] SHALL support creating users identified using an [LDAP] server using +the following RBAC command + +```sql +CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name' +``` + +#### RQ.SRS-007.LDAP.Configuration.User.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following example syntax to create a user that is authenticated using +an [LDAP] server inside the `users.xml` file or any configuration file inside the `users.d` directory. + +```xml + + + + + my_ldap_server + + + + +``` + +#### RQ.SRS-007.LDAP.Configuration.User.Name.Empty +version: 1.0 + +[ClickHouse] SHALL not support empty string as a user name. + +#### RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP +version: 1.0 + +[ClickHouse] SHALL throw an error if `` is specified for the user and at the same +time user configuration contains any of the `` entries. + +#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined +version: 1.0 + +[ClickHouse] SHALL throw an error during any authentification attempt +if the name of the [LDAP] server used inside the `` entry +is not defined in the `` section. + +#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty +version: 1.0 + +[ClickHouse] SHALL throw an error during any authentification attempt +if the name of the [LDAP] server used inside the `` entry +is empty. + +#### RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer +version: 1.0 + +[ClickHouse] SHALL support specifying only one [LDAP] server for a given user. + +#### RQ.SRS-007.LDAP.Configuration.User.Name.Long +version: 1.0 + +[ClickHouse] SHALL support long user names of at least 256 bytes +to specify users that can be authenticated using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.User.Name.UTF8 +version: 1.0 + +[ClickHouse] SHALL support user names that contain [UTF-8] characters. + +#### RQ.SRS-007.LDAP.Authentication.Username.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty username. + +#### RQ.SRS-007.LDAP.Authentication.Username.Long +version: 1.0 + +[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes. + +#### RQ.SRS-007.LDAP.Authentication.Username.UTF8 +version: 1.0 + +[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters. + +#### RQ.SRS-007.LDAP.Authentication.Password.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty passwords +even if an empty password is valid for the user and +is allowed by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Password.Long +version: 1.0 + +[ClickHouse] SHALL support long password of at least 256 bytes +that can be used to authenticate users using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Password.UTF8 +version: 1.0 + +[ClickHouse] SHALL support [UTF-8] characters in passwords +used to authenticate users using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance +version: 1.0 + +[ClickHouse] SHALL provide better login performance of [LDAP] authenticated users +when `verification_cooldown` parameter is set to a positive value when comparing +to the the case when `verification_cooldown` is turned off either for a single user or multiple users +making a large number of repeated requests. + +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters +version: 1.0 + +[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the +`verification_cooldown` parameter in the [LDAP] server configuration section +if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values +change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user +to result in contacting the [LDAP] server to verify user's username and password. + +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword +version: 1.0 + +[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the +`verification_cooldown` parameter in the [LDAP] server configuration section +for the user if the password provided in the current authentication attempt does not match +the valid password provided during the first successful authentication request that was cached +for this exact user. The reset SHALL cause the next authentication attempt for this user +to result in contacting the [LDAP] server to verify user's username and password. + +## References + +* **ClickHouse:** https://clickhouse.tech + +[Anonymous Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-AnonymousAuthenticationMechanismOfSimpleBind +[Unauthenticated Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-UnauthenticatedAuthenticationMechanismOfSimpleBind +[Name/Password Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-NamePasswordAuthenticationMechanismOfSimpleBind +[UTF-8]: https://en.wikipedia.org/wiki/UTF-8 +[OpenSSL]: https://www.openssl.org/ +[OpenSSL Ciphers]: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html +[CA]: https://en.wikipedia.org/wiki/Certificate_authority +[TLS]: https://en.wikipedia.org/wiki/Transport_Layer_Security +[LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol +[ClickHouse]: https://clickhouse.tech +[GitHub]: https://github.com +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/authentication/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/authentication/requirements/requirements.md +[Git]: https://git-scm.com/ diff --git a/ldap/authentication/requirements/requirements.py b/ldap/authentication/requirements/requirements.py new file mode 100644 index 00000000000..f934e6c7a99 --- /dev/null +++ b/ldap/authentication/requirements/requirements.py @@ -0,0 +1,1687 @@ +# These requirements were auto generated +# from software requirements specification (SRS) +# document by TestFlows v1.6.201101.1131719. +# Do not edit by hand but re-generate instead +# using 'tfs requirements generate' command. +from testflows.core import Specification +from testflows.core import Requirement + +SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( + name='SRS-007 ClickHouse Authentication of Users via LDAP', + description=None, + author=None, + date=None, + status=None, + approved_by=None, + approved_date=None, + approved_version=None, + version=None, + group=None, + type=None, + link=None, + uid=None, + parent=None, + children=None, + content=''' +# SRS-007 ClickHouse Authentication of Users via LDAP + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [RQ.SRS-007.LDAP.Authentication](#rqsrs-007ldapauthentication) + * 4.1.2 [RQ.SRS-007.LDAP.Authentication.MultipleServers](#rqsrs-007ldapauthenticationmultipleservers) + * 4.1.3 [RQ.SRS-007.LDAP.Authentication.Protocol.PlainText](#rqsrs-007ldapauthenticationprotocolplaintext) + * 4.1.4 [RQ.SRS-007.LDAP.Authentication.Protocol.TLS](#rqsrs-007ldapauthenticationprotocoltls) + * 4.1.5 [RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS](#rqsrs-007ldapauthenticationprotocolstarttls) + * 4.1.6 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation](#rqsrs-007ldapauthenticationtlscertificatevalidation) + * 4.1.7 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned](#rqsrs-007ldapauthenticationtlscertificateselfsigned) + * 4.1.8 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority](#rqsrs-007ldapauthenticationtlscertificatespecificcertificationauthority) + * 4.1.9 [RQ.SRS-007.LDAP.Server.Configuration.Invalid](#rqsrs-007ldapserverconfigurationinvalid) + * 4.1.10 [RQ.SRS-007.LDAP.User.Configuration.Invalid](#rqsrs-007ldapuserconfigurationinvalid) + * 4.1.11 [RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous](#rqsrs-007ldapauthenticationmechanismanonymous) + * 4.1.12 [RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated](#rqsrs-007ldapauthenticationmechanismunauthenticated) + * 4.1.13 [RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword](#rqsrs-007ldapauthenticationmechanismnamepassword) + * 4.1.14 [RQ.SRS-007.LDAP.Authentication.Valid](#rqsrs-007ldapauthenticationvalid) + * 4.1.15 [RQ.SRS-007.LDAP.Authentication.Invalid](#rqsrs-007ldapauthenticationinvalid) + * 4.1.16 [RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser](#rqsrs-007ldapauthenticationinvaliddeleteduser) + * 4.1.17 [RQ.SRS-007.LDAP.Authentication.UsernameChanged](#rqsrs-007ldapauthenticationusernamechanged) + * 4.1.18 [RQ.SRS-007.LDAP.Authentication.PasswordChanged](#rqsrs-007ldapauthenticationpasswordchanged) + * 4.1.19 [RQ.SRS-007.LDAP.Authentication.LDAPServerRestart](#rqsrs-007ldapauthenticationldapserverrestart) + * 4.1.20 [RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart](#rqsrs-007ldapauthenticationclickhouseserverrestart) + * 4.1.21 [RQ.SRS-007.LDAP.Authentication.Parallel](#rqsrs-007ldapauthenticationparallel) + * 4.1.22 [RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid](#rqsrs-007ldapauthenticationparallelvalidandinvalid) + * 4.2 [Specific](#specific) + * 4.2.1 [RQ.SRS-007.LDAP.UnreachableServer](#rqsrs-007ldapunreachableserver) + * 4.2.2 [RQ.SRS-007.LDAP.Configuration.Server.Name](#rqsrs-007ldapconfigurationservername) + * 4.2.3 [RQ.SRS-007.LDAP.Configuration.Server.Host](#rqsrs-007ldapconfigurationserverhost) + * 4.2.4 [RQ.SRS-007.LDAP.Configuration.Server.Port](#rqsrs-007ldapconfigurationserverport) + * 4.2.5 [RQ.SRS-007.LDAP.Configuration.Server.Port.Default](#rqsrs-007ldapconfigurationserverportdefault) + * 4.2.6 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix](#rqsrs-007ldapconfigurationserverauthdnprefix) + * 4.2.7 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix](#rqsrs-007ldapconfigurationserverauthdnsuffix) + * 4.2.8 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value](#rqsrs-007ldapconfigurationserverauthdnvalue) + * 4.2.9 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS](#rqsrs-007ldapconfigurationserverenabletls) + * 4.2.10 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default](#rqsrs-007ldapconfigurationserverenabletlsoptionsdefault) + * 4.2.11 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No](#rqsrs-007ldapconfigurationserverenabletlsoptionsno) + * 4.2.12 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes](#rqsrs-007ldapconfigurationserverenabletlsoptionsyes) + * 4.2.13 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS](#rqsrs-007ldapconfigurationserverenabletlsoptionsstarttls) + * 4.2.14 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion](#rqsrs-007ldapconfigurationservertlsminimumprotocolversion) + * 4.2.15 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values](#rqsrs-007ldapconfigurationservertlsminimumprotocolversionvalues) + * 4.2.16 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default](#rqsrs-007ldapconfigurationservertlsminimumprotocolversiondefault) + * 4.2.17 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert](#rqsrs-007ldapconfigurationservertlsrequirecert) + * 4.2.18 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdefault) + * 4.2.19 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdemand) + * 4.2.20 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsallow) + * 4.2.21 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try](#rqsrs-007ldapconfigurationservertlsrequirecertoptionstry) + * 4.2.22 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsnever) + * 4.2.23 [RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile](#rqsrs-007ldapconfigurationservertlscertfile) + * 4.2.24 [RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile](#rqsrs-007ldapconfigurationservertlskeyfile) + * 4.2.25 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir](#rqsrs-007ldapconfigurationservertlscacertdir) + * 4.2.26 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile](#rqsrs-007ldapconfigurationservertlscacertfile) + * 4.2.27 [RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite](#rqsrs-007ldapconfigurationservertlsciphersuite) + * 4.2.28 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown](#rqsrs-007ldapconfigurationserververificationcooldown) + * 4.2.29 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default](#rqsrs-007ldapconfigurationserververificationcooldowndefault) + * 4.2.30 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid](#rqsrs-007ldapconfigurationserververificationcooldowninvalid) + * 4.2.31 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) + * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) + * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) + * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) + * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) + * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) + * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) + * 4.2.38 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) + * 4.2.39 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) + * 4.2.40 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) + * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) + * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) + * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) + * 4.2.44 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) + * 4.2.45 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) + * 4.2.46 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) + * 4.2.47 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance](#rqsrs-007ldapauthenticationverificationcooldownperformance) + * 4.2.48 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters](#rqsrs-007ldapauthenticationverificationcooldownresetchangeincoreserverparameters) + * 4.2.49 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword](#rqsrs-007ldapauthenticationverificationcooldownresetinvalidpassword) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Git]'s [Revision History]. + +## Introduction + +[ClickHouse] currently does not have any integration with [LDAP]. +As the initial step in integrating with [LDAP] this software requirements specification covers +only the requirements to enable authentication of users using an [LDAP] server. + +## Terminology + +* **CA** - + Certificate Authority ([CA]) + +* **LDAP** - + Lightweight Directory Access Protocol ([LDAP]) + +## Requirements + +### Generic + +#### RQ.SRS-007.LDAP.Authentication +version: 1.0 + +[ClickHouse] SHALL support user authentication via an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.MultipleServers +version: 1.0 + +[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate +users. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.PlainText +version: 1.0 + +[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.TLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a +plain text `ldap://` protocol that is upgraded to [TLS]. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation +version: 1.0 + +[ClickHouse] SHALL support certificate validation used for [TLS] connections. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned +version: 1.0 + +[ClickHouse] SHALL support self-signed certificates for [TLS] connections. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority +version: 1.0 + +[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections. + +#### RQ.SRS-007.LDAP.Server.Configuration.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid. + +#### RQ.SRS-007.LDAP.User.Configuration.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword +version: 1.0 + +[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Valid +version: 1.0 + +[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if +user name and password match [LDAP] server records for the user. + +#### RQ.SRS-007.LDAP.Authentication.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if either user name or password +do not match [LDAP] server records for the user. + +#### RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the user +has been deleted from the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.UsernameChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the username is changed +on the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.PasswordChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the password +for the user is changed on the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.LDAPServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted. + +#### RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after server is restarted. + +#### RQ.SRS-007.LDAP.Authentication.Parallel +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users using [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid +version: 1.0 + +[ClickHouse] SHALL support authentication of valid users and +prohibit authentication of invalid users using [LDAP] server +in parallel without having invalid attempts affecting valid authentications. + +### Specific + +#### RQ.SRS-007.LDAP.UnreachableServer +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable. + +#### RQ.SRS-007.LDAP.Configuration.Server.Name +version: 1.0 + +[ClickHouse] SHALL not support empty string as a server name. + +#### RQ.SRS-007.LDAP.Configuration.Server.Host +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] +server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty. + +#### RQ.SRS-007.LDAP.Configuration.Server.Port +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] server port. + +#### RQ.SRS-007.LDAP.Configuration.Server.Port.Default +version: 1.0 + +[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the prefix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the suffix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value +version: 1.0 + +[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string. + +> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS +version: 1.0 + +[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `yes` value as the default for `` parameter +to enable SSL/TLS `ldaps://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No +version: 1.0 + +[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable +plain text `ldap://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes +version: 1.0 + +[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable +SSL/TLS `ldaps://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS +version: 1.0 + +[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable +legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS]. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify +the minimum protocol version of SSL/TLS. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values +version: 1.0 + +[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2` +as a value of the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default +version: 1.0 + +[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [TLS] peer +certificate verification behavior. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `demand` value as the default for the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand +version: 1.0 + +[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, or a bad certificate is +provided, the session SHALL be immediately terminated. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow +version: 1.0 + +[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to +enable requesting of client certificate. If no +certificate is provided, the session SHALL proceed normally. +If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try +version: 1.0 + +[ClickHouse] SHALL support specifying `try` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, the session +SHALL proceed normally. If a bad certificate is provided, the session SHALL be +immediately terminated. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never +version: 1.0 + +[ClickHouse] SHALL support specifying `never` as the value of `` parameter to +disable requesting of client certificate. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to certificate file used by +[ClickHouse] to establish connection with the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to key file for the certificate +specified by the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify to a path to +the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify a path to a specific +[CA] certificate file used to verify certificates provided by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite +version: 1.0 + +[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites. +The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers]. + +For example, + +```xml +ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 +``` + +The available suites SHALL depend on the [OpenSSL] library version and variant used to build +[ClickHouse] and therefore might change. + +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown +version: 1.0 + +[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section +that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed +to be successfully authenticated for all consecutive requests without contacting the [LDAP] server. +After period of time since the last successful attempt expires then on the authentication attempt +SHALL result in contacting the [LDAP] server to verify the username and password. + +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default +version: 1.0 + +[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section +SHALL have a default value of `0` that disables caching and forces contacting +the [LDAP] server for each authentication request. + +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid +version: 1.0 + +[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer. + +For example: + +* negative integer +* string +* empty value +* extremely large positive value (overflow) +* extremely large negative value (overflow) + +The error SHALL appear in the log and SHALL be similar to the following: + +```bash + Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value* +``` + +#### RQ.SRS-007.LDAP.Configuration.Server.Syntax +version: 2.0 + +[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml` +configuration file or of any configuration file inside the `config.d` directory. + +```xml + + + localhost + 636 + cn= + , ou=users, dc=example, dc=com + 0 + yes + tls1.2 + demand + /path/to/tls_cert_file + /path/to/tls_key_file + /path/to/tls_ca_cert_file + /path/to/tls_ca_cert_dir + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + +``` + +#### RQ.SRS-007.LDAP.Configuration.User.RBAC +version: 1.0 + +[ClickHouse] SHALL support creating users identified using an [LDAP] server using +the following RBAC command + +```sql +CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name' +``` + +#### RQ.SRS-007.LDAP.Configuration.User.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following example syntax to create a user that is authenticated using +an [LDAP] server inside the `users.xml` file or any configuration file inside the `users.d` directory. + +```xml + + + + + my_ldap_server + + + + +``` + +#### RQ.SRS-007.LDAP.Configuration.User.Name.Empty +version: 1.0 + +[ClickHouse] SHALL not support empty string as a user name. + +#### RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP +version: 1.0 + +[ClickHouse] SHALL throw an error if `` is specified for the user and at the same +time user configuration contains any of the `` entries. + +#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined +version: 1.0 + +[ClickHouse] SHALL throw an error during any authentification attempt +if the name of the [LDAP] server used inside the `` entry +is not defined in the `` section. + +#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty +version: 1.0 + +[ClickHouse] SHALL throw an error during any authentification attempt +if the name of the [LDAP] server used inside the `` entry +is empty. + +#### RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer +version: 1.0 + +[ClickHouse] SHALL support specifying only one [LDAP] server for a given user. + +#### RQ.SRS-007.LDAP.Configuration.User.Name.Long +version: 1.0 + +[ClickHouse] SHALL support long user names of at least 256 bytes +to specify users that can be authenticated using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.User.Name.UTF8 +version: 1.0 + +[ClickHouse] SHALL support user names that contain [UTF-8] characters. + +#### RQ.SRS-007.LDAP.Authentication.Username.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty username. + +#### RQ.SRS-007.LDAP.Authentication.Username.Long +version: 1.0 + +[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes. + +#### RQ.SRS-007.LDAP.Authentication.Username.UTF8 +version: 1.0 + +[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters. + +#### RQ.SRS-007.LDAP.Authentication.Password.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty passwords +even if an empty password is valid for the user and +is allowed by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Password.Long +version: 1.0 + +[ClickHouse] SHALL support long password of at least 256 bytes +that can be used to authenticate users using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Password.UTF8 +version: 1.0 + +[ClickHouse] SHALL support [UTF-8] characters in passwords +used to authenticate users using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance +version: 1.0 + +[ClickHouse] SHALL provide better login performance of [LDAP] authenticated users +when `verification_cooldown` parameter is set to a positive value when comparing +to the the case when `verification_cooldown` is turned off either for a single user or multiple users +making a large number of repeated requests. + +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters +version: 1.0 + +[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the +`verification_cooldown` parameter in the [LDAP] server configuration section +if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values +change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user +to result in contacting the [LDAP] server to verify user's username and password. + +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword +version: 1.0 + +[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the +`verification_cooldown` parameter in the [LDAP] server configuration section +for the user if the password provided in the current authentication attempt does not match +the valid password provided during the first successful authentication request that was cached +for this exact user. The reset SHALL cause the next authentication attempt for this user +to result in contacting the [LDAP] server to verify user's username and password. + +## References + +* **ClickHouse:** https://clickhouse.tech + +[Anonymous Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-AnonymousAuthenticationMechanismOfSimpleBind +[Unauthenticated Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-UnauthenticatedAuthenticationMechanismOfSimpleBind +[Name/Password Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-NamePasswordAuthenticationMechanismOfSimpleBind +[UTF-8]: https://en.wikipedia.org/wiki/UTF-8 +[OpenSSL]: https://www.openssl.org/ +[OpenSSL Ciphers]: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html +[CA]: https://en.wikipedia.org/wiki/Certificate_authority +[TLS]: https://en.wikipedia.org/wiki/Transport_Layer_Security +[LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol +[ClickHouse]: https://clickhouse.tech +[GitHub]: https://github.com +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/authentication/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/authentication/requirements/requirements.md +[Git]: https://git-scm.com/ +''') + +RQ_SRS_007_LDAP_Authentication = Requirement( + name='RQ.SRS-007.LDAP.Authentication', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support user authentication via an [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_MultipleServers = Requirement( + name='RQ.SRS-007.LDAP.Authentication.MultipleServers', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate\n' + 'users.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Protocol_PlainText = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Protocol.PlainText', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Protocol_TLS = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Protocol.TLS', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Protocol_StartTLS = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a\n' + 'plain text `ldap://` protocol that is upgraded to [TLS].\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_TLS_Certificate_Validation = Requirement( + name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support certificate validation used for [TLS] connections.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SelfSigned = Requirement( + name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support self-signed certificates for [TLS] connections.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SpecificCertificationAuthority = Requirement( + name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Server_Configuration_Invalid = Requirement( + name='RQ.SRS-007.LDAP.Server.Configuration.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_User_Configuration_Invalid = Requirement( + name='RQ.SRS-007.LDAP.User.Configuration.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Mechanism_Anonymous = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind]\n' + 'authentication mechanism.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Mechanism_Unauthenticated = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind]\n' + 'authentication mechanism.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind]\n' + 'authentication mechanism.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Valid = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Valid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if\n' + 'user name and password match [LDAP] server records for the user.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Invalid = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication if either user name or password\n' + 'do not match [LDAP] server records for the user.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication if the user\n' + 'has been deleted from the [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_UsernameChanged = Requirement( + name='RQ.SRS-007.LDAP.Authentication.UsernameChanged', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication if the username is changed\n' + 'on the [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_PasswordChanged = Requirement( + name='RQ.SRS-007.LDAP.Authentication.PasswordChanged', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication if the password\n' + 'for the user is changed on the [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_LDAPServerRestart = Requirement( + name='RQ.SRS-007.LDAP.Authentication.LDAPServerRestart', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart = Requirement( + name='RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users after server is restarted.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Parallel = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Parallel', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support parallel authentication of users using [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authentication of valid users and\n' + 'prohibit authentication of invalid users using [LDAP] server\n' + 'in parallel without having invalid attempts affecting valid authentications.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_UnreachableServer = Requirement( + name='RQ.SRS-007.LDAP.UnreachableServer', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_Name = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.Name', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support empty string as a server name.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_Host = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.Host', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify [LDAP]\n' + 'server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_Port = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.Port', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify [LDAP] server port.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_Port_Default = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.Port.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Prefix = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify the prefix\n' + 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify the suffix\n' + 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Value = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string.\n' + '\n' + "> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character.\n" + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_EnableTLS = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Default = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use `yes` value as the default for `` parameter\n' + 'to enable SSL/TLS `ldaps://` protocol.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_No = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable\n' + 'plain text `ldap://` protocol.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Yes = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable\n' + 'SSL/TLS `ldaps://` protocol.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_StartTLS = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable\n' + 'legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS].\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify\n' + 'the minimum protocol version of SSL/TLS.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Values = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2`\n' + 'as a value of the `` parameter.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Default = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify [TLS] peer\n' + 'certificate verification behavior.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Default = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use `demand` value as the default for the `` parameter.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Demand = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to\n' + 'enable requesting of client certificate. If no certificate is provided, or a bad certificate is\n' + 'provided, the session SHALL be immediately terminated.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Allow = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to\n' + 'enable requesting of client certificate. If no\n' + 'certificate is provided, the session SHALL proceed normally.\n' + 'If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Try = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `try` as the value of `` parameter to\n' + 'enable requesting of client certificate. If no certificate is provided, the session\n' + 'SHALL proceed normally. If a bad certificate is provided, the session SHALL be\n' + 'immediately terminated.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Never = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `never` as the value of `` parameter to\n' + 'disable requesting of client certificate.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_TLSCertFile = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` to specify the path to certificate file used by\n' + '[ClickHouse] to establish connection with the [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_TLSKeyFile = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` to specify the path to key file for the certificate\n' + 'specified by the `` parameter.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_TLSCACertDir = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify to a path to\n' + 'the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_TLSCACertFile = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify a path to a specific\n' + '[CA] certificate file used to verify certificates provided by the [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites.\n' + 'The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers].\n' + '\n' + 'For example,\n' + '\n' + '```xml\n' + 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n' + '```\n' + '\n' + 'The available suites SHALL depend on the [OpenSSL] library version and variant used to build\n' + '[ClickHouse] and therefore might change.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed\n' + 'to be successfully authenticated for all consecutive requests without contacting the [LDAP] server.\n' + 'After period of time since the last successful attempt expires then on the authentication attempt\n' + 'SHALL result in contacting the [LDAP] server to verify the username and password. \n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Default = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'SHALL have a default value of `0` that disables caching and forces contacting\n' + 'the [LDAP] server for each authentication request.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Invalid = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer.\n' + '\n' + 'For example:\n' + '\n' + '* negative integer\n' + '* string\n' + '* empty value\n' + '* extremely large positive value (overflow)\n' + '* extremely large negative value (overflow)\n' + '\n' + 'The error SHALL appear in the log and SHALL be similar to the following:\n' + '\n' + '```bash\n' + ' Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value*\n' + '```\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.Syntax', + version='2.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml`\n' + 'configuration file or of any configuration file inside the `config.d` directory.\n' + '\n' + '```xml\n' + '\n' + ' \n' + ' localhost\n' + ' 636\n' + ' cn=\n' + ' , ou=users, dc=example, dc=com\n' + ' 0\n' + ' yes\n' + ' tls1.2\n' + ' demand\n' + ' /path/to/tls_cert_file\n' + ' /path/to/tls_key_file\n' + ' /path/to/tls_ca_cert_file\n' + ' /path/to/tls_ca_cert_dir\n' + ' ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n' + ' \n' + '\n' + '```\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_User_RBAC = Requirement( + name='RQ.SRS-007.LDAP.Configuration.User.RBAC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support creating users identified using an [LDAP] server using\n' + 'the following RBAC command\n' + '\n' + '```sql\n' + "CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name'\n" + '```\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_User_Syntax = Requirement( + name='RQ.SRS-007.LDAP.Configuration.User.Syntax', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support the following example syntax to create a user that is authenticated using\n' + 'an [LDAP] server inside the `users.xml` file or any configuration file inside the `users.d` directory.\n' + '\n' + '```xml\n' + '\n' + ' \n' + ' \n' + ' \n' + ' my_ldap_server\n' + ' \n' + ' \n' + ' \n' + '\n' + '```\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_User_Name_Empty = Requirement( + name='RQ.SRS-007.LDAP.Configuration.User.Name.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support empty string as a user name.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP = Requirement( + name='RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL throw an error if `` is specified for the user and at the same\n' + 'time user configuration contains any of the `` entries.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined = Requirement( + name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL throw an error during any authentification attempt\n' + 'if the name of the [LDAP] server used inside the `` entry\n' + 'is not defined in the `` section.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty = Requirement( + name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL throw an error during any authentification attempt\n' + 'if the name of the [LDAP] server used inside the `` entry\n' + 'is empty.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer = Requirement( + name='RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying only one [LDAP] server for a given user.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_User_Name_Long = Requirement( + name='RQ.SRS-007.LDAP.Configuration.User.Name.Long', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support long user names of at least 256 bytes\n' + 'to specify users that can be authenticated using an [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Configuration_User_Name_UTF8 = Requirement( + name='RQ.SRS-007.LDAP.Configuration.User.Name.UTF8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support user names that contain [UTF-8] characters.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Username_Empty = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Username.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support authenticating users with empty username.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Username_Long = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Username.Long', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Username_UTF8 = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Username.UTF8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Password_Empty = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Password.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support authenticating users with empty passwords\n' + 'even if an empty password is valid for the user and\n' + 'is allowed by the [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Password_Long = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Password.Long', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support long password of at least 256 bytes\n' + 'that can be used to authenticate users using an [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_Password_UTF8 = Requirement( + name='RQ.SRS-007.LDAP.Authentication.Password.UTF8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support [UTF-8] characters in passwords\n' + 'used to authenticate users using an [LDAP] server.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Performance = Requirement( + name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL provide better login performance of [LDAP] authenticated users\n' + 'when `verification_cooldown` parameter is set to a positive value when comparing\n' + 'to the the case when `verification_cooldown` is turned off either for a single user or multiple users\n' + 'making a large number of repeated requests.\n' + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters = Requirement( + name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the\n' + '`verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values\n' + 'change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user\n' + "to result in contacting the [LDAP] server to verify user's username and password.\n" + '\n' + ), + link=None) + +RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_InvalidPassword = Requirement( + name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the\n' + '`verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'for the user if the password provided in the current authentication attempt does not match\n' + 'the valid password provided during the first successful authentication request that was cached\n' + 'for this exact user. The reset SHALL cause the next authentication attempt for this user\n' + "to result in contacting the [LDAP] server to verify user's username and password.\n" + '\n' + ), + link=None) diff --git a/ldap/authentication/tests/authentications.py b/ldap/authentication/tests/authentications.py new file mode 100644 index 00000000000..b1a109f87ce --- /dev/null +++ b/ldap/authentication/tests/authentications.py @@ -0,0 +1,969 @@ +# -*- coding: utf-8 -*- +import random +import time + +from multiprocessing.dummy import Pool +from testflows.core import * +from testflows.asserts import error +from ldap.authentication.tests.common import * +from ldap.authentication.requirements import * + +servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + "openldap2": { + "host": "openldap2", + "port": "636", + "enable_tls": "yes", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "never", + } +} + +@TestStep(When) +@Name("I login as {username} and execute query") +@Args(format_name=True) +def login_and_execute_query(self, username, password, exitcode=None, message=None, steps=True): + """Execute query as some user. + """ + self.context.node.query("SELECT 1", + settings=[("user", username), ("password", password)], + exitcode=exitcode or 0, + message=message, steps=steps) + +@TestScenario +def add_user_to_ldap_and_login(self, server, user=None, ch_user=None, login=None, exitcode=None, message=None, rbac=False): + """Add user to LDAP and ClickHouse and then try to login. + """ + self.context.ldap_node = self.context.cluster.node(server) + + if ch_user is None: + ch_user = {} + if login is None: + login = {} + if user is None: + user = {"cn": "myuser", "userpassword": "myuser"} + + with ldap_user(**user) as user: + ch_user["username"] = ch_user.get("username", user["cn"]) + ch_user["server"] = ch_user.get("server", user["_server"]) + + with ldap_authenticated_users(ch_user, config_file=f"ldap_users_{getuid()}.xml", restart=True, rbac=rbac): + username = login.get("username", user["cn"]) + password = login.get("password", user["userpassword"]) + login_and_execute_query(username=username, password=password, exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Parallel("1.0"), + RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid("1.0") +) +def parallel_login(self, server, user_count=10, timeout=200, rbac=False): + """Check that login of valid and invalid LDAP authenticated users works in parallel. + """ + self.context.ldap_node = self.context.cluster.node(server) + user = None + + users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + + with ldap_users(*users): + with ldap_authenticated_users(*[{"username": user["cn"], "server": server} for user in users], rbac=rbac): + + def login_with_valid_username_and_password(users, i, iterations=10): + with When(f"valid users try to login #{i}"): + for i in range(iterations): + random_user = users[random.randint(0, len(users)-1)] + login_and_execute_query(username=random_user["cn"], password=random_user["userpassword"], steps=False) + + def login_with_valid_username_and_invalid_password(users, i, iterations=10): + with When(f"users try to login with valid username and invalid password #{i}"): + for i in range(iterations): + random_user = users[random.randint(0, len(users)-1)] + login_and_execute_query(username=random_user["cn"], + password=(random_user["userpassword"] + randomword(1)), + exitcode=4, + message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", + steps=False) + + def login_with_invalid_username_and_valid_password(users, i, iterations=10): + with When(f"users try to login with invalid username and valid password #{i}"): + for i in range(iterations): + random_user = dict(users[random.randint(0, len(users)-1)]) + random_user["cn"] += randomword(1) + login_and_execute_query(username=random_user["cn"], + password=random_user["userpassword"], + exitcode=4, + message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", + steps=False) + + with When("I login in parallel"): + p = Pool(15) + tasks = [] + for i in range(5): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + + with Then("it should work"): + for task in tasks: + task.get(timeout=timeout) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), + RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser("1.0") +) +def login_after_user_is_deleted_from_ldap(self, server, rbac=False): + """Check that login fails after user is deleted from LDAP. + """ + self.context.ldap_node = self.context.cluster.node(server) + user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) + + with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml", + restart=True, rbac=rbac): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I delete this user from LDAP"): + delete_user_from_ldap(user) + + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), + RQ_SRS_007_LDAP_Authentication_PasswordChanged("1.0") +) +def login_after_user_password_changed_in_ldap(self, server, rbac=False): + """Check that login fails after user password is changed in LDAP. + """ + self.context.ldap_node = self.context.cluster.node(server) + user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) + + with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml", + restart=True, rbac=rbac): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") + + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + + with And("when I try to login with the new password it should work"): + login_and_execute_query(username=user["cn"], password="newpassword") + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), + RQ_SRS_007_LDAP_Authentication_UsernameChanged("1.0") +) +def login_after_user_cn_changed_in_ldap(self, server, rbac=False): + """Check that login fails after user cn is changed in LDAP. + """ + self.context.ldap_node = self.context.cluster.node(server) + user = None + new_user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) + + with ldap_authenticated_users({"username": user["cn"], "server": server}, + config_file=f"ldap_users_{getuid()}.xml", restart=True, rbac=rbac): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I change user password in LDAP"): + new_user = change_user_cn_in_ldap(user, "myuser2") + + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + finally: + with Finally("I make sure LDAP user is deleted"): + if new_user is not None: + delete_user_from_ldap(new_user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Valid("1.0"), + RQ_SRS_007_LDAP_Authentication_LDAPServerRestart("1.0") +) +def login_after_ldap_server_is_restarted(self, server, timeout=60, rbac=False): + """Check that login succeeds after LDAP server is restarted. + """ + self.context.ldap_node = self.context.cluster.node(server) + user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": getuid()} + user = add_user_to_ldap(**user) + + with ldap_authenticated_users({"username": user["cn"], "server": server}, rbac=rbac): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I restart LDAP server"): + self.context.ldap_node.restart() + + with Then("I try to login until it works", description=f"timeout {timeout} sec"): + started = time.time() + while True: + r = self.context.node.query("SELECT 1", + settings=[("user", user["cn"]), ("password", user["userpassword"])], + no_checks=True) + if r.exitcode == 0: + break + assert time.time() - started < timeout, error(r.output) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Valid("1.0"), + RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart("1.0") +) +def login_after_clickhouse_server_is_restarted(self, server, timeout=60, rbac=False): + """Check that login succeeds after ClickHouse server is restarted. + """ + self.context.ldap_node = self.context.cluster.node(server) + user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": getuid()} + user = add_user_to_ldap(**user) + + with ldap_authenticated_users({"username": user["cn"], "server": server}, rbac=rbac): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I restart ClickHouse server"): + self.context.node.restart() + + with Then("I try to login until it works", description=f"timeout {timeout} sec"): + started = time.time() + while True: + r = self.context.node.query("SELECT 1", + settings=[("user", user["cn"]), ("password", user["userpassword"])], + no_checks=True) + if r.exitcode == 0: + break + assert time.time() - started < timeout, error(r.output) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), + RQ_SRS_007_LDAP_Authentication_Password_Empty("1.0") +) +def valid_username_with_valid_empty_password(self, server, rbac=False): + """Check that we can't login using valid username that has empty password. + """ + user = {"cn": "empty_password", "userpassword": ""} + exitcode = 4 + message = f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, exitcode=exitcode, message=message, server=server, rbac=rbac) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), + RQ_SRS_007_LDAP_Authentication_Password_Empty("1.0") +) +def valid_username_and_invalid_empty_password(self, server, rbac=False): + """Check that we can't login using valid username but invalid empty password. + """ + username = "user_non_empty_password" + user = {"cn": username, "userpassword": username} + login = {"password": ""} + + exitcode = 4 + message = f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Valid("1.0") +) +def valid_username_and_password(self, server, rbac=False): + """Check that we can login using valid username and password. + """ + username = "valid_username_and_password" + user = {"cn": username, "userpassword": username} + + with When(f"I add user {username} to LDAP and try to login"): + add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Invalid("1.0") +) +def valid_username_and_password_invalid_server(self, server=None, rbac=False): + """Check that we can't login using valid username and valid + password but for a different server. + """ + self.context.ldap_node = self.context.cluster.node("openldap1") + + user = {"username": "user2", "userpassword": "user2", "server": "openldap1"} + + exitcode = 4 + message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" + + with ldap_authenticated_users(user, config_file=f"ldap_users_{getuid()}.xml", restart=True, rbac=rbac): + login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Valid("1.0"), + RQ_SRS_007_LDAP_Authentication_Username_Long("1.0"), + RQ_SRS_007_LDAP_Configuration_User_Name_Long("1.0") +) +def valid_long_username_and_short_password(self, server, rbac=False): + """Check that we can login using valid very long username and short password. + """ + username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" + user = {"cn": username, "userpassword": "long_username"} + + add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Invalid("1.0") +) +def invalid_long_username_and_valid_short_password(self, server, rbac=False): + """Check that we can't login using slightly invalid long username but valid password. + """ + username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" + user = {"cn": username, "userpassword": "long_username"} + login = {"username": f"{username}?"} + + exitcode = 4 + message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Valid("1.0"), + RQ_SRS_007_LDAP_Authentication_Password_Long("1.0") +) +def valid_short_username_and_long_password(self, server, rbac=False): + """Check that we can login using valid short username with very long password. + """ + username = "long_password" + user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} + add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Invalid("1.0") +) +def valid_short_username_and_invalid_long_password(self, server, rbac=False): + """Check that we can't login using valid short username and invalid long password. + """ + username = "long_password" + user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} + login = {"password": user["userpassword"] + "1"} + + exitcode = 4 + message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Invalid("1.0") +) +def valid_username_and_invalid_password(self, server, rbac=False): + """Check that we can't login using valid username and invalid password. + """ + username = "valid_username_and_invalid_password" + user = {"cn": username, "userpassword": username} + login = {"password": user["userpassword"] + "1"} + + exitcode = 4 + message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Invalid("1.0") +) +def invalid_username_and_valid_password(self, server, rbac=False): + """Check that we can't login using slightly invalid username but valid password. + """ + username = "invalid_username_and_valid_password" + user = {"cn": username, "userpassword": username} + login = {"username": user["cn"] + "1"} + + exitcode = 4 + message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Valid("1.0"), + RQ_SRS_007_LDAP_Authentication_Username_UTF8("1.0"), + RQ_SRS_007_LDAP_Configuration_User_Name_UTF8("1.0") +) +def valid_utf8_username_and_ascii_password(self, server, rbac=False): + """Check that we can login using valid utf-8 username with ascii password. + """ + username = "utf8_username_Gãńdåłf_Thê_Gręât" + user = {"cn": username, "userpassword": "utf8_username"} + + add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Authentication_Valid("1.0"), + RQ_SRS_007_LDAP_Authentication_Password_UTF8("1.0") +) +def valid_ascii_username_and_utf8_password(self, server, rbac=False): + """Check that we can login using valid ascii username with utf-8 password. + """ + username = "utf8_password" + user = {"cn": username, "userpassword": "utf8_password_Gãńdåłf_Thê_Gręât"} + + add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) + +@TestScenario +def empty_username_and_empty_password(self, server=None, rbac=False): + """Check that we can login using empty username and empty password as + it will use the default user and that has an empty password. + """ + login_and_execute_query(username="", password="") + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Default("1.0") +) +def default_verification_cooldown_value(self, server, rbac=False, timeout=20): + """Check that the default value (0) for the verification cooldown parameter + disables caching and forces contacting the LDAP server for each + authentication request. + """ + + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + + with Given("I have an LDAP configuration that uses the default verification_cooldown value (0)"): + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") + + with Then("when I try to login immediately with the old user password it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message) + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0") +) +def valid_verification_cooldown_value_cn_change(self, server, rbac=False, timeout=20): + """Check that we can perform requests without contacting the LDAP server + after successful authentication when the verification_cooldown parameter + is set and the user cn is changed. + """ + + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + new_user = None + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "2" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user cn in LDAP"): + new_user = change_user_cn_in_ldap(user, "testVCD2") + + with Then("when I try to login again with the old user cn it should work"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("when I sleep for 2 seconds and try to log in, it should fail"): + time.sleep(2) + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message) + + finally: + with Finally("I make sure LDAP user is deleted"): + if new_user is not None: + delete_user_from_ldap(new_user, exitcode=None) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0") +) +def valid_verification_cooldown_value_password_change(self, server, rbac=False, timeout=20): + """Check that we can perform requests without contacting the LDAP server + after successful authentication when the verification_cooldown parameter + is set and the user password is changed. + """ + + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "2" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") + + with Then("when I try to login again with the old password it should work"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("when I sleep for 2 seconds and try to log in, it should fail"): + time.sleep(2) + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message) + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0") +) +def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False, timeout=20): + """Check that we can perform requests without contacting the LDAP server + after successful authentication when the verification_cooldown parameter + is set, even when the LDAP server is offline. + """ + + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "2" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add a new user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with ldap_authenticated_users({"username": user["cn"], "server": server}, + config_file=f"ldap_users_{getuid()}.xml"): + + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + try: + with And("then I stop the ldap server"): + self.context.ldap_node.stop() + + with Then("when I try to login again with the server offline it should work"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("when I sleep for 2 seconds and try to log in, it should fail"): + time.sleep(2) + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message) + + finally: + with Finally("I start the ldap server back up"): + self.context.ldap_node.start() + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestOutline +def repeat_requests(self, server, iterations, vcd_value, rbac=False): + """Run repeated requests from some user to the LDAP server. + """ + + user = None + + with Given(f"I have an LDAP configuration that sets verification_cooldown parameter to {vcd_value} sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": vcd_value + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with And("I add a new user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): + with When(f"I login and execute some query {iterations} times"): + start_time = time.time() + r = self.context.node.command(f"time for i in {{1..{iterations}}}; do clickhouse client -q \"SELECT 1\" --user {user['cn']} --password {user['userpassword']} > /dev/null; done") + end_time = time.time() + + return end_time - start_time + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Performance("1.0") +) +def verification_cooldown_performance(self, server, rbac=False, iterations=5000): + """Check that login performance is better when the verification cooldown + parameter is set to a positive value when comparing to the case when + the verification cooldown parameter is turned off. + """ + + vcd_time = 0 + no_vcd_time = 0 + + with Example(f"Repeated requests with verification cooldown parameter set to 600 seconds, {iterations} iterations"): + vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="600", rbac=rbac) + metric("login_with_vcd_value_600", units="seconds", value=vcd_time) + + with Example(f"Repeated requests with verification cooldown parameter set to 0 seconds, {iterations} iterations"): + no_vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="0", rbac=rbac) + metric("login_with_vcd_value_0", units="seconds", value=no_vcd_time) + + with Then("The performance with verification cooldown parameter set is better than the performance with no verification cooldown parameter."): + assert no_vcd_time > vcd_time, error() + + with And("Log the performance improvement as a percentage."): + metric("percentage_improvement", units="%", value=100*(no_vcd_time - vcd_time)/vcd_time) + +@TestOutline +def check_verification_cooldown_reset_on_core_server_parameter_change(self, server, + parameter_name, parameter_value, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after one of the core server + parameters is changed in the LDAP server configuration. + """ + + config_d_dir="/etc/clickhouse-server/config.d" + config_file="ldap_servers.xml" + error_message = "DB::Exception: {user}: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + config=None + updated_config=None + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 600 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "600" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + with And("LDAP authenticated user"): + users = [ + {"cn": f"testVCD_0", "userpassword": "testVCD_0"}, + {"cn": f"testVCD_1", "userpassword": "testVCD_1"} + ] + + with And("I create LDAP servers configuration file"): + config = create_ldap_servers_config_content(servers, config_d_dir, config_file) + + with ldap_users(*users) as users: + with ldap_servers(servers, restart=True): + with ldap_authenticated_users(*[{"username": user["cn"], "server": server} for user in users]): + with When("I login and execute a query"): + for user in users: + with By(f"as user {user['cn']}"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user password in LDAP"): + for user in users: + with By(f"for user {user['cn']}"): + change_user_password_in_ldap(user, "newpassword") + + with And(f"I change the server {parameter_name} core parameter", description=f"{parameter_value}"): + servers["openldap1"][parameter_name] = parameter_value + + with And("I create an updated the config file that has a different server host name"): + updated_config = create_ldap_servers_config_content(servers, config_d_dir, config_file) + + with modify_config(updated_config, restart=False): + with Then("when I try to log in it should fail as cache should have been reset"): + for user in users: + with By(f"as user {user['cn']}"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message.format(user=user["cn"])) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") +) +def verification_cooldown_reset_on_server_host_parameter_change(self, server, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after server host name + is changed in the LDAP server configuration. + """ + + check_verification_cooldown_reset_on_core_server_parameter_change(server=server, + parameter_name="host", parameter_value="openldap2", rbac=rbac) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") +) +def verification_cooldown_reset_on_server_port_parameter_change(self, server, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after server port is changed in the + LDAP server configuration. + """ + + check_verification_cooldown_reset_on_core_server_parameter_change(server=server, + parameter_name="port", parameter_value="9006", rbac=rbac) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") +) +def verification_cooldown_reset_on_server_auth_dn_prefix_parameter_change(self, server, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after server auth_dn_prefix + is changed in the LDAP server configuration. + """ + + check_verification_cooldown_reset_on_core_server_parameter_change(server=server, + parameter_name="auth_dn_prefix", parameter_value="cxx=", rbac=rbac) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") +) +def verification_cooldown_reset_on_server_auth_dn_suffix_parameter_change(self, server, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after server auth_dn_suffix + is changed in the LDAP server configuration. + """ + + check_verification_cooldown_reset_on_core_server_parameter_change(server=server, + parameter_name="auth_dn_suffix", + parameter_value=",ou=company,dc=users,dc=com", rbac=rbac) + + +@TestScenario +@Name("verification cooldown reset when invalid password is provided") +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_InvalidPassword("1.0") +) +def scenario(self, server, rbac=False): + """Check that cached bind requests for the user are discarded when + the user provides invalid login credentials. + """ + + user = None + error_exitcode = 4 + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 600 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "600" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add a new user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with ldap_authenticated_users({"username": user["cn"], "server": server}, + config_file=f"ldap_users_{getuid()}.xml"): + + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") + + with Then("When I try to log in with the cached password it should work"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("When I try to log in with an incorrect password it should fail"): + login_and_execute_query(username=user["cn"], password="incorrect", exitcode=error_exitcode, + message=error_message) + + with And("When I try to log in with the cached password it should fail"): + login_and_execute_query(username=user["cn"], password="incorrect", exitcode=error_exitcode, + message=error_message) + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestFeature +def verification_cooldown(self, rbac, servers=None, node="clickhouse1"): + """Check verification cooldown parameter functionality. + """ + for scenario in loads(current_module(), Scenario, filter=has.tag("verification_cooldown")): + scenario(server="openldap1", rbac=rbac) + + +@TestOutline(Feature) +@Name("user authentications") +@Requirements( + RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword("1.0") +) +@Examples("rbac", [ + (False,), + (True, Requirements(RQ_SRS_007_LDAP_Configuration_User_RBAC("1.0"))) +]) +def feature(self, rbac, servers=None, node="clickhouse1"): + """Check that users can be authenticated using an LDAP server when + users are configured either using an XML configuration file or RBAC. + """ + self.context.node = self.context.cluster.node(node) + + if servers is None: + servers = globals()["servers"] + + with ldap_servers(servers): + for scenario in loads(current_module(), Scenario, filter=~has.tag("verification_cooldown")): + scenario(server="openldap1", rbac=rbac) + + Feature(test=verification_cooldown)(rbac=rbac, servers=servers, node=node) + + + + diff --git a/ldap/authentication/tests/common.py b/ldap/authentication/tests/common.py new file mode 100644 index 00000000000..8efb389a23f --- /dev/null +++ b/ldap/authentication/tests/common.py @@ -0,0 +1,466 @@ +import os +import uuid +import time +import string +import random +import textwrap +import xml.etree.ElementTree as xmltree + +from collections import namedtuple +from contextlib import contextmanager + +import testflows.settings as settings + +from testflows.core import * +from testflows.asserts import error + +def getuid(): + return str(uuid.uuid1()).replace('-', '_') + +xml_with_utf8 = '\n' + +def xml_indent(elem, level=0, by=" "): + i = "\n" + level * by + if len(elem): + if not elem.text or not elem.text.strip(): + elem.text = i + by + if not elem.tail or not elem.tail.strip(): + elem.tail = i + for elem in elem: + xml_indent(elem, level + 1) + if not elem.tail or not elem.tail.strip(): + elem.tail = i + else: + if level and (not elem.tail or not elem.tail.strip()): + elem.tail = i + +def xml_append(root, tag, text): + element = xmltree.Element(tag) + element.text = text + root.append(element) + return element + +Config = namedtuple("Config", "content path name uid preprocessed_name") + +ASCII_CHARS = string.ascii_lowercase + string.ascii_uppercase + string.digits + +def randomword(length, chars=ASCII_CHARS): + return ''.join(random.choice(chars) for i in range(length)) + +def restart(node=None, safe=False, timeout=60): + """Restart ClickHouse server and wait for config to be reloaded. + """ + with When("I restart ClickHouse server node"): + if node is None: + node = current().context.node + + with node.cluster.shell(node.name) as bash: + bash.expect(bash.prompt) + + with By("closing terminal to the node to be restarted"): + bash.close() + + with And("getting current log size"): + logsize = \ + node.command("stat --format=%s /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[ + 0].strip() + + with And("restarting ClickHouse server"): + node.restart(safe=safe) + + with Then("tailing the log file from using previous log size as the offset"): + bash.prompt = bash.__class__.prompt + bash.open() + bash.send(f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log") + + with And("waiting for config reload message in the log file"): + bash.expect( + f"ConfigReloader: Loaded config '/etc/clickhouse-server/config.xml', performed update on configuration", + timeout=timeout) + +def add_config(config, timeout=60, restart=False, modify=False): + """Add dynamic configuration file to ClickHouse. + + :param node: node + :param config: configuration file description + :param timeout: timeout, default: 20 sec + """ + node = current().context.node + + def check_preprocessed_config_is_updated(after_removal=False): + """Check that preprocessed config is updated. + """ + started = time.time() + command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" + + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if after_removal: + if exitcode == 1: + break + else: + if exitcode == 0: + break + time.sleep(1) + + if settings.debug: + node.command(f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name}") + + if after_removal: + assert exitcode == 1, error() + else: + assert exitcode == 0, error() + + def wait_for_config_to_be_loaded(): + """Wait for config to be loaded. + """ + if restart: + with When("I close terminal to the node to be restarted"): + bash.close() + + with And("I stop ClickHouse to apply the config changes"): + node.stop(safe=False) + + with And("I get the current log size"): + cmd = node.cluster.command(None, + f"stat --format=%s {os.environ['CLICKHOUSE_TESTS_DIR']}/_instances/{node.name}/logs/clickhouse-server.log") + logsize = cmd.output.split(" ")[0].strip() + + with And("I start ClickHouse back up"): + node.start() + + with Then("I tail the log file from using previous log size as the offset"): + bash.prompt = bash.__class__.prompt + bash.open() + bash.send(f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log") + + with Then("I wait for config reload message in the log file"): + if restart: + bash.expect( + f"ConfigReloader: Loaded config '/etc/clickhouse-server/config.xml', performed update on configuration", + timeout=timeout) + else: + bash.expect( + f"ConfigReloader: Loaded config '/etc/clickhouse-server/{config.preprocessed_name}', performed update on configuration", + timeout=timeout) + + try: + with Given(f"{config.name}"): + if settings.debug: + with When("I output the content of the config"): + debug(config.content) + + with node.cluster.shell(node.name) as bash: + bash.expect(bash.prompt) + bash.send("tail -n 0 -f /var/log/clickhouse-server/clickhouse-server.log") + + with When("I add the config", description=config.path): + command = f"cat < {config.path}\n{config.content}\nHEREDOC" + node.command(command, steps=False, exitcode=0) + + with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): + check_preprocessed_config_is_updated() + + with And("I wait for config to be reloaded"): + wait_for_config_to_be_loaded() + yield + finally: + if not modify: + with Finally(f"I remove {config.name}"): + with node.cluster.shell(node.name) as bash: + bash.expect(bash.prompt) + bash.send("tail -n 0 -f /var/log/clickhouse-server/clickhouse-server.log") + + with By("removing the config file", description=config.path): + node.command(f"rm -rf {config.path}", exitcode=0) + + with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): + check_preprocessed_config_is_updated(after_removal=True) + + with And("I wait for config to be reloaded"): + wait_for_config_to_be_loaded() + +def create_ldap_servers_config_content(servers, config_d_dir="/etc/clickhouse-server/config.d", config_file="ldap_servers.xml"): + """Create LDAP servers configuration content. + """ + uid = getuid() + path = os.path.join(config_d_dir, config_file) + name = config_file + + root = xmltree.fromstring("") + xml_servers = root.find("ldap_servers") + xml_servers.append(xmltree.Comment(text=f"LDAP servers {uid}")) + + for _name, server in list(servers.items()): + xml_server = xmltree.Element(_name) + for key, value in list(server.items()): + xml_append(xml_server, key, value) + xml_servers.append(xml_server) + + xml_indent(root) + content = xml_with_utf8 + str(xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8") + + return Config(content, path, name, uid, "config.xml") + +@contextmanager +def modify_config(config, restart=False): + """Apply updated configuration file. + """ + return add_config(config, restart=restart, modify=True) + +@contextmanager +def ldap_servers(servers, config_d_dir="/etc/clickhouse-server/config.d", config_file="ldap_servers.xml", + timeout=60, restart=False, config=None): + """Add LDAP servers configuration. + """ + if config is None: + config = create_ldap_servers_config_content(servers, config_d_dir, config_file) + return add_config(config, restart=restart) + +def create_ldap_users_config_content(*users, config_d_dir="/etc/clickhouse-server/users.d", config_file="ldap_users.xml"): + """Create LDAP users configuration file content. + """ + uid = getuid() + path = os.path.join(config_d_dir, config_file) + name = config_file + + root = xmltree.fromstring("") + xml_users = root.find("users") + xml_users.append(xmltree.Comment(text=f"LDAP users {uid}")) + + for user in users: + xml_user = xmltree.Element(user['username']) + xml_user_server = xmltree.Element("ldap") + xml_append(xml_user_server, "server", user["server"]) + xml_user.append(xml_user_server) + xml_users.append(xml_user) + + xml_indent(root) + content = xml_with_utf8 + str(xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8") + + return Config(content, path, name, uid, "users.xml") + +def add_users_identified_with_ldap(*users): + """Add one or more users that are identified via + an ldap server using RBAC. + """ + node = current().context.node + try: + with Given("I create users"): + for user in users: + node.query(f"CREATE USER '{user['username']}' IDENTIFIED WITH ldap_server BY '{user['server']}'") + yield + finally: + with Finally("I remove users"): + for user in users: + with By(f"dropping user {user['username']}", flags=TE): + node.query(f"DROP USER IF EXISTS '{user['username']}'") + +@contextmanager +def ldap_authenticated_users(*users, config_d_dir="/etc/clickhouse-server/users.d", + config_file=None, timeout=60, restart=True, config=None, rbac=False): + """Add LDAP authenticated users. + """ + if rbac: + return add_users_identified_with_ldap(*users) + else: + if config_file is None: + config_file = f"ldap_users_{getuid()}.xml" + if config is None: + config = create_ldap_users_config_content(*users, config_d_dir=config_d_dir, config_file=config_file) + return add_config(config, restart=restart) + +def invalid_server_config(servers, message=None, tail=13, timeout=60): + """Check that ClickHouse errors when trying to load invalid LDAP servers configuration file. + """ + node = current().context.node + if message is None: + message = "Exception: Failed to merge config with '/etc/clickhouse-server/config.d/ldap_servers.xml'" + + config = create_ldap_servers_config_content(servers) + try: + node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) + + with When("I add the config", description=config.path): + command = f"cat < {config.path}\n{config.content}\nHEREDOC" + node.command(command, steps=False, exitcode=0) + + with Then("server shall fail to merge the new config"): + started = time.time() + command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + finally: + with Finally(f"I remove {config.name}"): + with By("removing the config file", description=config.path): + node.command(f"rm -rf {config.path}", exitcode=0) + +def invalid_user_config(servers, config, message=None, tail=13, timeout=60): + """Check that ClickHouse errors when trying to load invalid LDAP users configuration file. + """ + node = current().context.node + if message is None: + message = "Exception: Failed to merge config with '/etc/clickhouse-server/users.d/ldap_users.xml'" + + with ldap_servers(servers): + try: + node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("\\n" * tail)) + with When("I add the config", description=config.path): + command = f"cat < {config.path}\n{config.content}\nHEREDOC" + node.command(command, steps=False, exitcode=0) + + with Then("server shall fail to merge the new config"): + started = time.time() + command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + finally: + with Finally(f"I remove {config.name}"): + with By("removing the config file", description=config.path): + node.command(f"rm -rf {config.path}", exitcode=0) + +def add_user_to_ldap(cn, userpassword, givenname=None, homedirectory=None, sn=None, uid=None, uidnumber=None, node=None): + """Add user entry to LDAP.""" + if node is None: + node = current().context.ldap_node + if uid is None: + uid = cn + if givenname is None: + givenname = "John" + if homedirectory is None: + homedirectory = "/home/users" + if sn is None: + sn = "User" + if uidnumber is None: + uidnumber = 2000 + + user = { + "dn": f"cn={cn},ou=users,dc=company,dc=com", + "cn": cn, + "gidnumber": 501, + "givenname": givenname, + "homedirectory": homedirectory, + "objectclass": ["inetOrgPerson", "posixAccount", "top"], + "sn": sn, + "uid": uid, + "uidnumber": uidnumber, + "userpassword": userpassword, + "_server": node.name + } + + lines = [] + + for key, value in list(user.items()): + if key.startswith("_"): + continue + elif key == "objectclass": + for cls in value: + lines.append(f"objectclass: {cls}") + else: + lines.append(f"{key}: {value}") + + ldif = "\n".join(lines) + + r = node.command( + f"echo -e \"{ldif}\" | ldapadd -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin") + assert r.exitcode == 0, error() + + return user + +def delete_user_from_ldap(user, node=None, exitcode=0): + """Delete user entry from LDAP.""" + if node is None: + node = current().context.ldap_node + r = node.command( + f"ldapdelete -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin \"{user['dn']}\"") + if exitcode is not None: + assert r.exitcode == exitcode, error() + +def change_user_password_in_ldap(user, new_password, node=None, exitcode=0): + """Change user password in LDAP.""" + if node is None: + node = current().context.ldap_node + + ldif = (f"dn: {user['dn']}\n" + "changetype: modify\n" + "replace: userpassword\n" + f"userpassword: {new_password}") + + r = node.command( + f"echo -e \"{ldif}\" | ldapmodify -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin") + + if exitcode is not None: + assert r.exitcode == exitcode, error() + +def change_user_cn_in_ldap(user, new_cn, node=None, exitcode=0): + """Change user password in LDAP.""" + if node is None: + node = current().context.ldap_node + + new_user = dict(user) + new_user['dn'] = f"cn={new_cn},ou=users,dc=company,dc=com" + new_user['cn'] = new_cn + + ldif = ( + f"dn: {user['dn']}\n" + "changetype: modrdn\n" + f"newrdn: cn = {new_user['cn']}\n" + f"deleteoldrdn: 1\n" + ) + + r = node.command( + f"echo -e \"{ldif}\" | ldapmodify -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin") + + if exitcode is not None: + assert r.exitcode == exitcode, error() + + return new_user + +@contextmanager +def ldap_user(cn, userpassword, givenname=None, homedirectory=None, sn=None, uid=None, uidnumber=None, node=None): + """Add new user to the LDAP server.""" + try: + user = None + with Given(f"I add user {cn} to LDAP"): + user = add_user_to_ldap(cn, userpassword, givenname, homedirectory, sn, uid, uidnumber, node=node) + yield user + finally: + with Finally(f"I delete user {cn} from LDAP"): + if user is not None: + delete_user_from_ldap(user, node=node) + +@contextmanager +def ldap_users(*users, node=None): + """Add multiple new users to the LDAP server.""" + try: + _users = [] + with Given("I add users to LDAP"): + for user in users: + with By(f"adding user {user['cn']}"): + _users.append(add_user_to_ldap(**user, node=node)) + yield _users + finally: + with Finally(f"I delete users from LDAP"): + for _user in _users: + delete_user_from_ldap(_user, node=node) + +def login(servers, *users, config=None): + """Configure LDAP server and LDAP authenticated users and + try to login and execute a query""" + with ldap_servers(servers): + with ldap_authenticated_users(*users, restart=True, config=config): + for user in users: + if user.get("login", False): + with When(f"I login as {user['username']} and execute query"): + current().context.node.query("SELECT 1", + settings=[("user", user["username"]), ("password", user["password"])], + exitcode=user.get("exitcode", None), + message=user.get("message", None)) diff --git a/ldap/authentication/tests/server_config.py b/ldap/authentication/tests/server_config.py new file mode 100644 index 00000000000..38ec859226b --- /dev/null +++ b/ldap/authentication/tests/server_config.py @@ -0,0 +1,304 @@ +from testflows.core import * + +from ldap.authentication.tests.common import * +from ldap.authentication.requirements import * + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0"), + RQ_SRS_007_LDAP_Configuration_Server_Name("1.0") +) +def empty_server_name(self, timeout=20): + """Check that empty string as a server name is not allowed. + """ + servers = {"": {"host": "foo", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + invalid_server_config(servers, timeout=timeout) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0"), + RQ_SRS_007_LDAP_UnreachableServer("1.0") +) +def invalid_host(self): + """Check that server returns an error when LDAP server + host name is invalid. + """ + servers = {"foo": {"host": "foo", "port": "389", "enable_tls": "no"}} + users = [{ + "server": "foo", "username": "user1", "password": "user1", "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" + }] + login(servers, *users) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0"), + RQ_SRS_007_LDAP_Configuration_Server_Host("1.0") +) +def empty_host(self): + """Check that server returns an error when LDAP server + host value is empty. + """ + servers = {"foo": {"host": "", "port": "389", "enable_tls": "no"}} + users = [{ + "server": "foo", "username": "user1", "password": "user1", "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" + }] + login(servers, *users) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0"), + RQ_SRS_007_LDAP_Configuration_Server_Host("1.0") +) +def missing_host(self): + """Check that server returns an error when LDAP server + host is missing. + """ + servers = {"foo": {"port": "389", "enable_tls": "no"}} + users = [{ + "server": "foo", "username": "user1", "password": "user1", "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" + }] + login(servers, *users) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") +) +def invalid_port(self): + """Check that server returns an error when LDAP server + port is not valid. + """ + servers = {"openldap1": {"host": "openldap1", "port": "3890", "enable_tls": "no"}} + users = [{ + "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" + }] + login(servers, *users) + + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") +) +def invalid_auth_dn_prefix(self): + """Check that server returns an error when LDAP server + port is not valid. + """ + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "foo=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + users = [{ + "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" + }] + login(servers, *users) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") +) +def invalid_auth_dn_suffix(self): + """Check that server returns an error when LDAP server + port is not valid. + """ + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",foo=users,dc=company,dc=com" + }} + users = [{ + "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" + }] + login(servers, *users) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") +) +def invalid_enable_tls_value(self): + """Check that server returns an error when enable_tls + option has invalid value. + """ + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "foo", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + users = [{ + "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" + }] + login(servers, *users) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") +) +def invalid_tls_require_cert_value(self): + """Check that server returns an error when tls_require_cert + option has invalid value. + """ + servers = {"openldap2": { + "host": "openldap2", "port": "636", "enable_tls": "yes", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "foo", + "ca_cert_dir": "/container/service/slapd/assets/certs/", + "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt" + }} + users = [{ + "server": "openldap2", "username": "user2", "password": "user2", "login": True, + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" + }] + login(servers, *users) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") +) +def empty_ca_cert_dir(self): + """Check that server returns an error when ca_cert_dir is empty. + """ + servers = {"openldap2": {"host": "openldap2", "port": "636", "enable_tls": "yes", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "demand", + "ca_cert_dir": "", + "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt" + }} + users = [{ + "server": "openldap2", "username": "user2", "password": "user2", "login": True, + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" + }] + login(servers, *users) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") +) +def empty_ca_cert_file(self): + """Check that server returns an error when ca_cert_file is empty. + """ + servers = {"openldap2": {"host": "openldap2", "port": "636", "enable_tls": "yes", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "demand", + "ca_cert_dir": "/container/service/slapd/assets/certs/", + "ca_cert_file": "" + }} + users = [{ + "server": "openldap2", "username": "user2", "password": "user2", "login": True, + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" + }] + login(servers, *users) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Value("1.0"), + RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Prefix("1.0"), + RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix("1.0") +) +def auth_dn_value(self): + """Check that server configuration can properly define the `dn` value of the user.""" + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + + login(servers, user) + +@TestOutline(Scenario) +@Examples("invalid_value", [ + ("-1", Name("negative int")), + ("foo", Name("string")), + ("", Name("empty string")), + ("36893488147419103232", Name("overflow with extremely large int value")), + ("-36893488147419103232", Name("overflow with extremely large negative int value")), + ("@#", Name("special characters")) +]) +@Requirements( + RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Invalid("1.0") +) +def invalid_verification_cooldown_value(self, invalid_value, timeout=20): + """Check that server returns an error when LDAP server + verification cooldown parameter is invalid. + """ + + error_message = (" Access(user directories): Could not parse LDAP server" + " \\`openldap1\\`: Poco::Exception. Code: 1000, e.code() = 0," + f" e.displayText() = Syntax error: Not a valid unsigned integer{': ' + invalid_value if invalid_value else invalid_value}") + + with Given("LDAP server configuration that uses a negative integer for the verification_cooldown parameter"): + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": f"{invalid_value}" + }} + + with When("I try to use this configuration then it should not work"): + invalid_server_config(servers, message=error_message, tail=17, timeout=timeout) + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Configuration_Server_Syntax("2.0") +) +def syntax(self): + """Check that server configuration with valid syntax can be loaded. + ```xml + + + localhost + 636 + cn= + , ou=users, dc=example, dc=com + 0 + yes + tls1.2 + demand + /path/to/tls_cert_file + /path/to/tls_key_file + /path/to/tls_ca_cert_file + /path/to/tls_ca_cert_dir + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + + ``` + """ + servers = { + "openldap2": { + "host": "openldap2", + "port": "389", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "0", + "enable_tls": "yes", + "tls_minimum_protocol_version": "tls1.2" , + "tls_require_cert": "demand", + "tls_cert_file": "/container/service/slapd/assets/certs/ldap.crt", + "tls_key_file": "/container/service/slapd/assets/certs/ldap.key", + "tls_ca_cert_file": "/container/service/slapd/assets/certs/ca.crt", + "tls_ca_cert_dir": "/container/service/slapd/assets/certs/", + "tls_cipher_suite": "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384" + } + } + with ldap_servers(servers): + pass + +@TestFeature +@Name("server config") +def feature(self, node="clickhouse1"): + """Check that LDAP server configuration. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + scenario() From e42ea7b8b935fce14ce41a07c6c4e7fdfa9a7266 Mon Sep 17 00:00:00 2001 From: Tai White Date: Tue, 17 Nov 2020 19:41:28 +0100 Subject: [PATCH 007/284] Deleted ldap directory added by mistake. Added verification cooldown tests to the ldap/authentication feature. --- .../requirements/requirements.md | 610 ------ .../requirements/requirements.py | 1687 ----------------- ldap/authentication/tests/authentications.py | 969 ---------- ldap/authentication/tests/common.py | 466 ----- ldap/authentication/tests/server_config.py | 304 --- tests/testflows/helpers/cluster.py | 17 + .../ldap/authentication/regression.py | 5 +- .../requirements/requirements.md | 58 +- .../requirements/requirements.py | 85 +- .../authentication/tests/authentications.py | 478 ++++- .../ldap/authentication/tests/common.py | 34 +- .../authentication/tests/server_config.py | 30 + 12 files changed, 651 insertions(+), 4092 deletions(-) delete mode 100644 ldap/authentication/requirements/requirements.md delete mode 100644 ldap/authentication/requirements/requirements.py delete mode 100644 ldap/authentication/tests/authentications.py delete mode 100644 ldap/authentication/tests/common.py delete mode 100644 ldap/authentication/tests/server_config.py diff --git a/ldap/authentication/requirements/requirements.md b/ldap/authentication/requirements/requirements.md deleted file mode 100644 index 17d46584772..00000000000 --- a/ldap/authentication/requirements/requirements.md +++ /dev/null @@ -1,610 +0,0 @@ -# SRS-007 ClickHouse Authentication of Users via LDAP - -## Table of Contents - -* 1 [Revision History](#revision-history) -* 2 [Introduction](#introduction) -* 3 [Terminology](#terminology) -* 4 [Requirements](#requirements) - * 4.1 [Generic](#generic) - * 4.1.1 [RQ.SRS-007.LDAP.Authentication](#rqsrs-007ldapauthentication) - * 4.1.2 [RQ.SRS-007.LDAP.Authentication.MultipleServers](#rqsrs-007ldapauthenticationmultipleservers) - * 4.1.3 [RQ.SRS-007.LDAP.Authentication.Protocol.PlainText](#rqsrs-007ldapauthenticationprotocolplaintext) - * 4.1.4 [RQ.SRS-007.LDAP.Authentication.Protocol.TLS](#rqsrs-007ldapauthenticationprotocoltls) - * 4.1.5 [RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS](#rqsrs-007ldapauthenticationprotocolstarttls) - * 4.1.6 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation](#rqsrs-007ldapauthenticationtlscertificatevalidation) - * 4.1.7 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned](#rqsrs-007ldapauthenticationtlscertificateselfsigned) - * 4.1.8 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority](#rqsrs-007ldapauthenticationtlscertificatespecificcertificationauthority) - * 4.1.9 [RQ.SRS-007.LDAP.Server.Configuration.Invalid](#rqsrs-007ldapserverconfigurationinvalid) - * 4.1.10 [RQ.SRS-007.LDAP.User.Configuration.Invalid](#rqsrs-007ldapuserconfigurationinvalid) - * 4.1.11 [RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous](#rqsrs-007ldapauthenticationmechanismanonymous) - * 4.1.12 [RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated](#rqsrs-007ldapauthenticationmechanismunauthenticated) - * 4.1.13 [RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword](#rqsrs-007ldapauthenticationmechanismnamepassword) - * 4.1.14 [RQ.SRS-007.LDAP.Authentication.Valid](#rqsrs-007ldapauthenticationvalid) - * 4.1.15 [RQ.SRS-007.LDAP.Authentication.Invalid](#rqsrs-007ldapauthenticationinvalid) - * 4.1.16 [RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser](#rqsrs-007ldapauthenticationinvaliddeleteduser) - * 4.1.17 [RQ.SRS-007.LDAP.Authentication.UsernameChanged](#rqsrs-007ldapauthenticationusernamechanged) - * 4.1.18 [RQ.SRS-007.LDAP.Authentication.PasswordChanged](#rqsrs-007ldapauthenticationpasswordchanged) - * 4.1.19 [RQ.SRS-007.LDAP.Authentication.LDAPServerRestart](#rqsrs-007ldapauthenticationldapserverrestart) - * 4.1.20 [RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart](#rqsrs-007ldapauthenticationclickhouseserverrestart) - * 4.1.21 [RQ.SRS-007.LDAP.Authentication.Parallel](#rqsrs-007ldapauthenticationparallel) - * 4.1.22 [RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid](#rqsrs-007ldapauthenticationparallelvalidandinvalid) - * 4.2 [Specific](#specific) - * 4.2.1 [RQ.SRS-007.LDAP.UnreachableServer](#rqsrs-007ldapunreachableserver) - * 4.2.2 [RQ.SRS-007.LDAP.Configuration.Server.Name](#rqsrs-007ldapconfigurationservername) - * 4.2.3 [RQ.SRS-007.LDAP.Configuration.Server.Host](#rqsrs-007ldapconfigurationserverhost) - * 4.2.4 [RQ.SRS-007.LDAP.Configuration.Server.Port](#rqsrs-007ldapconfigurationserverport) - * 4.2.5 [RQ.SRS-007.LDAP.Configuration.Server.Port.Default](#rqsrs-007ldapconfigurationserverportdefault) - * 4.2.6 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix](#rqsrs-007ldapconfigurationserverauthdnprefix) - * 4.2.7 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix](#rqsrs-007ldapconfigurationserverauthdnsuffix) - * 4.2.8 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value](#rqsrs-007ldapconfigurationserverauthdnvalue) - * 4.2.9 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS](#rqsrs-007ldapconfigurationserverenabletls) - * 4.2.10 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default](#rqsrs-007ldapconfigurationserverenabletlsoptionsdefault) - * 4.2.11 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No](#rqsrs-007ldapconfigurationserverenabletlsoptionsno) - * 4.2.12 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes](#rqsrs-007ldapconfigurationserverenabletlsoptionsyes) - * 4.2.13 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS](#rqsrs-007ldapconfigurationserverenabletlsoptionsstarttls) - * 4.2.14 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion](#rqsrs-007ldapconfigurationservertlsminimumprotocolversion) - * 4.2.15 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values](#rqsrs-007ldapconfigurationservertlsminimumprotocolversionvalues) - * 4.2.16 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default](#rqsrs-007ldapconfigurationservertlsminimumprotocolversiondefault) - * 4.2.17 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert](#rqsrs-007ldapconfigurationservertlsrequirecert) - * 4.2.18 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdefault) - * 4.2.19 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdemand) - * 4.2.20 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsallow) - * 4.2.21 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try](#rqsrs-007ldapconfigurationservertlsrequirecertoptionstry) - * 4.2.22 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsnever) - * 4.2.23 [RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile](#rqsrs-007ldapconfigurationservertlscertfile) - * 4.2.24 [RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile](#rqsrs-007ldapconfigurationservertlskeyfile) - * 4.2.25 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir](#rqsrs-007ldapconfigurationservertlscacertdir) - * 4.2.26 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile](#rqsrs-007ldapconfigurationservertlscacertfile) - * 4.2.27 [RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite](#rqsrs-007ldapconfigurationservertlsciphersuite) - * 4.2.28 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown](#rqsrs-007ldapconfigurationserververificationcooldown) - * 4.2.29 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default](#rqsrs-007ldapconfigurationserververificationcooldowndefault) - * 4.2.30 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid](#rqsrs-007ldapconfigurationserververificationcooldowninvalid) - * 4.2.31 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) - * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) - * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) - * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) - * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) - * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) - * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) - * 4.2.38 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) - * 4.2.39 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) - * 4.2.40 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) - * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) - * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) - * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) - * 4.2.44 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) - * 4.2.45 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) - * 4.2.46 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) - * 4.2.47 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance](#rqsrs-007ldapauthenticationverificationcooldownperformance) - * 4.2.48 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters](#rqsrs-007ldapauthenticationverificationcooldownresetchangeincoreserverparameters) - * 4.2.49 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword](#rqsrs-007ldapauthenticationverificationcooldownresetinvalidpassword) -* 5 [References](#references) - -## Revision History - -This document is stored in an electronic form using [Git] source control management software -hosted in a [GitHub Repository]. -All the updates are tracked using the [Git]'s [Revision History]. - -## Introduction - -[ClickHouse] currently does not have any integration with [LDAP]. -As the initial step in integrating with [LDAP] this software requirements specification covers -only the requirements to enable authentication of users using an [LDAP] server. - -## Terminology - -* **CA** - - Certificate Authority ([CA]) - -* **LDAP** - - Lightweight Directory Access Protocol ([LDAP]) - -## Requirements - -### Generic - -#### RQ.SRS-007.LDAP.Authentication -version: 1.0 - -[ClickHouse] SHALL support user authentication via an [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.MultipleServers -version: 1.0 - -[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate -users. - -#### RQ.SRS-007.LDAP.Authentication.Protocol.PlainText -version: 1.0 - -[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol. - -#### RQ.SRS-007.LDAP.Authentication.Protocol.TLS -version: 1.0 - -[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol. - -#### RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS -version: 1.0 - -[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a -plain text `ldap://` protocol that is upgraded to [TLS]. - -#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation -version: 1.0 - -[ClickHouse] SHALL support certificate validation used for [TLS] connections. - -#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned -version: 1.0 - -[ClickHouse] SHALL support self-signed certificates for [TLS] connections. - -#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority -version: 1.0 - -[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections. - -#### RQ.SRS-007.LDAP.Server.Configuration.Invalid -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid. - -#### RQ.SRS-007.LDAP.User.Configuration.Invalid -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid. - -#### RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind] -authentication mechanism. - -#### RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind] -authentication mechanism. - -#### RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword -version: 1.0 - -[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind] -authentication mechanism. - -#### RQ.SRS-007.LDAP.Authentication.Valid -version: 1.0 - -[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if -user name and password match [LDAP] server records for the user. - -#### RQ.SRS-007.LDAP.Authentication.Invalid -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit authentication if either user name or password -do not match [LDAP] server records for the user. - -#### RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit authentication if the user -has been deleted from the [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.UsernameChanged -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit authentication if the username is changed -on the [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.PasswordChanged -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit authentication if the password -for the user is changed on the [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.LDAPServerRestart -version: 1.0 - -[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted. - -#### RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart -version: 1.0 - -[ClickHouse] SHALL support authenticating users after server is restarted. - -#### RQ.SRS-007.LDAP.Authentication.Parallel -version: 1.0 - -[ClickHouse] SHALL support parallel authentication of users using [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid -version: 1.0 - -[ClickHouse] SHALL support authentication of valid users and -prohibit authentication of invalid users using [LDAP] server -in parallel without having invalid attempts affecting valid authentications. - -### Specific - -#### RQ.SRS-007.LDAP.UnreachableServer -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable. - -#### RQ.SRS-007.LDAP.Configuration.Server.Name -version: 1.0 - -[ClickHouse] SHALL not support empty string as a server name. - -#### RQ.SRS-007.LDAP.Configuration.Server.Host -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify [LDAP] -server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty. - -#### RQ.SRS-007.LDAP.Configuration.Server.Port -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify [LDAP] server port. - -#### RQ.SRS-007.LDAP.Configuration.Server.Port.Default -version: 1.0 - -[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise. - -#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify the prefix -of value used to construct the DN to bound to during authentication via [LDAP] server. - -#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify the suffix -of value used to construct the DN to bound to during authentication via [LDAP] server. - -#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value -version: 1.0 - -[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string. - -> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character. - -#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS -version: 1.0 - -[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server. - -#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default -version: 1.0 - -[ClickHouse] SHALL use `yes` value as the default for `` parameter -to enable SSL/TLS `ldaps://` protocol. - -#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No -version: 1.0 - -[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable -plain text `ldap://` protocol. - -#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes -version: 1.0 - -[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable -SSL/TLS `ldaps://` protocol. - -#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS -version: 1.0 - -[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable -legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS]. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify -the minimum protocol version of SSL/TLS. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values -version: 1.0 - -[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2` -as a value of the `` parameter. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default -version: 1.0 - -[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify [TLS] peer -certificate verification behavior. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default -version: 1.0 - -[ClickHouse] SHALL use `demand` value as the default for the `` parameter. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand -version: 1.0 - -[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to -enable requesting of client certificate. If no certificate is provided, or a bad certificate is -provided, the session SHALL be immediately terminated. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow -version: 1.0 - -[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to -enable requesting of client certificate. If no -certificate is provided, the session SHALL proceed normally. -If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try -version: 1.0 - -[ClickHouse] SHALL support specifying `try` as the value of `` parameter to -enable requesting of client certificate. If no certificate is provided, the session -SHALL proceed normally. If a bad certificate is provided, the session SHALL be -immediately terminated. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never -version: 1.0 - -[ClickHouse] SHALL support specifying `never` as the value of `` parameter to -disable requesting of client certificate. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile -version: 1.0 - -[ClickHouse] SHALL support `` to specify the path to certificate file used by -[ClickHouse] to establish connection with the [LDAP] server. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile -version: 1.0 - -[ClickHouse] SHALL support `` to specify the path to key file for the certificate -specified by the `` parameter. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify to a path to -the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify a path to a specific -[CA] certificate file used to verify certificates provided by the [LDAP] server. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite -version: 1.0 - -[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites. -The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers]. - -For example, - -```xml -ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 -``` - -The available suites SHALL depend on the [OpenSSL] library version and variant used to build -[ClickHouse] and therefore might change. - -#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown -version: 1.0 - -[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section -that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed -to be successfully authenticated for all consecutive requests without contacting the [LDAP] server. -After period of time since the last successful attempt expires then on the authentication attempt -SHALL result in contacting the [LDAP] server to verify the username and password. - -#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default -version: 1.0 - -[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section -SHALL have a default value of `0` that disables caching and forces contacting -the [LDAP] server for each authentication request. - -#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid -version: 1.0 - -[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer. - -For example: - -* negative integer -* string -* empty value -* extremely large positive value (overflow) -* extremely large negative value (overflow) - -The error SHALL appear in the log and SHALL be similar to the following: - -```bash - Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value* -``` - -#### RQ.SRS-007.LDAP.Configuration.Server.Syntax -version: 2.0 - -[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml` -configuration file or of any configuration file inside the `config.d` directory. - -```xml - - - localhost - 636 - cn= - , ou=users, dc=example, dc=com - 0 - yes - tls1.2 - demand - /path/to/tls_cert_file - /path/to/tls_key_file - /path/to/tls_ca_cert_file - /path/to/tls_ca_cert_dir - ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 - - -``` - -#### RQ.SRS-007.LDAP.Configuration.User.RBAC -version: 1.0 - -[ClickHouse] SHALL support creating users identified using an [LDAP] server using -the following RBAC command - -```sql -CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name' -``` - -#### RQ.SRS-007.LDAP.Configuration.User.Syntax -version: 1.0 - -[ClickHouse] SHALL support the following example syntax to create a user that is authenticated using -an [LDAP] server inside the `users.xml` file or any configuration file inside the `users.d` directory. - -```xml - - - - - my_ldap_server - - - - -``` - -#### RQ.SRS-007.LDAP.Configuration.User.Name.Empty -version: 1.0 - -[ClickHouse] SHALL not support empty string as a user name. - -#### RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP -version: 1.0 - -[ClickHouse] SHALL throw an error if `` is specified for the user and at the same -time user configuration contains any of the `` entries. - -#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined -version: 1.0 - -[ClickHouse] SHALL throw an error during any authentification attempt -if the name of the [LDAP] server used inside the `` entry -is not defined in the `` section. - -#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty -version: 1.0 - -[ClickHouse] SHALL throw an error during any authentification attempt -if the name of the [LDAP] server used inside the `` entry -is empty. - -#### RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer -version: 1.0 - -[ClickHouse] SHALL support specifying only one [LDAP] server for a given user. - -#### RQ.SRS-007.LDAP.Configuration.User.Name.Long -version: 1.0 - -[ClickHouse] SHALL support long user names of at least 256 bytes -to specify users that can be authenticated using an [LDAP] server. - -#### RQ.SRS-007.LDAP.Configuration.User.Name.UTF8 -version: 1.0 - -[ClickHouse] SHALL support user names that contain [UTF-8] characters. - -#### RQ.SRS-007.LDAP.Authentication.Username.Empty -version: 1.0 - -[ClickHouse] SHALL not support authenticating users with empty username. - -#### RQ.SRS-007.LDAP.Authentication.Username.Long -version: 1.0 - -[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes. - -#### RQ.SRS-007.LDAP.Authentication.Username.UTF8 -version: 1.0 - -[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters. - -#### RQ.SRS-007.LDAP.Authentication.Password.Empty -version: 1.0 - -[ClickHouse] SHALL not support authenticating users with empty passwords -even if an empty password is valid for the user and -is allowed by the [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.Password.Long -version: 1.0 - -[ClickHouse] SHALL support long password of at least 256 bytes -that can be used to authenticate users using an [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.Password.UTF8 -version: 1.0 - -[ClickHouse] SHALL support [UTF-8] characters in passwords -used to authenticate users using an [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance -version: 1.0 - -[ClickHouse] SHALL provide better login performance of [LDAP] authenticated users -when `verification_cooldown` parameter is set to a positive value when comparing -to the the case when `verification_cooldown` is turned off either for a single user or multiple users -making a large number of repeated requests. - -#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters -version: 1.0 - -[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the -`verification_cooldown` parameter in the [LDAP] server configuration section -if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values -change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user -to result in contacting the [LDAP] server to verify user's username and password. - -#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword -version: 1.0 - -[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the -`verification_cooldown` parameter in the [LDAP] server configuration section -for the user if the password provided in the current authentication attempt does not match -the valid password provided during the first successful authentication request that was cached -for this exact user. The reset SHALL cause the next authentication attempt for this user -to result in contacting the [LDAP] server to verify user's username and password. - -## References - -* **ClickHouse:** https://clickhouse.tech - -[Anonymous Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-AnonymousAuthenticationMechanismOfSimpleBind -[Unauthenticated Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-UnauthenticatedAuthenticationMechanismOfSimpleBind -[Name/Password Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-NamePasswordAuthenticationMechanismOfSimpleBind -[UTF-8]: https://en.wikipedia.org/wiki/UTF-8 -[OpenSSL]: https://www.openssl.org/ -[OpenSSL Ciphers]: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html -[CA]: https://en.wikipedia.org/wiki/Certificate_authority -[TLS]: https://en.wikipedia.org/wiki/Transport_Layer_Security -[LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol -[ClickHouse]: https://clickhouse.tech -[GitHub]: https://github.com -[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/authentication/requirements/requirements.md -[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/authentication/requirements/requirements.md -[Git]: https://git-scm.com/ diff --git a/ldap/authentication/requirements/requirements.py b/ldap/authentication/requirements/requirements.py deleted file mode 100644 index f934e6c7a99..00000000000 --- a/ldap/authentication/requirements/requirements.py +++ /dev/null @@ -1,1687 +0,0 @@ -# These requirements were auto generated -# from software requirements specification (SRS) -# document by TestFlows v1.6.201101.1131719. -# Do not edit by hand but re-generate instead -# using 'tfs requirements generate' command. -from testflows.core import Specification -from testflows.core import Requirement - -SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( - name='SRS-007 ClickHouse Authentication of Users via LDAP', - description=None, - author=None, - date=None, - status=None, - approved_by=None, - approved_date=None, - approved_version=None, - version=None, - group=None, - type=None, - link=None, - uid=None, - parent=None, - children=None, - content=''' -# SRS-007 ClickHouse Authentication of Users via LDAP - -## Table of Contents - -* 1 [Revision History](#revision-history) -* 2 [Introduction](#introduction) -* 3 [Terminology](#terminology) -* 4 [Requirements](#requirements) - * 4.1 [Generic](#generic) - * 4.1.1 [RQ.SRS-007.LDAP.Authentication](#rqsrs-007ldapauthentication) - * 4.1.2 [RQ.SRS-007.LDAP.Authentication.MultipleServers](#rqsrs-007ldapauthenticationmultipleservers) - * 4.1.3 [RQ.SRS-007.LDAP.Authentication.Protocol.PlainText](#rqsrs-007ldapauthenticationprotocolplaintext) - * 4.1.4 [RQ.SRS-007.LDAP.Authentication.Protocol.TLS](#rqsrs-007ldapauthenticationprotocoltls) - * 4.1.5 [RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS](#rqsrs-007ldapauthenticationprotocolstarttls) - * 4.1.6 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation](#rqsrs-007ldapauthenticationtlscertificatevalidation) - * 4.1.7 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned](#rqsrs-007ldapauthenticationtlscertificateselfsigned) - * 4.1.8 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority](#rqsrs-007ldapauthenticationtlscertificatespecificcertificationauthority) - * 4.1.9 [RQ.SRS-007.LDAP.Server.Configuration.Invalid](#rqsrs-007ldapserverconfigurationinvalid) - * 4.1.10 [RQ.SRS-007.LDAP.User.Configuration.Invalid](#rqsrs-007ldapuserconfigurationinvalid) - * 4.1.11 [RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous](#rqsrs-007ldapauthenticationmechanismanonymous) - * 4.1.12 [RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated](#rqsrs-007ldapauthenticationmechanismunauthenticated) - * 4.1.13 [RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword](#rqsrs-007ldapauthenticationmechanismnamepassword) - * 4.1.14 [RQ.SRS-007.LDAP.Authentication.Valid](#rqsrs-007ldapauthenticationvalid) - * 4.1.15 [RQ.SRS-007.LDAP.Authentication.Invalid](#rqsrs-007ldapauthenticationinvalid) - * 4.1.16 [RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser](#rqsrs-007ldapauthenticationinvaliddeleteduser) - * 4.1.17 [RQ.SRS-007.LDAP.Authentication.UsernameChanged](#rqsrs-007ldapauthenticationusernamechanged) - * 4.1.18 [RQ.SRS-007.LDAP.Authentication.PasswordChanged](#rqsrs-007ldapauthenticationpasswordchanged) - * 4.1.19 [RQ.SRS-007.LDAP.Authentication.LDAPServerRestart](#rqsrs-007ldapauthenticationldapserverrestart) - * 4.1.20 [RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart](#rqsrs-007ldapauthenticationclickhouseserverrestart) - * 4.1.21 [RQ.SRS-007.LDAP.Authentication.Parallel](#rqsrs-007ldapauthenticationparallel) - * 4.1.22 [RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid](#rqsrs-007ldapauthenticationparallelvalidandinvalid) - * 4.2 [Specific](#specific) - * 4.2.1 [RQ.SRS-007.LDAP.UnreachableServer](#rqsrs-007ldapunreachableserver) - * 4.2.2 [RQ.SRS-007.LDAP.Configuration.Server.Name](#rqsrs-007ldapconfigurationservername) - * 4.2.3 [RQ.SRS-007.LDAP.Configuration.Server.Host](#rqsrs-007ldapconfigurationserverhost) - * 4.2.4 [RQ.SRS-007.LDAP.Configuration.Server.Port](#rqsrs-007ldapconfigurationserverport) - * 4.2.5 [RQ.SRS-007.LDAP.Configuration.Server.Port.Default](#rqsrs-007ldapconfigurationserverportdefault) - * 4.2.6 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix](#rqsrs-007ldapconfigurationserverauthdnprefix) - * 4.2.7 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix](#rqsrs-007ldapconfigurationserverauthdnsuffix) - * 4.2.8 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value](#rqsrs-007ldapconfigurationserverauthdnvalue) - * 4.2.9 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS](#rqsrs-007ldapconfigurationserverenabletls) - * 4.2.10 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default](#rqsrs-007ldapconfigurationserverenabletlsoptionsdefault) - * 4.2.11 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No](#rqsrs-007ldapconfigurationserverenabletlsoptionsno) - * 4.2.12 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes](#rqsrs-007ldapconfigurationserverenabletlsoptionsyes) - * 4.2.13 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS](#rqsrs-007ldapconfigurationserverenabletlsoptionsstarttls) - * 4.2.14 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion](#rqsrs-007ldapconfigurationservertlsminimumprotocolversion) - * 4.2.15 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values](#rqsrs-007ldapconfigurationservertlsminimumprotocolversionvalues) - * 4.2.16 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default](#rqsrs-007ldapconfigurationservertlsminimumprotocolversiondefault) - * 4.2.17 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert](#rqsrs-007ldapconfigurationservertlsrequirecert) - * 4.2.18 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdefault) - * 4.2.19 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdemand) - * 4.2.20 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsallow) - * 4.2.21 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try](#rqsrs-007ldapconfigurationservertlsrequirecertoptionstry) - * 4.2.22 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsnever) - * 4.2.23 [RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile](#rqsrs-007ldapconfigurationservertlscertfile) - * 4.2.24 [RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile](#rqsrs-007ldapconfigurationservertlskeyfile) - * 4.2.25 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir](#rqsrs-007ldapconfigurationservertlscacertdir) - * 4.2.26 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile](#rqsrs-007ldapconfigurationservertlscacertfile) - * 4.2.27 [RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite](#rqsrs-007ldapconfigurationservertlsciphersuite) - * 4.2.28 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown](#rqsrs-007ldapconfigurationserververificationcooldown) - * 4.2.29 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default](#rqsrs-007ldapconfigurationserververificationcooldowndefault) - * 4.2.30 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid](#rqsrs-007ldapconfigurationserververificationcooldowninvalid) - * 4.2.31 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) - * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) - * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) - * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) - * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) - * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) - * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) - * 4.2.38 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) - * 4.2.39 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) - * 4.2.40 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) - * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) - * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) - * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) - * 4.2.44 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) - * 4.2.45 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) - * 4.2.46 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) - * 4.2.47 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance](#rqsrs-007ldapauthenticationverificationcooldownperformance) - * 4.2.48 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters](#rqsrs-007ldapauthenticationverificationcooldownresetchangeincoreserverparameters) - * 4.2.49 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword](#rqsrs-007ldapauthenticationverificationcooldownresetinvalidpassword) -* 5 [References](#references) - -## Revision History - -This document is stored in an electronic form using [Git] source control management software -hosted in a [GitHub Repository]. -All the updates are tracked using the [Git]'s [Revision History]. - -## Introduction - -[ClickHouse] currently does not have any integration with [LDAP]. -As the initial step in integrating with [LDAP] this software requirements specification covers -only the requirements to enable authentication of users using an [LDAP] server. - -## Terminology - -* **CA** - - Certificate Authority ([CA]) - -* **LDAP** - - Lightweight Directory Access Protocol ([LDAP]) - -## Requirements - -### Generic - -#### RQ.SRS-007.LDAP.Authentication -version: 1.0 - -[ClickHouse] SHALL support user authentication via an [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.MultipleServers -version: 1.0 - -[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate -users. - -#### RQ.SRS-007.LDAP.Authentication.Protocol.PlainText -version: 1.0 - -[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol. - -#### RQ.SRS-007.LDAP.Authentication.Protocol.TLS -version: 1.0 - -[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol. - -#### RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS -version: 1.0 - -[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a -plain text `ldap://` protocol that is upgraded to [TLS]. - -#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation -version: 1.0 - -[ClickHouse] SHALL support certificate validation used for [TLS] connections. - -#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned -version: 1.0 - -[ClickHouse] SHALL support self-signed certificates for [TLS] connections. - -#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority -version: 1.0 - -[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections. - -#### RQ.SRS-007.LDAP.Server.Configuration.Invalid -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid. - -#### RQ.SRS-007.LDAP.User.Configuration.Invalid -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid. - -#### RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind] -authentication mechanism. - -#### RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind] -authentication mechanism. - -#### RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword -version: 1.0 - -[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind] -authentication mechanism. - -#### RQ.SRS-007.LDAP.Authentication.Valid -version: 1.0 - -[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if -user name and password match [LDAP] server records for the user. - -#### RQ.SRS-007.LDAP.Authentication.Invalid -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit authentication if either user name or password -do not match [LDAP] server records for the user. - -#### RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit authentication if the user -has been deleted from the [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.UsernameChanged -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit authentication if the username is changed -on the [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.PasswordChanged -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit authentication if the password -for the user is changed on the [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.LDAPServerRestart -version: 1.0 - -[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted. - -#### RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart -version: 1.0 - -[ClickHouse] SHALL support authenticating users after server is restarted. - -#### RQ.SRS-007.LDAP.Authentication.Parallel -version: 1.0 - -[ClickHouse] SHALL support parallel authentication of users using [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid -version: 1.0 - -[ClickHouse] SHALL support authentication of valid users and -prohibit authentication of invalid users using [LDAP] server -in parallel without having invalid attempts affecting valid authentications. - -### Specific - -#### RQ.SRS-007.LDAP.UnreachableServer -version: 1.0 - -[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable. - -#### RQ.SRS-007.LDAP.Configuration.Server.Name -version: 1.0 - -[ClickHouse] SHALL not support empty string as a server name. - -#### RQ.SRS-007.LDAP.Configuration.Server.Host -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify [LDAP] -server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty. - -#### RQ.SRS-007.LDAP.Configuration.Server.Port -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify [LDAP] server port. - -#### RQ.SRS-007.LDAP.Configuration.Server.Port.Default -version: 1.0 - -[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise. - -#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify the prefix -of value used to construct the DN to bound to during authentication via [LDAP] server. - -#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify the suffix -of value used to construct the DN to bound to during authentication via [LDAP] server. - -#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value -version: 1.0 - -[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string. - -> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character. - -#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS -version: 1.0 - -[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server. - -#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default -version: 1.0 - -[ClickHouse] SHALL use `yes` value as the default for `` parameter -to enable SSL/TLS `ldaps://` protocol. - -#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No -version: 1.0 - -[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable -plain text `ldap://` protocol. - -#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes -version: 1.0 - -[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable -SSL/TLS `ldaps://` protocol. - -#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS -version: 1.0 - -[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable -legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS]. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify -the minimum protocol version of SSL/TLS. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values -version: 1.0 - -[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2` -as a value of the `` parameter. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default -version: 1.0 - -[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify [TLS] peer -certificate verification behavior. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default -version: 1.0 - -[ClickHouse] SHALL use `demand` value as the default for the `` parameter. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand -version: 1.0 - -[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to -enable requesting of client certificate. If no certificate is provided, or a bad certificate is -provided, the session SHALL be immediately terminated. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow -version: 1.0 - -[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to -enable requesting of client certificate. If no -certificate is provided, the session SHALL proceed normally. -If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try -version: 1.0 - -[ClickHouse] SHALL support specifying `try` as the value of `` parameter to -enable requesting of client certificate. If no certificate is provided, the session -SHALL proceed normally. If a bad certificate is provided, the session SHALL be -immediately terminated. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never -version: 1.0 - -[ClickHouse] SHALL support specifying `never` as the value of `` parameter to -disable requesting of client certificate. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile -version: 1.0 - -[ClickHouse] SHALL support `` to specify the path to certificate file used by -[ClickHouse] to establish connection with the [LDAP] server. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile -version: 1.0 - -[ClickHouse] SHALL support `` to specify the path to key file for the certificate -specified by the `` parameter. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify to a path to -the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile -version: 1.0 - -[ClickHouse] SHALL support `` parameter to specify a path to a specific -[CA] certificate file used to verify certificates provided by the [LDAP] server. - -#### RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite -version: 1.0 - -[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites. -The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers]. - -For example, - -```xml -ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 -``` - -The available suites SHALL depend on the [OpenSSL] library version and variant used to build -[ClickHouse] and therefore might change. - -#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown -version: 1.0 - -[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section -that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed -to be successfully authenticated for all consecutive requests without contacting the [LDAP] server. -After period of time since the last successful attempt expires then on the authentication attempt -SHALL result in contacting the [LDAP] server to verify the username and password. - -#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default -version: 1.0 - -[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section -SHALL have a default value of `0` that disables caching and forces contacting -the [LDAP] server for each authentication request. - -#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid -version: 1.0 - -[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer. - -For example: - -* negative integer -* string -* empty value -* extremely large positive value (overflow) -* extremely large negative value (overflow) - -The error SHALL appear in the log and SHALL be similar to the following: - -```bash - Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value* -``` - -#### RQ.SRS-007.LDAP.Configuration.Server.Syntax -version: 2.0 - -[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml` -configuration file or of any configuration file inside the `config.d` directory. - -```xml - - - localhost - 636 - cn= - , ou=users, dc=example, dc=com - 0 - yes - tls1.2 - demand - /path/to/tls_cert_file - /path/to/tls_key_file - /path/to/tls_ca_cert_file - /path/to/tls_ca_cert_dir - ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 - - -``` - -#### RQ.SRS-007.LDAP.Configuration.User.RBAC -version: 1.0 - -[ClickHouse] SHALL support creating users identified using an [LDAP] server using -the following RBAC command - -```sql -CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name' -``` - -#### RQ.SRS-007.LDAP.Configuration.User.Syntax -version: 1.0 - -[ClickHouse] SHALL support the following example syntax to create a user that is authenticated using -an [LDAP] server inside the `users.xml` file or any configuration file inside the `users.d` directory. - -```xml - - - - - my_ldap_server - - - - -``` - -#### RQ.SRS-007.LDAP.Configuration.User.Name.Empty -version: 1.0 - -[ClickHouse] SHALL not support empty string as a user name. - -#### RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP -version: 1.0 - -[ClickHouse] SHALL throw an error if `` is specified for the user and at the same -time user configuration contains any of the `` entries. - -#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined -version: 1.0 - -[ClickHouse] SHALL throw an error during any authentification attempt -if the name of the [LDAP] server used inside the `` entry -is not defined in the `` section. - -#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty -version: 1.0 - -[ClickHouse] SHALL throw an error during any authentification attempt -if the name of the [LDAP] server used inside the `` entry -is empty. - -#### RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer -version: 1.0 - -[ClickHouse] SHALL support specifying only one [LDAP] server for a given user. - -#### RQ.SRS-007.LDAP.Configuration.User.Name.Long -version: 1.0 - -[ClickHouse] SHALL support long user names of at least 256 bytes -to specify users that can be authenticated using an [LDAP] server. - -#### RQ.SRS-007.LDAP.Configuration.User.Name.UTF8 -version: 1.0 - -[ClickHouse] SHALL support user names that contain [UTF-8] characters. - -#### RQ.SRS-007.LDAP.Authentication.Username.Empty -version: 1.0 - -[ClickHouse] SHALL not support authenticating users with empty username. - -#### RQ.SRS-007.LDAP.Authentication.Username.Long -version: 1.0 - -[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes. - -#### RQ.SRS-007.LDAP.Authentication.Username.UTF8 -version: 1.0 - -[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters. - -#### RQ.SRS-007.LDAP.Authentication.Password.Empty -version: 1.0 - -[ClickHouse] SHALL not support authenticating users with empty passwords -even if an empty password is valid for the user and -is allowed by the [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.Password.Long -version: 1.0 - -[ClickHouse] SHALL support long password of at least 256 bytes -that can be used to authenticate users using an [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.Password.UTF8 -version: 1.0 - -[ClickHouse] SHALL support [UTF-8] characters in passwords -used to authenticate users using an [LDAP] server. - -#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance -version: 1.0 - -[ClickHouse] SHALL provide better login performance of [LDAP] authenticated users -when `verification_cooldown` parameter is set to a positive value when comparing -to the the case when `verification_cooldown` is turned off either for a single user or multiple users -making a large number of repeated requests. - -#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters -version: 1.0 - -[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the -`verification_cooldown` parameter in the [LDAP] server configuration section -if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values -change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user -to result in contacting the [LDAP] server to verify user's username and password. - -#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword -version: 1.0 - -[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the -`verification_cooldown` parameter in the [LDAP] server configuration section -for the user if the password provided in the current authentication attempt does not match -the valid password provided during the first successful authentication request that was cached -for this exact user. The reset SHALL cause the next authentication attempt for this user -to result in contacting the [LDAP] server to verify user's username and password. - -## References - -* **ClickHouse:** https://clickhouse.tech - -[Anonymous Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-AnonymousAuthenticationMechanismOfSimpleBind -[Unauthenticated Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-UnauthenticatedAuthenticationMechanismOfSimpleBind -[Name/Password Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-NamePasswordAuthenticationMechanismOfSimpleBind -[UTF-8]: https://en.wikipedia.org/wiki/UTF-8 -[OpenSSL]: https://www.openssl.org/ -[OpenSSL Ciphers]: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html -[CA]: https://en.wikipedia.org/wiki/Certificate_authority -[TLS]: https://en.wikipedia.org/wiki/Transport_Layer_Security -[LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol -[ClickHouse]: https://clickhouse.tech -[GitHub]: https://github.com -[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/authentication/requirements/requirements.md -[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/authentication/requirements/requirements.md -[Git]: https://git-scm.com/ -''') - -RQ_SRS_007_LDAP_Authentication = Requirement( - name='RQ.SRS-007.LDAP.Authentication', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support user authentication via an [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_MultipleServers = Requirement( - name='RQ.SRS-007.LDAP.Authentication.MultipleServers', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate\n' - 'users.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Protocol_PlainText = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Protocol.PlainText', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Protocol_TLS = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Protocol.TLS', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Protocol_StartTLS = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a\n' - 'plain text `ldap://` protocol that is upgraded to [TLS].\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_TLS_Certificate_Validation = Requirement( - name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support certificate validation used for [TLS] connections.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SelfSigned = Requirement( - name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support self-signed certificates for [TLS] connections.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SpecificCertificationAuthority = Requirement( - name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Server_Configuration_Invalid = Requirement( - name='RQ.SRS-007.LDAP.Server.Configuration.Invalid', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_User_Configuration_Invalid = Requirement( - name='RQ.SRS-007.LDAP.User.Configuration.Invalid', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Mechanism_Anonymous = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind]\n' - 'authentication mechanism.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Mechanism_Unauthenticated = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind]\n' - 'authentication mechanism.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind]\n' - 'authentication mechanism.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Valid = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Valid', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if\n' - 'user name and password match [LDAP] server records for the user.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Invalid = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Invalid', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL return an error and prohibit authentication if either user name or password\n' - 'do not match [LDAP] server records for the user.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL return an error and prohibit authentication if the user\n' - 'has been deleted from the [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_UsernameChanged = Requirement( - name='RQ.SRS-007.LDAP.Authentication.UsernameChanged', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL return an error and prohibit authentication if the username is changed\n' - 'on the [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_PasswordChanged = Requirement( - name='RQ.SRS-007.LDAP.Authentication.PasswordChanged', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL return an error and prohibit authentication if the password\n' - 'for the user is changed on the [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_LDAPServerRestart = Requirement( - name='RQ.SRS-007.LDAP.Authentication.LDAPServerRestart', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart = Requirement( - name='RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support authenticating users after server is restarted.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Parallel = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Parallel', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support parallel authentication of users using [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support authentication of valid users and\n' - 'prohibit authentication of invalid users using [LDAP] server\n' - 'in parallel without having invalid attempts affecting valid authentications.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_UnreachableServer = Requirement( - name='RQ.SRS-007.LDAP.UnreachableServer', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_Name = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.Name', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL not support empty string as a server name.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_Host = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.Host', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support `` parameter to specify [LDAP]\n' - 'server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_Port = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.Port', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support `` parameter to specify [LDAP] server port.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_Port_Default = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.Port.Default', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Prefix = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support `` parameter to specify the prefix\n' - 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support `` parameter to specify the suffix\n' - 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Value = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string.\n' - '\n' - "> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character.\n" - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_EnableTLS = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Default = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL use `yes` value as the default for `` parameter\n' - 'to enable SSL/TLS `ldaps://` protocol.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_No = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable\n' - 'plain text `ldap://` protocol.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Yes = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable\n' - 'SSL/TLS `ldaps://` protocol.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_StartTLS = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable\n' - 'legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS].\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support `` parameter to specify\n' - 'the minimum protocol version of SSL/TLS.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Values = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2`\n' - 'as a value of the `` parameter.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Default = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support `` parameter to specify [TLS] peer\n' - 'certificate verification behavior.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Default = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL use `demand` value as the default for the `` parameter.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Demand = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to\n' - 'enable requesting of client certificate. If no certificate is provided, or a bad certificate is\n' - 'provided, the session SHALL be immediately terminated.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Allow = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to\n' - 'enable requesting of client certificate. If no\n' - 'certificate is provided, the session SHALL proceed normally.\n' - 'If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Try = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support specifying `try` as the value of `` parameter to\n' - 'enable requesting of client certificate. If no certificate is provided, the session\n' - 'SHALL proceed normally. If a bad certificate is provided, the session SHALL be\n' - 'immediately terminated.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Never = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support specifying `never` as the value of `` parameter to\n' - 'disable requesting of client certificate.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_TLSCertFile = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support `` to specify the path to certificate file used by\n' - '[ClickHouse] to establish connection with the [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_TLSKeyFile = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support `` to specify the path to key file for the certificate\n' - 'specified by the `` parameter.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_TLSCACertDir = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support `` parameter to specify to a path to\n' - 'the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_TLSCACertFile = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support `` parameter to specify a path to a specific\n' - '[CA] certificate file used to verify certificates provided by the [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites.\n' - 'The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers].\n' - '\n' - 'For example,\n' - '\n' - '```xml\n' - 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n' - '```\n' - '\n' - 'The available suites SHALL depend on the [OpenSSL] library version and variant used to build\n' - '[ClickHouse] and therefore might change.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section\n' - 'that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed\n' - 'to be successfully authenticated for all consecutive requests without contacting the [LDAP] server.\n' - 'After period of time since the last successful attempt expires then on the authentication attempt\n' - 'SHALL result in contacting the [LDAP] server to verify the username and password. \n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Default = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section\n' - 'SHALL have a default value of `0` that disables caching and forces contacting\n' - 'the [LDAP] server for each authentication request.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Invalid = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer.\n' - '\n' - 'For example:\n' - '\n' - '* negative integer\n' - '* string\n' - '* empty value\n' - '* extremely large positive value (overflow)\n' - '* extremely large negative value (overflow)\n' - '\n' - 'The error SHALL appear in the log and SHALL be similar to the following:\n' - '\n' - '```bash\n' - ' Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value*\n' - '```\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.Syntax', - version='2.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml`\n' - 'configuration file or of any configuration file inside the `config.d` directory.\n' - '\n' - '```xml\n' - '\n' - ' \n' - ' localhost\n' - ' 636\n' - ' cn=\n' - ' , ou=users, dc=example, dc=com\n' - ' 0\n' - ' yes\n' - ' tls1.2\n' - ' demand\n' - ' /path/to/tls_cert_file\n' - ' /path/to/tls_key_file\n' - ' /path/to/tls_ca_cert_file\n' - ' /path/to/tls_ca_cert_dir\n' - ' ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n' - ' \n' - '\n' - '```\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_User_RBAC = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.RBAC', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support creating users identified using an [LDAP] server using\n' - 'the following RBAC command\n' - '\n' - '```sql\n' - "CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name'\n" - '```\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_User_Syntax = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.Syntax', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support the following example syntax to create a user that is authenticated using\n' - 'an [LDAP] server inside the `users.xml` file or any configuration file inside the `users.d` directory.\n' - '\n' - '```xml\n' - '\n' - ' \n' - ' \n' - ' \n' - ' my_ldap_server\n' - ' \n' - ' \n' - ' \n' - '\n' - '```\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_User_Name_Empty = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.Name.Empty', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL not support empty string as a user name.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL throw an error if `` is specified for the user and at the same\n' - 'time user configuration contains any of the `` entries.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL throw an error during any authentification attempt\n' - 'if the name of the [LDAP] server used inside the `` entry\n' - 'is not defined in the `` section.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL throw an error during any authentification attempt\n' - 'if the name of the [LDAP] server used inside the `` entry\n' - 'is empty.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support specifying only one [LDAP] server for a given user.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_User_Name_Long = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.Name.Long', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support long user names of at least 256 bytes\n' - 'to specify users that can be authenticated using an [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Configuration_User_Name_UTF8 = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.Name.UTF8', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support user names that contain [UTF-8] characters.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Username_Empty = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Username.Empty', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL not support authenticating users with empty username.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Username_Long = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Username.Long', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Username_UTF8 = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Username.UTF8', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Password_Empty = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Password.Empty', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL not support authenticating users with empty passwords\n' - 'even if an empty password is valid for the user and\n' - 'is allowed by the [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Password_Long = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Password.Long', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support long password of at least 256 bytes\n' - 'that can be used to authenticate users using an [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_Password_UTF8 = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Password.UTF8', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL support [UTF-8] characters in passwords\n' - 'used to authenticate users using an [LDAP] server.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Performance = Requirement( - name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL provide better login performance of [LDAP] authenticated users\n' - 'when `verification_cooldown` parameter is set to a positive value when comparing\n' - 'to the the case when `verification_cooldown` is turned off either for a single user or multiple users\n' - 'making a large number of repeated requests.\n' - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters = Requirement( - name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the\n' - '`verification_cooldown` parameter in the [LDAP] server configuration section\n' - 'if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values\n' - 'change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user\n' - "to result in contacting the [LDAP] server to verify user's username and password.\n" - '\n' - ), - link=None) - -RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_InvalidPassword = Requirement( - name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the\n' - '`verification_cooldown` parameter in the [LDAP] server configuration section\n' - 'for the user if the password provided in the current authentication attempt does not match\n' - 'the valid password provided during the first successful authentication request that was cached\n' - 'for this exact user. The reset SHALL cause the next authentication attempt for this user\n' - "to result in contacting the [LDAP] server to verify user's username and password.\n" - '\n' - ), - link=None) diff --git a/ldap/authentication/tests/authentications.py b/ldap/authentication/tests/authentications.py deleted file mode 100644 index b1a109f87ce..00000000000 --- a/ldap/authentication/tests/authentications.py +++ /dev/null @@ -1,969 +0,0 @@ -# -*- coding: utf-8 -*- -import random -import time - -from multiprocessing.dummy import Pool -from testflows.core import * -from testflows.asserts import error -from ldap.authentication.tests.common import * -from ldap.authentication.requirements import * - -servers = { - "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }, - "openldap2": { - "host": "openldap2", - "port": "636", - "enable_tls": "yes", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "tls_require_cert": "never", - } -} - -@TestStep(When) -@Name("I login as {username} and execute query") -@Args(format_name=True) -def login_and_execute_query(self, username, password, exitcode=None, message=None, steps=True): - """Execute query as some user. - """ - self.context.node.query("SELECT 1", - settings=[("user", username), ("password", password)], - exitcode=exitcode or 0, - message=message, steps=steps) - -@TestScenario -def add_user_to_ldap_and_login(self, server, user=None, ch_user=None, login=None, exitcode=None, message=None, rbac=False): - """Add user to LDAP and ClickHouse and then try to login. - """ - self.context.ldap_node = self.context.cluster.node(server) - - if ch_user is None: - ch_user = {} - if login is None: - login = {} - if user is None: - user = {"cn": "myuser", "userpassword": "myuser"} - - with ldap_user(**user) as user: - ch_user["username"] = ch_user.get("username", user["cn"]) - ch_user["server"] = ch_user.get("server", user["_server"]) - - with ldap_authenticated_users(ch_user, config_file=f"ldap_users_{getuid()}.xml", restart=True, rbac=rbac): - username = login.get("username", user["cn"]) - password = login.get("password", user["userpassword"]) - login_and_execute_query(username=username, password=password, exitcode=exitcode, message=message) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Parallel("1.0"), - RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid("1.0") -) -def parallel_login(self, server, user_count=10, timeout=200, rbac=False): - """Check that login of valid and invalid LDAP authenticated users works in parallel. - """ - self.context.ldap_node = self.context.cluster.node(server) - user = None - - users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] - - with ldap_users(*users): - with ldap_authenticated_users(*[{"username": user["cn"], "server": server} for user in users], rbac=rbac): - - def login_with_valid_username_and_password(users, i, iterations=10): - with When(f"valid users try to login #{i}"): - for i in range(iterations): - random_user = users[random.randint(0, len(users)-1)] - login_and_execute_query(username=random_user["cn"], password=random_user["userpassword"], steps=False) - - def login_with_valid_username_and_invalid_password(users, i, iterations=10): - with When(f"users try to login with valid username and invalid password #{i}"): - for i in range(iterations): - random_user = users[random.randint(0, len(users)-1)] - login_and_execute_query(username=random_user["cn"], - password=(random_user["userpassword"] + randomword(1)), - exitcode=4, - message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", - steps=False) - - def login_with_invalid_username_and_valid_password(users, i, iterations=10): - with When(f"users try to login with invalid username and valid password #{i}"): - for i in range(iterations): - random_user = dict(users[random.randint(0, len(users)-1)]) - random_user["cn"] += randomword(1) - login_and_execute_query(username=random_user["cn"], - password=random_user["userpassword"], - exitcode=4, - message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", - steps=False) - - with When("I login in parallel"): - p = Pool(15) - tasks = [] - for i in range(5): - tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) - - with Then("it should work"): - for task in tasks: - task.get(timeout=timeout) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), - RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser("1.0") -) -def login_after_user_is_deleted_from_ldap(self, server, rbac=False): - """Check that login fails after user is deleted from LDAP. - """ - self.context.ldap_node = self.context.cluster.node(server) - user = None - - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": "myuser"} - user = add_user_to_ldap(**user) - - with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml", - restart=True, rbac=rbac): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - with When("I delete this user from LDAP"): - delete_user_from_ldap(user) - - with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - ) - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), - RQ_SRS_007_LDAP_Authentication_PasswordChanged("1.0") -) -def login_after_user_password_changed_in_ldap(self, server, rbac=False): - """Check that login fails after user password is changed in LDAP. - """ - self.context.ldap_node = self.context.cluster.node(server) - user = None - - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": "myuser"} - user = add_user_to_ldap(**user) - - with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml", - restart=True, rbac=rbac): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - with When("I change user password in LDAP"): - change_user_password_in_ldap(user, "newpassword") - - with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - ) - - with And("when I try to login with the new password it should work"): - login_and_execute_query(username=user["cn"], password="newpassword") - - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), - RQ_SRS_007_LDAP_Authentication_UsernameChanged("1.0") -) -def login_after_user_cn_changed_in_ldap(self, server, rbac=False): - """Check that login fails after user cn is changed in LDAP. - """ - self.context.ldap_node = self.context.cluster.node(server) - user = None - new_user = None - - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": "myuser"} - user = add_user_to_ldap(**user) - - with ldap_authenticated_users({"username": user["cn"], "server": server}, - config_file=f"ldap_users_{getuid()}.xml", restart=True, rbac=rbac): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - with When("I change user password in LDAP"): - new_user = change_user_cn_in_ldap(user, "myuser2") - - with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - ) - finally: - with Finally("I make sure LDAP user is deleted"): - if new_user is not None: - delete_user_from_ldap(new_user, exitcode=None) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Valid("1.0"), - RQ_SRS_007_LDAP_Authentication_LDAPServerRestart("1.0") -) -def login_after_ldap_server_is_restarted(self, server, timeout=60, rbac=False): - """Check that login succeeds after LDAP server is restarted. - """ - self.context.ldap_node = self.context.cluster.node(server) - user = None - - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": getuid()} - user = add_user_to_ldap(**user) - - with ldap_authenticated_users({"username": user["cn"], "server": server}, rbac=rbac): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - with When("I restart LDAP server"): - self.context.ldap_node.restart() - - with Then("I try to login until it works", description=f"timeout {timeout} sec"): - started = time.time() - while True: - r = self.context.node.query("SELECT 1", - settings=[("user", user["cn"]), ("password", user["userpassword"])], - no_checks=True) - if r.exitcode == 0: - break - assert time.time() - started < timeout, error(r.output) - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Valid("1.0"), - RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart("1.0") -) -def login_after_clickhouse_server_is_restarted(self, server, timeout=60, rbac=False): - """Check that login succeeds after ClickHouse server is restarted. - """ - self.context.ldap_node = self.context.cluster.node(server) - user = None - - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": getuid()} - user = add_user_to_ldap(**user) - - with ldap_authenticated_users({"username": user["cn"], "server": server}, rbac=rbac): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - with When("I restart ClickHouse server"): - self.context.node.restart() - - with Then("I try to login until it works", description=f"timeout {timeout} sec"): - started = time.time() - while True: - r = self.context.node.query("SELECT 1", - settings=[("user", user["cn"]), ("password", user["userpassword"])], - no_checks=True) - if r.exitcode == 0: - break - assert time.time() - started < timeout, error(r.output) - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), - RQ_SRS_007_LDAP_Authentication_Password_Empty("1.0") -) -def valid_username_with_valid_empty_password(self, server, rbac=False): - """Check that we can't login using valid username that has empty password. - """ - user = {"cn": "empty_password", "userpassword": ""} - exitcode = 4 - message = f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - - add_user_to_ldap_and_login(user=user, exitcode=exitcode, message=message, server=server, rbac=rbac) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), - RQ_SRS_007_LDAP_Authentication_Password_Empty("1.0") -) -def valid_username_and_invalid_empty_password(self, server, rbac=False): - """Check that we can't login using valid username but invalid empty password. - """ - username = "user_non_empty_password" - user = {"cn": username, "userpassword": username} - login = {"password": ""} - - exitcode = 4 - message = f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" - - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Valid("1.0") -) -def valid_username_and_password(self, server, rbac=False): - """Check that we can login using valid username and password. - """ - username = "valid_username_and_password" - user = {"cn": username, "userpassword": username} - - with When(f"I add user {username} to LDAP and try to login"): - add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0") -) -def valid_username_and_password_invalid_server(self, server=None, rbac=False): - """Check that we can't login using valid username and valid - password but for a different server. - """ - self.context.ldap_node = self.context.cluster.node("openldap1") - - user = {"username": "user2", "userpassword": "user2", "server": "openldap1"} - - exitcode = 4 - message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" - - with ldap_authenticated_users(user, config_file=f"ldap_users_{getuid()}.xml", restart=True, rbac=rbac): - login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Valid("1.0"), - RQ_SRS_007_LDAP_Authentication_Username_Long("1.0"), - RQ_SRS_007_LDAP_Configuration_User_Name_Long("1.0") -) -def valid_long_username_and_short_password(self, server, rbac=False): - """Check that we can login using valid very long username and short password. - """ - username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" - user = {"cn": username, "userpassword": "long_username"} - - add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0") -) -def invalid_long_username_and_valid_short_password(self, server, rbac=False): - """Check that we can't login using slightly invalid long username but valid password. - """ - username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" - user = {"cn": username, "userpassword": "long_username"} - login = {"username": f"{username}?"} - - exitcode = 4 - message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" - - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Valid("1.0"), - RQ_SRS_007_LDAP_Authentication_Password_Long("1.0") -) -def valid_short_username_and_long_password(self, server, rbac=False): - """Check that we can login using valid short username with very long password. - """ - username = "long_password" - user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} - add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0") -) -def valid_short_username_and_invalid_long_password(self, server, rbac=False): - """Check that we can't login using valid short username and invalid long password. - """ - username = "long_password" - user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} - login = {"password": user["userpassword"] + "1"} - - exitcode = 4 - message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" - - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0") -) -def valid_username_and_invalid_password(self, server, rbac=False): - """Check that we can't login using valid username and invalid password. - """ - username = "valid_username_and_invalid_password" - user = {"cn": username, "userpassword": username} - login = {"password": user["userpassword"] + "1"} - - exitcode = 4 - message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" - - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0") -) -def invalid_username_and_valid_password(self, server, rbac=False): - """Check that we can't login using slightly invalid username but valid password. - """ - username = "invalid_username_and_valid_password" - user = {"cn": username, "userpassword": username} - login = {"username": user["cn"] + "1"} - - exitcode = 4 - message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" - - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Valid("1.0"), - RQ_SRS_007_LDAP_Authentication_Username_UTF8("1.0"), - RQ_SRS_007_LDAP_Configuration_User_Name_UTF8("1.0") -) -def valid_utf8_username_and_ascii_password(self, server, rbac=False): - """Check that we can login using valid utf-8 username with ascii password. - """ - username = "utf8_username_Gãńdåłf_Thê_Gręât" - user = {"cn": username, "userpassword": "utf8_username"} - - add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Valid("1.0"), - RQ_SRS_007_LDAP_Authentication_Password_UTF8("1.0") -) -def valid_ascii_username_and_utf8_password(self, server, rbac=False): - """Check that we can login using valid ascii username with utf-8 password. - """ - username = "utf8_password" - user = {"cn": username, "userpassword": "utf8_password_Gãńdåłf_Thê_Gręât"} - - add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) - -@TestScenario -def empty_username_and_empty_password(self, server=None, rbac=False): - """Check that we can login using empty username and empty password as - it will use the default user and that has an empty password. - """ - login_and_execute_query(username="", password="") - -@TestScenario -@Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Default("1.0") -) -def default_verification_cooldown_value(self, server, rbac=False, timeout=20): - """Check that the default value (0) for the verification cooldown parameter - disables caching and forces contacting the LDAP server for each - authentication request. - """ - - error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" - error_exitcode = 4 - user = None - - with Given("I have an LDAP configuration that uses the default verification_cooldown value (0)"): - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - - self.context.ldap_node = self.context.cluster.node(server) - - try: - with Given("I add user to LDAP"): - user = {"cn": "testVCD", "userpassword": "testVCD"} - user = add_user_to_ldap(**user) - - with ldap_servers(servers): - with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): - with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - with And("I change user password in LDAP"): - change_user_password_in_ldap(user, "newpassword") - - with Then("when I try to login immediately with the old user password it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=error_exitcode, message=error_message) - - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) - -@TestScenario -@Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0") -) -def valid_verification_cooldown_value_cn_change(self, server, rbac=False, timeout=20): - """Check that we can perform requests without contacting the LDAP server - after successful authentication when the verification_cooldown parameter - is set and the user cn is changed. - """ - - error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" - error_exitcode = 4 - user = None - new_user = None - - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "2" - }} - - self.context.ldap_node = self.context.cluster.node(server) - - try: - with Given("I add user to LDAP"): - user = {"cn": "testVCD", "userpassword": "testVCD"} - user = add_user_to_ldap(**user) - - with ldap_servers(servers): - with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): - with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - with And("I change user cn in LDAP"): - new_user = change_user_cn_in_ldap(user, "testVCD2") - - with Then("when I try to login again with the old user cn it should work"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - with And("when I sleep for 2 seconds and try to log in, it should fail"): - time.sleep(2) - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=error_exitcode, message=error_message) - - finally: - with Finally("I make sure LDAP user is deleted"): - if new_user is not None: - delete_user_from_ldap(new_user, exitcode=None) - -@TestScenario -@Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0") -) -def valid_verification_cooldown_value_password_change(self, server, rbac=False, timeout=20): - """Check that we can perform requests without contacting the LDAP server - after successful authentication when the verification_cooldown parameter - is set and the user password is changed. - """ - - error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" - error_exitcode = 4 - user = None - - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "2" - }} - - self.context.ldap_node = self.context.cluster.node(server) - - try: - with Given("I add user to LDAP"): - user = {"cn": "testVCD", "userpassword": "testVCD"} - user = add_user_to_ldap(**user) - - with ldap_servers(servers): - with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): - with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - with And("I change user password in LDAP"): - change_user_password_in_ldap(user, "newpassword") - - with Then("when I try to login again with the old password it should work"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - with And("when I sleep for 2 seconds and try to log in, it should fail"): - time.sleep(2) - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=error_exitcode, message=error_message) - - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) - -@TestScenario -@Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0") -) -def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False, timeout=20): - """Check that we can perform requests without contacting the LDAP server - after successful authentication when the verification_cooldown parameter - is set, even when the LDAP server is offline. - """ - - error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" - error_exitcode = 4 - user = None - - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "2" - }} - - self.context.ldap_node = self.context.cluster.node(server) - - try: - with Given("I add a new user to LDAP"): - user = {"cn": "testVCD", "userpassword": "testVCD"} - user = add_user_to_ldap(**user) - - with ldap_servers(servers): - with ldap_authenticated_users({"username": user["cn"], "server": server}, - config_file=f"ldap_users_{getuid()}.xml"): - - with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - try: - with And("then I stop the ldap server"): - self.context.ldap_node.stop() - - with Then("when I try to login again with the server offline it should work"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - with And("when I sleep for 2 seconds and try to log in, it should fail"): - time.sleep(2) - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=error_exitcode, message=error_message) - - finally: - with Finally("I start the ldap server back up"): - self.context.ldap_node.start() - - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) - -@TestOutline -def repeat_requests(self, server, iterations, vcd_value, rbac=False): - """Run repeated requests from some user to the LDAP server. - """ - - user = None - - with Given(f"I have an LDAP configuration that sets verification_cooldown parameter to {vcd_value} sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": vcd_value - }} - - self.context.ldap_node = self.context.cluster.node(server) - - try: - with And("I add a new user to LDAP"): - user = {"cn": "testVCD", "userpassword": "testVCD"} - user = add_user_to_ldap(**user) - - with ldap_servers(servers): - with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): - with When(f"I login and execute some query {iterations} times"): - start_time = time.time() - r = self.context.node.command(f"time for i in {{1..{iterations}}}; do clickhouse client -q \"SELECT 1\" --user {user['cn']} --password {user['userpassword']} > /dev/null; done") - end_time = time.time() - - return end_time - start_time - - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) - -@TestScenario -@Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Performance("1.0") -) -def verification_cooldown_performance(self, server, rbac=False, iterations=5000): - """Check that login performance is better when the verification cooldown - parameter is set to a positive value when comparing to the case when - the verification cooldown parameter is turned off. - """ - - vcd_time = 0 - no_vcd_time = 0 - - with Example(f"Repeated requests with verification cooldown parameter set to 600 seconds, {iterations} iterations"): - vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="600", rbac=rbac) - metric("login_with_vcd_value_600", units="seconds", value=vcd_time) - - with Example(f"Repeated requests with verification cooldown parameter set to 0 seconds, {iterations} iterations"): - no_vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="0", rbac=rbac) - metric("login_with_vcd_value_0", units="seconds", value=no_vcd_time) - - with Then("The performance with verification cooldown parameter set is better than the performance with no verification cooldown parameter."): - assert no_vcd_time > vcd_time, error() - - with And("Log the performance improvement as a percentage."): - metric("percentage_improvement", units="%", value=100*(no_vcd_time - vcd_time)/vcd_time) - -@TestOutline -def check_verification_cooldown_reset_on_core_server_parameter_change(self, server, - parameter_name, parameter_value, rbac=False): - """Check that the LDAP login cache is reset for all the LDAP authentication users - when verification_cooldown parameter is set after one of the core server - parameters is changed in the LDAP server configuration. - """ - - config_d_dir="/etc/clickhouse-server/config.d" - config_file="ldap_servers.xml" - error_message = "DB::Exception: {user}: Authentication failed: password is incorrect or there is no user with such name" - error_exitcode = 4 - user = None - config=None - updated_config=None - - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 600 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "600" - }} - - self.context.ldap_node = self.context.cluster.node(server) - - with And("LDAP authenticated user"): - users = [ - {"cn": f"testVCD_0", "userpassword": "testVCD_0"}, - {"cn": f"testVCD_1", "userpassword": "testVCD_1"} - ] - - with And("I create LDAP servers configuration file"): - config = create_ldap_servers_config_content(servers, config_d_dir, config_file) - - with ldap_users(*users) as users: - with ldap_servers(servers, restart=True): - with ldap_authenticated_users(*[{"username": user["cn"], "server": server} for user in users]): - with When("I login and execute a query"): - for user in users: - with By(f"as user {user['cn']}"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - with And("I change user password in LDAP"): - for user in users: - with By(f"for user {user['cn']}"): - change_user_password_in_ldap(user, "newpassword") - - with And(f"I change the server {parameter_name} core parameter", description=f"{parameter_value}"): - servers["openldap1"][parameter_name] = parameter_value - - with And("I create an updated the config file that has a different server host name"): - updated_config = create_ldap_servers_config_content(servers, config_d_dir, config_file) - - with modify_config(updated_config, restart=False): - with Then("when I try to log in it should fail as cache should have been reset"): - for user in users: - with By(f"as user {user['cn']}"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=error_exitcode, message=error_message.format(user=user["cn"])) - -@TestScenario -@Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") -) -def verification_cooldown_reset_on_server_host_parameter_change(self, server, rbac=False): - """Check that the LDAP login cache is reset for all the LDAP authentication users - when verification_cooldown parameter is set after server host name - is changed in the LDAP server configuration. - """ - - check_verification_cooldown_reset_on_core_server_parameter_change(server=server, - parameter_name="host", parameter_value="openldap2", rbac=rbac) - -@TestScenario -@Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") -) -def verification_cooldown_reset_on_server_port_parameter_change(self, server, rbac=False): - """Check that the LDAP login cache is reset for all the LDAP authentication users - when verification_cooldown parameter is set after server port is changed in the - LDAP server configuration. - """ - - check_verification_cooldown_reset_on_core_server_parameter_change(server=server, - parameter_name="port", parameter_value="9006", rbac=rbac) - -@TestScenario -@Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") -) -def verification_cooldown_reset_on_server_auth_dn_prefix_parameter_change(self, server, rbac=False): - """Check that the LDAP login cache is reset for all the LDAP authentication users - when verification_cooldown parameter is set after server auth_dn_prefix - is changed in the LDAP server configuration. - """ - - check_verification_cooldown_reset_on_core_server_parameter_change(server=server, - parameter_name="auth_dn_prefix", parameter_value="cxx=", rbac=rbac) - -@TestScenario -@Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") -) -def verification_cooldown_reset_on_server_auth_dn_suffix_parameter_change(self, server, rbac=False): - """Check that the LDAP login cache is reset for all the LDAP authentication users - when verification_cooldown parameter is set after server auth_dn_suffix - is changed in the LDAP server configuration. - """ - - check_verification_cooldown_reset_on_core_server_parameter_change(server=server, - parameter_name="auth_dn_suffix", - parameter_value=",ou=company,dc=users,dc=com", rbac=rbac) - - -@TestScenario -@Name("verification cooldown reset when invalid password is provided") -@Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_InvalidPassword("1.0") -) -def scenario(self, server, rbac=False): - """Check that cached bind requests for the user are discarded when - the user provides invalid login credentials. - """ - - user = None - error_exitcode = 4 - error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" - - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 600 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "600" - }} - - self.context.ldap_node = self.context.cluster.node(server) - - try: - with Given("I add a new user to LDAP"): - user = {"cn": "testVCD", "userpassword": "testVCD"} - user = add_user_to_ldap(**user) - - with ldap_servers(servers): - with ldap_authenticated_users({"username": user["cn"], "server": server}, - config_file=f"ldap_users_{getuid()}.xml"): - - with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - with And("I change user password in LDAP"): - change_user_password_in_ldap(user, "newpassword") - - with Then("When I try to log in with the cached password it should work"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) - - with And("When I try to log in with an incorrect password it should fail"): - login_and_execute_query(username=user["cn"], password="incorrect", exitcode=error_exitcode, - message=error_message) - - with And("When I try to log in with the cached password it should fail"): - login_and_execute_query(username=user["cn"], password="incorrect", exitcode=error_exitcode, - message=error_message) - - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) - -@TestFeature -def verification_cooldown(self, rbac, servers=None, node="clickhouse1"): - """Check verification cooldown parameter functionality. - """ - for scenario in loads(current_module(), Scenario, filter=has.tag("verification_cooldown")): - scenario(server="openldap1", rbac=rbac) - - -@TestOutline(Feature) -@Name("user authentications") -@Requirements( - RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword("1.0") -) -@Examples("rbac", [ - (False,), - (True, Requirements(RQ_SRS_007_LDAP_Configuration_User_RBAC("1.0"))) -]) -def feature(self, rbac, servers=None, node="clickhouse1"): - """Check that users can be authenticated using an LDAP server when - users are configured either using an XML configuration file or RBAC. - """ - self.context.node = self.context.cluster.node(node) - - if servers is None: - servers = globals()["servers"] - - with ldap_servers(servers): - for scenario in loads(current_module(), Scenario, filter=~has.tag("verification_cooldown")): - scenario(server="openldap1", rbac=rbac) - - Feature(test=verification_cooldown)(rbac=rbac, servers=servers, node=node) - - - - diff --git a/ldap/authentication/tests/common.py b/ldap/authentication/tests/common.py deleted file mode 100644 index 8efb389a23f..00000000000 --- a/ldap/authentication/tests/common.py +++ /dev/null @@ -1,466 +0,0 @@ -import os -import uuid -import time -import string -import random -import textwrap -import xml.etree.ElementTree as xmltree - -from collections import namedtuple -from contextlib import contextmanager - -import testflows.settings as settings - -from testflows.core import * -from testflows.asserts import error - -def getuid(): - return str(uuid.uuid1()).replace('-', '_') - -xml_with_utf8 = '\n' - -def xml_indent(elem, level=0, by=" "): - i = "\n" + level * by - if len(elem): - if not elem.text or not elem.text.strip(): - elem.text = i + by - if not elem.tail or not elem.tail.strip(): - elem.tail = i - for elem in elem: - xml_indent(elem, level + 1) - if not elem.tail or not elem.tail.strip(): - elem.tail = i - else: - if level and (not elem.tail or not elem.tail.strip()): - elem.tail = i - -def xml_append(root, tag, text): - element = xmltree.Element(tag) - element.text = text - root.append(element) - return element - -Config = namedtuple("Config", "content path name uid preprocessed_name") - -ASCII_CHARS = string.ascii_lowercase + string.ascii_uppercase + string.digits - -def randomword(length, chars=ASCII_CHARS): - return ''.join(random.choice(chars) for i in range(length)) - -def restart(node=None, safe=False, timeout=60): - """Restart ClickHouse server and wait for config to be reloaded. - """ - with When("I restart ClickHouse server node"): - if node is None: - node = current().context.node - - with node.cluster.shell(node.name) as bash: - bash.expect(bash.prompt) - - with By("closing terminal to the node to be restarted"): - bash.close() - - with And("getting current log size"): - logsize = \ - node.command("stat --format=%s /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[ - 0].strip() - - with And("restarting ClickHouse server"): - node.restart(safe=safe) - - with Then("tailing the log file from using previous log size as the offset"): - bash.prompt = bash.__class__.prompt - bash.open() - bash.send(f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log") - - with And("waiting for config reload message in the log file"): - bash.expect( - f"ConfigReloader: Loaded config '/etc/clickhouse-server/config.xml', performed update on configuration", - timeout=timeout) - -def add_config(config, timeout=60, restart=False, modify=False): - """Add dynamic configuration file to ClickHouse. - - :param node: node - :param config: configuration file description - :param timeout: timeout, default: 20 sec - """ - node = current().context.node - - def check_preprocessed_config_is_updated(after_removal=False): - """Check that preprocessed config is updated. - """ - started = time.time() - command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" - - while time.time() - started < timeout: - exitcode = node.command(command, steps=False).exitcode - if after_removal: - if exitcode == 1: - break - else: - if exitcode == 0: - break - time.sleep(1) - - if settings.debug: - node.command(f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name}") - - if after_removal: - assert exitcode == 1, error() - else: - assert exitcode == 0, error() - - def wait_for_config_to_be_loaded(): - """Wait for config to be loaded. - """ - if restart: - with When("I close terminal to the node to be restarted"): - bash.close() - - with And("I stop ClickHouse to apply the config changes"): - node.stop(safe=False) - - with And("I get the current log size"): - cmd = node.cluster.command(None, - f"stat --format=%s {os.environ['CLICKHOUSE_TESTS_DIR']}/_instances/{node.name}/logs/clickhouse-server.log") - logsize = cmd.output.split(" ")[0].strip() - - with And("I start ClickHouse back up"): - node.start() - - with Then("I tail the log file from using previous log size as the offset"): - bash.prompt = bash.__class__.prompt - bash.open() - bash.send(f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log") - - with Then("I wait for config reload message in the log file"): - if restart: - bash.expect( - f"ConfigReloader: Loaded config '/etc/clickhouse-server/config.xml', performed update on configuration", - timeout=timeout) - else: - bash.expect( - f"ConfigReloader: Loaded config '/etc/clickhouse-server/{config.preprocessed_name}', performed update on configuration", - timeout=timeout) - - try: - with Given(f"{config.name}"): - if settings.debug: - with When("I output the content of the config"): - debug(config.content) - - with node.cluster.shell(node.name) as bash: - bash.expect(bash.prompt) - bash.send("tail -n 0 -f /var/log/clickhouse-server/clickhouse-server.log") - - with When("I add the config", description=config.path): - command = f"cat < {config.path}\n{config.content}\nHEREDOC" - node.command(command, steps=False, exitcode=0) - - with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): - check_preprocessed_config_is_updated() - - with And("I wait for config to be reloaded"): - wait_for_config_to_be_loaded() - yield - finally: - if not modify: - with Finally(f"I remove {config.name}"): - with node.cluster.shell(node.name) as bash: - bash.expect(bash.prompt) - bash.send("tail -n 0 -f /var/log/clickhouse-server/clickhouse-server.log") - - with By("removing the config file", description=config.path): - node.command(f"rm -rf {config.path}", exitcode=0) - - with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): - check_preprocessed_config_is_updated(after_removal=True) - - with And("I wait for config to be reloaded"): - wait_for_config_to_be_loaded() - -def create_ldap_servers_config_content(servers, config_d_dir="/etc/clickhouse-server/config.d", config_file="ldap_servers.xml"): - """Create LDAP servers configuration content. - """ - uid = getuid() - path = os.path.join(config_d_dir, config_file) - name = config_file - - root = xmltree.fromstring("") - xml_servers = root.find("ldap_servers") - xml_servers.append(xmltree.Comment(text=f"LDAP servers {uid}")) - - for _name, server in list(servers.items()): - xml_server = xmltree.Element(_name) - for key, value in list(server.items()): - xml_append(xml_server, key, value) - xml_servers.append(xml_server) - - xml_indent(root) - content = xml_with_utf8 + str(xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8") - - return Config(content, path, name, uid, "config.xml") - -@contextmanager -def modify_config(config, restart=False): - """Apply updated configuration file. - """ - return add_config(config, restart=restart, modify=True) - -@contextmanager -def ldap_servers(servers, config_d_dir="/etc/clickhouse-server/config.d", config_file="ldap_servers.xml", - timeout=60, restart=False, config=None): - """Add LDAP servers configuration. - """ - if config is None: - config = create_ldap_servers_config_content(servers, config_d_dir, config_file) - return add_config(config, restart=restart) - -def create_ldap_users_config_content(*users, config_d_dir="/etc/clickhouse-server/users.d", config_file="ldap_users.xml"): - """Create LDAP users configuration file content. - """ - uid = getuid() - path = os.path.join(config_d_dir, config_file) - name = config_file - - root = xmltree.fromstring("") - xml_users = root.find("users") - xml_users.append(xmltree.Comment(text=f"LDAP users {uid}")) - - for user in users: - xml_user = xmltree.Element(user['username']) - xml_user_server = xmltree.Element("ldap") - xml_append(xml_user_server, "server", user["server"]) - xml_user.append(xml_user_server) - xml_users.append(xml_user) - - xml_indent(root) - content = xml_with_utf8 + str(xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8") - - return Config(content, path, name, uid, "users.xml") - -def add_users_identified_with_ldap(*users): - """Add one or more users that are identified via - an ldap server using RBAC. - """ - node = current().context.node - try: - with Given("I create users"): - for user in users: - node.query(f"CREATE USER '{user['username']}' IDENTIFIED WITH ldap_server BY '{user['server']}'") - yield - finally: - with Finally("I remove users"): - for user in users: - with By(f"dropping user {user['username']}", flags=TE): - node.query(f"DROP USER IF EXISTS '{user['username']}'") - -@contextmanager -def ldap_authenticated_users(*users, config_d_dir="/etc/clickhouse-server/users.d", - config_file=None, timeout=60, restart=True, config=None, rbac=False): - """Add LDAP authenticated users. - """ - if rbac: - return add_users_identified_with_ldap(*users) - else: - if config_file is None: - config_file = f"ldap_users_{getuid()}.xml" - if config is None: - config = create_ldap_users_config_content(*users, config_d_dir=config_d_dir, config_file=config_file) - return add_config(config, restart=restart) - -def invalid_server_config(servers, message=None, tail=13, timeout=60): - """Check that ClickHouse errors when trying to load invalid LDAP servers configuration file. - """ - node = current().context.node - if message is None: - message = "Exception: Failed to merge config with '/etc/clickhouse-server/config.d/ldap_servers.xml'" - - config = create_ldap_servers_config_content(servers) - try: - node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) - - with When("I add the config", description=config.path): - command = f"cat < {config.path}\n{config.content}\nHEREDOC" - node.command(command, steps=False, exitcode=0) - - with Then("server shall fail to merge the new config"): - started = time.time() - command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" - while time.time() - started < timeout: - exitcode = node.command(command, steps=False).exitcode - if exitcode == 0: - break - time.sleep(1) - assert exitcode == 0, error() - finally: - with Finally(f"I remove {config.name}"): - with By("removing the config file", description=config.path): - node.command(f"rm -rf {config.path}", exitcode=0) - -def invalid_user_config(servers, config, message=None, tail=13, timeout=60): - """Check that ClickHouse errors when trying to load invalid LDAP users configuration file. - """ - node = current().context.node - if message is None: - message = "Exception: Failed to merge config with '/etc/clickhouse-server/users.d/ldap_users.xml'" - - with ldap_servers(servers): - try: - node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("\\n" * tail)) - with When("I add the config", description=config.path): - command = f"cat < {config.path}\n{config.content}\nHEREDOC" - node.command(command, steps=False, exitcode=0) - - with Then("server shall fail to merge the new config"): - started = time.time() - command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" - while time.time() - started < timeout: - exitcode = node.command(command, steps=False).exitcode - if exitcode == 0: - break - time.sleep(1) - assert exitcode == 0, error() - finally: - with Finally(f"I remove {config.name}"): - with By("removing the config file", description=config.path): - node.command(f"rm -rf {config.path}", exitcode=0) - -def add_user_to_ldap(cn, userpassword, givenname=None, homedirectory=None, sn=None, uid=None, uidnumber=None, node=None): - """Add user entry to LDAP.""" - if node is None: - node = current().context.ldap_node - if uid is None: - uid = cn - if givenname is None: - givenname = "John" - if homedirectory is None: - homedirectory = "/home/users" - if sn is None: - sn = "User" - if uidnumber is None: - uidnumber = 2000 - - user = { - "dn": f"cn={cn},ou=users,dc=company,dc=com", - "cn": cn, - "gidnumber": 501, - "givenname": givenname, - "homedirectory": homedirectory, - "objectclass": ["inetOrgPerson", "posixAccount", "top"], - "sn": sn, - "uid": uid, - "uidnumber": uidnumber, - "userpassword": userpassword, - "_server": node.name - } - - lines = [] - - for key, value in list(user.items()): - if key.startswith("_"): - continue - elif key == "objectclass": - for cls in value: - lines.append(f"objectclass: {cls}") - else: - lines.append(f"{key}: {value}") - - ldif = "\n".join(lines) - - r = node.command( - f"echo -e \"{ldif}\" | ldapadd -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin") - assert r.exitcode == 0, error() - - return user - -def delete_user_from_ldap(user, node=None, exitcode=0): - """Delete user entry from LDAP.""" - if node is None: - node = current().context.ldap_node - r = node.command( - f"ldapdelete -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin \"{user['dn']}\"") - if exitcode is not None: - assert r.exitcode == exitcode, error() - -def change_user_password_in_ldap(user, new_password, node=None, exitcode=0): - """Change user password in LDAP.""" - if node is None: - node = current().context.ldap_node - - ldif = (f"dn: {user['dn']}\n" - "changetype: modify\n" - "replace: userpassword\n" - f"userpassword: {new_password}") - - r = node.command( - f"echo -e \"{ldif}\" | ldapmodify -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin") - - if exitcode is not None: - assert r.exitcode == exitcode, error() - -def change_user_cn_in_ldap(user, new_cn, node=None, exitcode=0): - """Change user password in LDAP.""" - if node is None: - node = current().context.ldap_node - - new_user = dict(user) - new_user['dn'] = f"cn={new_cn},ou=users,dc=company,dc=com" - new_user['cn'] = new_cn - - ldif = ( - f"dn: {user['dn']}\n" - "changetype: modrdn\n" - f"newrdn: cn = {new_user['cn']}\n" - f"deleteoldrdn: 1\n" - ) - - r = node.command( - f"echo -e \"{ldif}\" | ldapmodify -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin") - - if exitcode is not None: - assert r.exitcode == exitcode, error() - - return new_user - -@contextmanager -def ldap_user(cn, userpassword, givenname=None, homedirectory=None, sn=None, uid=None, uidnumber=None, node=None): - """Add new user to the LDAP server.""" - try: - user = None - with Given(f"I add user {cn} to LDAP"): - user = add_user_to_ldap(cn, userpassword, givenname, homedirectory, sn, uid, uidnumber, node=node) - yield user - finally: - with Finally(f"I delete user {cn} from LDAP"): - if user is not None: - delete_user_from_ldap(user, node=node) - -@contextmanager -def ldap_users(*users, node=None): - """Add multiple new users to the LDAP server.""" - try: - _users = [] - with Given("I add users to LDAP"): - for user in users: - with By(f"adding user {user['cn']}"): - _users.append(add_user_to_ldap(**user, node=node)) - yield _users - finally: - with Finally(f"I delete users from LDAP"): - for _user in _users: - delete_user_from_ldap(_user, node=node) - -def login(servers, *users, config=None): - """Configure LDAP server and LDAP authenticated users and - try to login and execute a query""" - with ldap_servers(servers): - with ldap_authenticated_users(*users, restart=True, config=config): - for user in users: - if user.get("login", False): - with When(f"I login as {user['username']} and execute query"): - current().context.node.query("SELECT 1", - settings=[("user", user["username"]), ("password", user["password"])], - exitcode=user.get("exitcode", None), - message=user.get("message", None)) diff --git a/ldap/authentication/tests/server_config.py b/ldap/authentication/tests/server_config.py deleted file mode 100644 index 38ec859226b..00000000000 --- a/ldap/authentication/tests/server_config.py +++ /dev/null @@ -1,304 +0,0 @@ -from testflows.core import * - -from ldap.authentication.tests.common import * -from ldap.authentication.requirements import * - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_Name("1.0") -) -def empty_server_name(self, timeout=20): - """Check that empty string as a server name is not allowed. - """ - servers = {"": {"host": "foo", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - invalid_server_config(servers, timeout=timeout) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0"), - RQ_SRS_007_LDAP_UnreachableServer("1.0") -) -def invalid_host(self): - """Check that server returns an error when LDAP server - host name is invalid. - """ - servers = {"foo": {"host": "foo", "port": "389", "enable_tls": "no"}} - users = [{ - "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] - login(servers, *users) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_Host("1.0") -) -def empty_host(self): - """Check that server returns an error when LDAP server - host value is empty. - """ - servers = {"foo": {"host": "", "port": "389", "enable_tls": "no"}} - users = [{ - "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] - login(servers, *users) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_Host("1.0") -) -def missing_host(self): - """Check that server returns an error when LDAP server - host is missing. - """ - servers = {"foo": {"port": "389", "enable_tls": "no"}} - users = [{ - "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] - login(servers, *users) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") -) -def invalid_port(self): - """Check that server returns an error when LDAP server - port is not valid. - """ - servers = {"openldap1": {"host": "openldap1", "port": "3890", "enable_tls": "no"}} - users = [{ - "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] - login(servers, *users) - - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") -) -def invalid_auth_dn_prefix(self): - """Check that server returns an error when LDAP server - port is not valid. - """ - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "foo=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - users = [{ - "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] - login(servers, *users) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") -) -def invalid_auth_dn_suffix(self): - """Check that server returns an error when LDAP server - port is not valid. - """ - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",foo=users,dc=company,dc=com" - }} - users = [{ - "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] - login(servers, *users) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") -) -def invalid_enable_tls_value(self): - """Check that server returns an error when enable_tls - option has invalid value. - """ - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "foo", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - users = [{ - "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] - login(servers, *users) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") -) -def invalid_tls_require_cert_value(self): - """Check that server returns an error when tls_require_cert - option has invalid value. - """ - servers = {"openldap2": { - "host": "openldap2", "port": "636", "enable_tls": "yes", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "tls_require_cert": "foo", - "ca_cert_dir": "/container/service/slapd/assets/certs/", - "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt" - }} - users = [{ - "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 4, - "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" - }] - login(servers, *users) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") -) -def empty_ca_cert_dir(self): - """Check that server returns an error when ca_cert_dir is empty. - """ - servers = {"openldap2": {"host": "openldap2", "port": "636", "enable_tls": "yes", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "tls_require_cert": "demand", - "ca_cert_dir": "", - "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt" - }} - users = [{ - "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 4, - "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" - }] - login(servers, *users) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") -) -def empty_ca_cert_file(self): - """Check that server returns an error when ca_cert_file is empty. - """ - servers = {"openldap2": {"host": "openldap2", "port": "636", "enable_tls": "yes", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "tls_require_cert": "demand", - "ca_cert_dir": "/container/service/slapd/assets/certs/", - "ca_cert_file": "" - }} - users = [{ - "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 4, - "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" - }] - login(servers, *users) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Value("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Prefix("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix("1.0") -) -def auth_dn_value(self): - """Check that server configuration can properly define the `dn` value of the user.""" - servers = { - "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} - - login(servers, user) - -@TestOutline(Scenario) -@Examples("invalid_value", [ - ("-1", Name("negative int")), - ("foo", Name("string")), - ("", Name("empty string")), - ("36893488147419103232", Name("overflow with extremely large int value")), - ("-36893488147419103232", Name("overflow with extremely large negative int value")), - ("@#", Name("special characters")) -]) -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Invalid("1.0") -) -def invalid_verification_cooldown_value(self, invalid_value, timeout=20): - """Check that server returns an error when LDAP server - verification cooldown parameter is invalid. - """ - - error_message = (" Access(user directories): Could not parse LDAP server" - " \\`openldap1\\`: Poco::Exception. Code: 1000, e.code() = 0," - f" e.displayText() = Syntax error: Not a valid unsigned integer{': ' + invalid_value if invalid_value else invalid_value}") - - with Given("LDAP server configuration that uses a negative integer for the verification_cooldown parameter"): - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": f"{invalid_value}" - }} - - with When("I try to use this configuration then it should not work"): - invalid_server_config(servers, message=error_message, tail=17, timeout=timeout) - -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_Syntax("2.0") -) -def syntax(self): - """Check that server configuration with valid syntax can be loaded. - ```xml - - - localhost - 636 - cn= - , ou=users, dc=example, dc=com - 0 - yes - tls1.2 - demand - /path/to/tls_cert_file - /path/to/tls_key_file - /path/to/tls_ca_cert_file - /path/to/tls_ca_cert_dir - ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 - - - ``` - """ - servers = { - "openldap2": { - "host": "openldap2", - "port": "389", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "0", - "enable_tls": "yes", - "tls_minimum_protocol_version": "tls1.2" , - "tls_require_cert": "demand", - "tls_cert_file": "/container/service/slapd/assets/certs/ldap.crt", - "tls_key_file": "/container/service/slapd/assets/certs/ldap.key", - "tls_ca_cert_file": "/container/service/slapd/assets/certs/ca.crt", - "tls_ca_cert_dir": "/container/service/slapd/assets/certs/", - "tls_cipher_suite": "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384" - } - } - with ldap_servers(servers): - pass - -@TestFeature -@Name("server config") -def feature(self, node="clickhouse1"): - """Check that LDAP server configuration. - """ - self.context.node = self.context.cluster.node(node) - - for scenario in loads(current_module(), Scenario): - scenario() diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py index d173547a916..3be79132ec3 100755 --- a/tests/testflows/helpers/cluster.py +++ b/tests/testflows/helpers/cluster.py @@ -37,6 +37,23 @@ class Node(object): self.cluster.command(None, f'{self.cluster.docker_compose} restart {self.name}', timeout=timeout) + def start(self, timeout=300, safe=True): + """Start node. + """ + self.cluster.command(None, f'{self.cluster.docker_compose} start {self.name}', timeout=timeout) + + + def stop(self, timeout=300, safe=True): + """Stop node. + """ + with self.cluster.lock: + for key in list(self.cluster._bash.keys()): + if key.endswith(f"-{self.name}"): + shell = self.cluster._bash.pop(key) + shell.__exit__(None, None, None) + + self.cluster.command(None, f'{self.cluster.docker_compose} stop {self.name}', timeout=timeout) + def command(self, *args, **kwargs): return self.cluster.command(self.name, *args, **kwargs) diff --git a/tests/testflows/ldap/authentication/regression.py b/tests/testflows/ldap/authentication/regression.py index ef24b29f3bb..ed75ce4fe75 100755 --- a/tests/testflows/ldap/authentication/regression.py +++ b/tests/testflows/ldap/authentication/regression.py @@ -29,8 +29,9 @@ xfails = { @TestFeature @Name("authentication") @ArgumentParser(argparser) -@Specifications(SRS_007_ClickHouse_Authentication_of_Users_via_LDAP) -@Requirements(RQ_SRS_007_LDAP_Authentication("1.0")) +@Requirements( + RQ_SRS_007_LDAP_Authentication("1.0") +) @XFails(xfails) def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """ClickHouse integration with LDAP regression module. diff --git a/tests/testflows/ldap/authentication/requirements/requirements.md b/tests/testflows/ldap/authentication/requirements/requirements.md index bade212c244..b0943d4a48b 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.md +++ b/tests/testflows/ldap/authentication/requirements/requirements.md @@ -59,25 +59,26 @@ * 4.2.27 [RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite](#rqsrs-007ldapconfigurationservertlsciphersuite) * 4.2.28 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown](#rqsrs-007ldapconfigurationserververificationcooldown) * 4.2.29 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default](#rqsrs-007ldapconfigurationserververificationcooldowndefault) - * 4.2.30 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) - * 4.2.31 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) - * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) - * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) - * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) - * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) - * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) - * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) - * 4.2.38 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) - * 4.2.39 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) - * 4.2.40 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) - * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) - * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) - * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) - * 4.2.44 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) - * 4.2.45 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) - * 4.2.46 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance](#rqsrs-007ldapauthenticationverificationcooldownperformance) - * 4.2.47 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters](#rqsrs-007ldapauthenticationverificationcooldownresetchangeincoreserverparameters) - * 4.2.48 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword](#rqsrs-007ldapauthenticationverificationcooldownresetinvalidpassword) + * 4.2.30 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid](#rqsrs-007ldapconfigurationserververificationcooldowninvalid) + * 4.2.31 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) + * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) + * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) + * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) + * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) + * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) + * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) + * 4.2.38 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) + * 4.2.39 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) + * 4.2.40 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) + * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) + * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) + * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) + * 4.2.44 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) + * 4.2.45 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) + * 4.2.46 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) + * 4.2.47 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance](#rqsrs-007ldapauthenticationverificationcooldownperformance) + * 4.2.48 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters](#rqsrs-007ldapauthenticationverificationcooldownresetchangeincoreserverparameters) + * 4.2.49 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword](#rqsrs-007ldapauthenticationverificationcooldownresetinvalidpassword) * 5 [References](#references) ## Revision History @@ -414,6 +415,25 @@ version: 1.0 SHALL have a default value of `0` that disables caching and forces contacting the [LDAP] server for each authentication request. +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid +version: 1.0 + +[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer. + +For example: + +* negative integer +* string +* empty value +* extremely large positive value (overflow) +* extremely large negative value (overflow) + +The error SHALL appear in the log and SHALL be similar to the following: + +```bash + Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value* +``` + #### RQ.SRS-007.LDAP.Configuration.Server.Syntax version: 2.0 diff --git a/tests/testflows/ldap/authentication/requirements/requirements.py b/tests/testflows/ldap/authentication/requirements/requirements.py index 4a9f18cd648..92c3f844781 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.py +++ b/tests/testflows/ldap/authentication/requirements/requirements.py @@ -84,25 +84,26 @@ SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( * 4.2.27 [RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite](#rqsrs-007ldapconfigurationservertlsciphersuite) * 4.2.28 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown](#rqsrs-007ldapconfigurationserververificationcooldown) * 4.2.29 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default](#rqsrs-007ldapconfigurationserververificationcooldowndefault) - * 4.2.30 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) - * 4.2.31 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) - * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) - * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) - * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) - * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) - * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) - * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) - * 4.2.38 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) - * 4.2.39 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) - * 4.2.40 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) - * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) - * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) - * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) - * 4.2.44 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) - * 4.2.45 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) - * 4.2.46 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance](#rqsrs-007ldapauthenticationverificationcooldownperformance) - * 4.2.47 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters](#rqsrs-007ldapauthenticationverificationcooldownresetchangeincoreserverparameters) - * 4.2.48 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword](#rqsrs-007ldapauthenticationverificationcooldownresetinvalidpassword) + * 4.2.30 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid](#rqsrs-007ldapconfigurationserververificationcooldowninvalid) + * 4.2.31 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) + * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) + * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) + * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) + * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) + * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) + * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) + * 4.2.38 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) + * 4.2.39 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) + * 4.2.40 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) + * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) + * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) + * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) + * 4.2.44 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) + * 4.2.45 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) + * 4.2.46 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) + * 4.2.47 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance](#rqsrs-007ldapauthenticationverificationcooldownperformance) + * 4.2.48 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters](#rqsrs-007ldapauthenticationverificationcooldownresetchangeincoreserverparameters) + * 4.2.49 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword](#rqsrs-007ldapauthenticationverificationcooldownresetinvalidpassword) * 5 [References](#references) ## Revision History @@ -439,6 +440,25 @@ version: 1.0 SHALL have a default value of `0` that disables caching and forces contacting the [LDAP] server for each authentication request. +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid +version: 1.0 + +[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer. + +For example: + +* negative integer +* string +* empty value +* extremely large positive value (overflow) +* extremely large negative value (overflow) + +The error SHALL appear in the log and SHALL be similar to the following: + +```bash + Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value* +``` + #### RQ.SRS-007.LDAP.Configuration.Server.Syntax version: 2.0 @@ -1331,6 +1351,33 @@ RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Default = Requirement( ), link=None) +RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Invalid = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer.\n' + '\n' + 'For example:\n' + '\n' + '* negative integer\n' + '* string\n' + '* empty value\n' + '* extremely large positive value (overflow)\n' + '* extremely large negative value (overflow)\n' + '\n' + 'The error SHALL appear in the log and SHALL be similar to the following:\n' + '\n' + '```bash\n' + ' Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value*\n' + '```\n' + '\n' + ), + link=None) + RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Syntax', version='2.0', diff --git a/tests/testflows/ldap/authentication/tests/authentications.py b/tests/testflows/ldap/authentication/tests/authentications.py index a64a37ed686..46bcae000b8 100644 --- a/tests/testflows/ldap/authentication/tests/authentications.py +++ b/tests/testflows/ldap/authentication/tests/authentications.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import random +import time from multiprocessing.dummy import Pool from testflows.core import * @@ -27,6 +28,7 @@ servers = { @TestStep(When) @Name("I login as {username} and execute query") +@Args(format_name=True) def login_and_execute_query(self, username, password, exitcode=None, message=None, steps=True): """Execute query as some user. """ @@ -129,7 +131,7 @@ def login_after_user_is_deleted_from_ldap(self, server, rbac=False): user = add_user_to_ldap(**user) with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml", - restart=True, rbac=rbac): + restart=True, rbac=rbac): login_and_execute_query(username=user["cn"], password=user["userpassword"]) with When("I delete this user from LDAP"): @@ -200,7 +202,7 @@ def login_after_user_cn_changed_in_ldap(self, server, rbac=False): user = add_user_to_ldap(**user) with ldap_authenticated_users({"username": user["cn"], "server": server}, - config_file=f"ldap_users_{getuid()}.xml", restart=True, rbac=rbac): + config_file=f"ldap_users_{getuid()}.xml", restart=True, rbac=rbac): login_and_execute_query(username=user["cn"], password=user["userpassword"]) with When("I change user password in LDAP"): @@ -474,6 +476,470 @@ def empty_username_and_empty_password(self, server=None, rbac=False): """ login_and_execute_query(username="", password="") +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Default("1.0") +) +def default_verification_cooldown_value(self, server, rbac=False, timeout=20): + """Check that the default value (0) for the verification cooldown parameter + disables caching and forces contacting the LDAP server for each + authentication request. + """ + + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + + with Given("I have an LDAP configuration that uses the default verification_cooldown value (0)"): + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") + + with Then("when I try to login immediately with the old user password it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message) + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0") +) +def valid_verification_cooldown_value_cn_change(self, server, rbac=False, timeout=20): + """Check that we can perform requests without contacting the LDAP server + after successful authentication when the verification_cooldown parameter + is set and the user cn is changed. + """ + + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + new_user = None + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "2" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user cn in LDAP"): + new_user = change_user_cn_in_ldap(user, "testVCD2") + + with Then("when I try to login again with the old user cn it should work"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("when I sleep for 2 seconds and try to log in, it should fail"): + time.sleep(2) + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message) + + finally: + with Finally("I make sure LDAP user is deleted"): + if new_user is not None: + delete_user_from_ldap(new_user, exitcode=None) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0") +) +def valid_verification_cooldown_value_password_change(self, server, rbac=False, timeout=20): + """Check that we can perform requests without contacting the LDAP server + after successful authentication when the verification_cooldown parameter + is set and the user password is changed. + """ + + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "2" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") + + with Then("when I try to login again with the old password it should work"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("when I sleep for 2 seconds and try to log in, it should fail"): + time.sleep(2) + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message) + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0") +) +def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False, timeout=20): + """Check that we can perform requests without contacting the LDAP server + after successful authentication when the verification_cooldown parameter + is set, even when the LDAP server is offline. + """ + + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "2" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add a new user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with ldap_authenticated_users({"username": user["cn"], "server": server}, + config_file=f"ldap_users_{getuid()}.xml"): + + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + try: + with And("then I stop the ldap server"): + self.context.ldap_node.stop() + + with Then("when I try to login again with the server offline it should work"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("when I sleep for 2 seconds and try to log in, it should fail"): + time.sleep(2) + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message) + + finally: + with Finally("I start the ldap server back up"): + self.context.ldap_node.start() + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestOutline +def repeat_requests(self, server, iterations, vcd_value, rbac=False): + """Run repeated requests from some user to the LDAP server. + """ + + user = None + + with Given(f"I have an LDAP configuration that sets verification_cooldown parameter to {vcd_value} sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": vcd_value + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with And("I add a new user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): + with When(f"I login and execute some query {iterations} times"): + start_time = time.time() + r = self.context.node.command(f"time for i in {{1..{iterations}}}; do clickhouse client -q \"SELECT 1\" --user {user['cn']} --password {user['userpassword']} > /dev/null; done") + end_time = time.time() + + return end_time - start_time + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Performance("1.0") +) +def verification_cooldown_performance(self, server, rbac=False, iterations=5000): + """Check that login performance is better when the verification cooldown + parameter is set to a positive value when comparing to the case when + the verification cooldown parameter is turned off. + """ + + vcd_time = 0 + no_vcd_time = 0 + + with Example(f"Repeated requests with verification cooldown parameter set to 600 seconds, {iterations} iterations"): + vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="600", rbac=rbac) + metric("login_with_vcd_value_600", units="seconds", value=vcd_time) + + with Example(f"Repeated requests with verification cooldown parameter set to 0 seconds, {iterations} iterations"): + no_vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="0", rbac=rbac) + metric("login_with_vcd_value_0", units="seconds", value=no_vcd_time) + + with Then("The performance with verification cooldown parameter set is better than the performance with no verification cooldown parameter."): + assert no_vcd_time > vcd_time, error() + + with And("Log the performance improvement as a percentage."): + metric("percentage_improvement", units="%", value=100*(no_vcd_time - vcd_time)/vcd_time) + +@TestOutline +def check_verification_cooldown_reset_on_core_server_parameter_change(self, server, + parameter_name, parameter_value, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after one of the core server + parameters is changed in the LDAP server configuration. + """ + + config_d_dir="/etc/clickhouse-server/config.d" + config_file="ldap_servers.xml" + error_message = "DB::Exception: {user}: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + config=None + updated_config=None + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 600 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "600" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + with And("LDAP authenticated user"): + users = [ + {"cn": f"testVCD_0", "userpassword": "testVCD_0"}, + {"cn": f"testVCD_1", "userpassword": "testVCD_1"} + ] + + with And("I create LDAP servers configuration file"): + config = create_ldap_servers_config_content(servers, config_d_dir, config_file) + + with ldap_users(*users) as users: + with ldap_servers(servers, restart=True): + with ldap_authenticated_users(*[{"username": user["cn"], "server": server} for user in users]): + with When("I login and execute a query"): + for user in users: + with By(f"as user {user['cn']}"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user password in LDAP"): + for user in users: + with By(f"for user {user['cn']}"): + change_user_password_in_ldap(user, "newpassword") + + with And(f"I change the server {parameter_name} core parameter", description=f"{parameter_value}"): + servers["openldap1"][parameter_name] = parameter_value + + with And("I create an updated the config file that has a different server host name"): + updated_config = create_ldap_servers_config_content(servers, config_d_dir, config_file) + + with modify_config(updated_config, restart=False): + with Then("when I try to log in it should fail as cache should have been reset"): + for user in users: + with By(f"as user {user['cn']}"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message.format(user=user["cn"])) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") +) +def verification_cooldown_reset_on_server_host_parameter_change(self, server, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after server host name + is changed in the LDAP server configuration. + """ + + check_verification_cooldown_reset_on_core_server_parameter_change(server=server, + parameter_name="host", parameter_value="openldap2", rbac=rbac) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") +) +def verification_cooldown_reset_on_server_port_parameter_change(self, server, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after server port is changed in the + LDAP server configuration. + """ + + check_verification_cooldown_reset_on_core_server_parameter_change(server=server, + parameter_name="port", parameter_value="9006", rbac=rbac) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") +) +def verification_cooldown_reset_on_server_auth_dn_prefix_parameter_change(self, server, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after server auth_dn_prefix + is changed in the LDAP server configuration. + """ + + check_verification_cooldown_reset_on_core_server_parameter_change(server=server, + parameter_name="auth_dn_prefix", parameter_value="cxx=", rbac=rbac) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") +) +def verification_cooldown_reset_on_server_auth_dn_suffix_parameter_change(self, server, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after server auth_dn_suffix + is changed in the LDAP server configuration. + """ + + check_verification_cooldown_reset_on_core_server_parameter_change(server=server, + parameter_name="auth_dn_suffix", + parameter_value=",ou=company,dc=users,dc=com", rbac=rbac) + + +@TestScenario +@Name("verification cooldown reset when invalid password is provided") +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_InvalidPassword("1.0") +) +def scenario(self, server, rbac=False): + """Check that cached bind requests for the user are discarded when + the user provides invalid login credentials. + """ + + user = None + error_exitcode = 4 + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 600 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "600" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add a new user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with ldap_authenticated_users({"username": user["cn"], "server": server}, + config_file=f"ldap_users_{getuid()}.xml"): + + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") + + with Then("When I try to log in with the cached password it should work"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("When I try to log in with an incorrect password it should fail"): + login_and_execute_query(username=user["cn"], password="incorrect", exitcode=error_exitcode, + message=error_message) + + with And("When I try to log in with the cached password it should fail"): + login_and_execute_query(username=user["cn"], password="incorrect", exitcode=error_exitcode, + message=error_message) + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestFeature +def verification_cooldown(self, rbac, servers=None, node="clickhouse1"): + """Check verification cooldown parameter functionality. + """ + for scenario in loads(current_module(), Scenario, filter=has.tag("verification_cooldown")): + scenario(server="openldap1", rbac=rbac) + + @TestOutline(Feature) @Name("user authentications") @Requirements( @@ -493,5 +959,11 @@ def feature(self, rbac, servers=None, node="clickhouse1"): servers = globals()["servers"] with ldap_servers(servers): - for scenario in loads(current_module(), Scenario): + for scenario in loads(current_module(), Scenario, filter=~has.tag("verification_cooldown")): scenario(server="openldap1", rbac=rbac) + + Feature(test=verification_cooldown)(rbac=rbac, servers=servers, node=node) + + + + diff --git a/tests/testflows/ldap/authentication/tests/common.py b/tests/testflows/ldap/authentication/tests/common.py index ed8d46df92b..8efb389a23f 100644 --- a/tests/testflows/ldap/authentication/tests/common.py +++ b/tests/testflows/ldap/authentication/tests/common.py @@ -78,7 +78,7 @@ def restart(node=None, safe=False, timeout=60): f"ConfigReloader: Loaded config '/etc/clickhouse-server/config.xml', performed update on configuration", timeout=timeout) -def add_config(config, timeout=60, restart=False): +def add_config(config, timeout=60, restart=False, modify=False): """Add dynamic configuration file to ClickHouse. :param node: node @@ -165,19 +165,20 @@ def add_config(config, timeout=60, restart=False): wait_for_config_to_be_loaded() yield finally: - with Finally(f"I remove {config.name}"): - with node.cluster.shell(node.name) as bash: - bash.expect(bash.prompt) - bash.send("tail -n 0 -f /var/log/clickhouse-server/clickhouse-server.log") + if not modify: + with Finally(f"I remove {config.name}"): + with node.cluster.shell(node.name) as bash: + bash.expect(bash.prompt) + bash.send("tail -n 0 -f /var/log/clickhouse-server/clickhouse-server.log") - with By("removing the config file", description=config.path): - node.command(f"rm -rf {config.path}", exitcode=0) + with By("removing the config file", description=config.path): + node.command(f"rm -rf {config.path}", exitcode=0) - with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): - check_preprocessed_config_is_updated(after_removal=True) + with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): + check_preprocessed_config_is_updated(after_removal=True) - with And("I wait for config to be reloaded"): - wait_for_config_to_be_loaded() + with And("I wait for config to be reloaded"): + wait_for_config_to_be_loaded() def create_ldap_servers_config_content(servers, config_d_dir="/etc/clickhouse-server/config.d", config_file="ldap_servers.xml"): """Create LDAP servers configuration content. @@ -201,12 +202,19 @@ def create_ldap_servers_config_content(servers, config_d_dir="/etc/clickhouse-se return Config(content, path, name, uid, "config.xml") +@contextmanager +def modify_config(config, restart=False): + """Apply updated configuration file. + """ + return add_config(config, restart=restart, modify=True) + @contextmanager def ldap_servers(servers, config_d_dir="/etc/clickhouse-server/config.d", config_file="ldap_servers.xml", - timeout=60, restart=False): + timeout=60, restart=False, config=None): """Add LDAP servers configuration. """ - config = create_ldap_servers_config_content(servers, config_d_dir, config_file) + if config is None: + config = create_ldap_servers_config_content(servers, config_d_dir, config_file) return add_config(config, restart=restart) def create_ldap_users_config_content(*users, config_d_dir="/etc/clickhouse-server/users.d", config_file="ldap_users.xml"): diff --git a/tests/testflows/ldap/authentication/tests/server_config.py b/tests/testflows/ldap/authentication/tests/server_config.py index 5b3a96caa9c..38ec859226b 100644 --- a/tests/testflows/ldap/authentication/tests/server_config.py +++ b/tests/testflows/ldap/authentication/tests/server_config.py @@ -217,6 +217,36 @@ def auth_dn_value(self): login(servers, user) +@TestOutline(Scenario) +@Examples("invalid_value", [ + ("-1", Name("negative int")), + ("foo", Name("string")), + ("", Name("empty string")), + ("36893488147419103232", Name("overflow with extremely large int value")), + ("-36893488147419103232", Name("overflow with extremely large negative int value")), + ("@#", Name("special characters")) +]) +@Requirements( + RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Invalid("1.0") +) +def invalid_verification_cooldown_value(self, invalid_value, timeout=20): + """Check that server returns an error when LDAP server + verification cooldown parameter is invalid. + """ + + error_message = (" Access(user directories): Could not parse LDAP server" + " \\`openldap1\\`: Poco::Exception. Code: 1000, e.code() = 0," + f" e.displayText() = Syntax error: Not a valid unsigned integer{': ' + invalid_value if invalid_value else invalid_value}") + + with Given("LDAP server configuration that uses a negative integer for the verification_cooldown parameter"): + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": f"{invalid_value}" + }} + + with When("I try to use this configuration then it should not work"): + invalid_server_config(servers, message=error_message, tail=17, timeout=timeout) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Configuration_Server_Syntax("2.0") From 61339ea3fc6a71496ae09d4d7a63f345b7205620 Mon Sep 17 00:00:00 2001 From: Tai White Date: Wed, 18 Nov 2020 01:36:35 +0100 Subject: [PATCH 008/284] fixed header for SRS in /ldap/authentication requirements --- tests/testflows/ldap/authentication/requirements/requirements.md | 1 + tests/testflows/ldap/authentication/requirements/requirements.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/testflows/ldap/authentication/requirements/requirements.md b/tests/testflows/ldap/authentication/requirements/requirements.md index b0943d4a48b..27ce8c921a0 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.md +++ b/tests/testflows/ldap/authentication/requirements/requirements.md @@ -1,4 +1,5 @@ # SRS-007 ClickHouse Authentication of Users via LDAP +# Software Requirements Specification ## Table of Contents diff --git a/tests/testflows/ldap/authentication/requirements/requirements.py b/tests/testflows/ldap/authentication/requirements/requirements.py index 92c3f844781..41d443c5df2 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.py +++ b/tests/testflows/ldap/authentication/requirements/requirements.py @@ -24,6 +24,7 @@ SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( children=None, content=''' # SRS-007 ClickHouse Authentication of Users via LDAP +# Software Requirements Specification ## Table of Contents From 2bc32fe29298e828f10c87014d1892f468aa0a5c Mon Sep 17 00:00:00 2001 From: Tai White Date: Fri, 20 Nov 2020 23:21:13 +0100 Subject: [PATCH 009/284] Added verification cooldown requirements and tests to the ldap/external_user_directory SRS and test files --- .../requirements/requirements.md | 51 +- .../requirements/requirements.py | 83 +++- .../tests/authentications.py | 465 +++++++++++++++++- .../external_user_directory/tests/common.py | 3 +- .../tests/server_config.py | 31 +- 5 files changed, 591 insertions(+), 42 deletions(-) diff --git a/tests/testflows/ldap/external_user_directory/requirements/requirements.md b/tests/testflows/ldap/external_user_directory/requirements/requirements.md index 74248196998..cf9650f2cae 100644 --- a/tests/testflows/ldap/external_user_directory/requirements/requirements.md +++ b/tests/testflows/ldap/external_user_directory/requirements/requirements.md @@ -82,20 +82,21 @@ * 4.2.3.28 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsciphersuite) * 4.2.3.29 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown](#rqsrs-009ldapexternaluserdirectoryconfigurationserververificationcooldown) * 4.2.3.30 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationserververificationcooldowndefault) - * 4.2.3.31 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationserversyntax) - * 4.2.3.32 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectory) - * 4.2.3.33 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectorymorethanone) - * 4.2.3.34 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationuserssyntax) - * 4.2.3.35 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserver) - * 4.2.3.36 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverempty) - * 4.2.3.37 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermissing) - * 4.2.3.38 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermorethanone) - * 4.2.3.39 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverinvalid) - * 4.2.3.40 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersroles) - * 4.2.3.41 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmorethanone) - * 4.2.3.42 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesinvalid) - * 4.2.3.43 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesempty) - * 4.2.3.44 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmissing) + * 4.2.3.31 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationserververificationcooldowninvalid) + * 4.2.3.32 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationserversyntax) + * 4.2.3.33 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectory) + * 4.2.3.34 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectorymorethanone) + * 4.2.3.35 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationuserssyntax) + * 4.2.3.36 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserver) + * 4.2.3.37 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverempty) + * 4.2.3.38 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermissing) + * 4.2.3.39 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermorethanone) + * 4.2.3.40 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverinvalid) + * 4.2.3.41 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersroles) + * 4.2.3.42 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmorethanone) + * 4.2.3.43 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesinvalid) + * 4.2.3.44 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesempty) + * 4.2.3.45 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmissing) * 4.2.4 [Authentication](#authentication) * 4.2.4.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty](#rqsrs-009ldapexternaluserdirectoryauthenticationusernameempty) * 4.2.4.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long](#rqsrs-009ldapexternaluserdirectoryauthenticationusernamelong) @@ -568,7 +569,7 @@ version: 1.0 that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed to be successfully authenticated for all consecutive requests without contacting the [LDAP] server. After period of time since the last successful attempt expires then on the authentication attempt -SHALL result in contacting the [LDAP] server to verify the username and password. +SHALL result in contacting the [LDAP] server to verify the username and password. ##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Default version: 1.0 @@ -577,6 +578,25 @@ version: 1.0 SHALL have a default value of `0` that disables caching and forces contacting the [LDAP] server for each authentication request. +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Invalid +version: 1.0 + +[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer. + +For example: + +* negative integer +* string +* empty value +* extremely large positive value (overflow) +* extremely large negative value (overflow) + +The error SHALL appear in the log and SHALL be similar to the following: + +```bash + Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value* +``` + ##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax version: 2.0 @@ -739,7 +759,6 @@ version: 1.0 [ClickHouse] SHALL support [UTF-8] characters in passwords used to authenticate users when using [LDAP] external user directory. - ##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Performance version: 1.0 diff --git a/tests/testflows/ldap/external_user_directory/requirements/requirements.py b/tests/testflows/ldap/external_user_directory/requirements/requirements.py index 3354d2b5dd7..fc370de7753 100644 --- a/tests/testflows/ldap/external_user_directory/requirements/requirements.py +++ b/tests/testflows/ldap/external_user_directory/requirements/requirements.py @@ -1,6 +1,6 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.201025.1200805. +# document by TestFlows v1.6.201102.1235648. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. from testflows.core import Specification @@ -107,20 +107,21 @@ SRS_009_ClickHouse_LDAP_External_User_Directory = Specification( * 4.2.3.28 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsciphersuite) * 4.2.3.29 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown](#rqsrs-009ldapexternaluserdirectoryconfigurationserververificationcooldown) * 4.2.3.30 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationserververificationcooldowndefault) - * 4.2.3.31 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationserversyntax) - * 4.2.3.32 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectory) - * 4.2.3.33 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectorymorethanone) - * 4.2.3.34 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationuserssyntax) - * 4.2.3.35 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserver) - * 4.2.3.36 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverempty) - * 4.2.3.37 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermissing) - * 4.2.3.38 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermorethanone) - * 4.2.3.39 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverinvalid) - * 4.2.3.40 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersroles) - * 4.2.3.41 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmorethanone) - * 4.2.3.42 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesinvalid) - * 4.2.3.43 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesempty) - * 4.2.3.44 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmissing) + * 4.2.3.31 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationserververificationcooldowninvalid) + * 4.2.3.32 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationserversyntax) + * 4.2.3.33 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectory) + * 4.2.3.34 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectorymorethanone) + * 4.2.3.35 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationuserssyntax) + * 4.2.3.36 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserver) + * 4.2.3.37 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverempty) + * 4.2.3.38 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermissing) + * 4.2.3.39 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermorethanone) + * 4.2.3.40 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverinvalid) + * 4.2.3.41 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersroles) + * 4.2.3.42 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmorethanone) + * 4.2.3.43 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesinvalid) + * 4.2.3.44 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesempty) + * 4.2.3.45 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmissing) * 4.2.4 [Authentication](#authentication) * 4.2.4.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty](#rqsrs-009ldapexternaluserdirectoryauthenticationusernameempty) * 4.2.4.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long](#rqsrs-009ldapexternaluserdirectoryauthenticationusernamelong) @@ -593,7 +594,7 @@ version: 1.0 that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed to be successfully authenticated for all consecutive requests without contacting the [LDAP] server. After period of time since the last successful attempt expires then on the authentication attempt -SHALL result in contacting the [LDAP] server to verify the username and password. +SHALL result in contacting the [LDAP] server to verify the username and password. ##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Default version: 1.0 @@ -602,6 +603,25 @@ version: 1.0 SHALL have a default value of `0` that disables caching and forces contacting the [LDAP] server for each authentication request. +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Invalid +version: 1.0 + +[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer. + +For example: + +* negative integer +* string +* empty value +* extremely large positive value (overflow) +* extremely large negative value (overflow) + +The error SHALL appear in the log and SHALL be similar to the following: + +```bash + Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value* +``` + ##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax version: 2.0 @@ -764,7 +784,6 @@ version: 1.0 [ClickHouse] SHALL support [UTF-8] characters in passwords used to authenticate users when using [LDAP] external user directory. - ##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Performance version: 1.0 @@ -1756,7 +1775,7 @@ RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown 'that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed\n' 'to be successfully authenticated for all consecutive requests without contacting the [LDAP] server.\n' 'After period of time since the last successful attempt expires then on the authentication attempt\n' - 'SHALL result in contacting the [LDAP] server to verify the username and password. \n' + 'SHALL result in contacting the [LDAP] server to verify the username and password.\n' '\n' ), link=None) @@ -1776,6 +1795,33 @@ RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown_ ), link=None) +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown_Invalid = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer.\n' + '\n' + 'For example:\n' + '\n' + '* negative integer\n' + '* string\n' + '* empty value\n' + '* extremely large positive value (overflow)\n' + '* extremely large negative value (overflow)\n' + '\n' + 'The error SHALL appear in the log and SHALL be similar to the following:\n' + '\n' + '```bash\n' + ' Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value*\n' + '```\n' + '\n' + ), + link=None) + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Syntax = Requirement( name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax', version='2.0', @@ -2093,7 +2139,6 @@ RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_UTF8 = Requirement '[ClickHouse] SHALL support [UTF-8] characters in passwords\n' 'used to authenticate users when using [LDAP] external user directory.\n' '\n' - '\n' ), link=None) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index 531a1b2f3ea..8651dd9903b 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -144,7 +144,6 @@ def parallel_login_with_the_same_user(self, server, timeout=200): join(tasks, timeout) @TestScenario -@Tags("custom config") def login_after_ldap_external_user_directory_is_removed(self, server): """Check that ClickHouse stops authenticating LDAP users after LDAP external user directory is removed. @@ -698,6 +697,460 @@ def empty_username_and_empty_password(self, server=None): """ login_and_execute_query(username="", password="") +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown_Default("1.0") +) +def default_verification_cooldown_value(self, server, rbac=False, timeout=20): + """Check that the default value (0) for the verification cooldown parameter + disables caching and forces contacting the LDAP server for each + authentication request. + """ + + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + + with Given("I have an LDAP configuration that uses the default verification_cooldown value (0)"): + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with rbac_roles("ldap_role") as roles: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") + + with Then("when I try to login immediately with the old user password it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message) + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown("1.0") +) +def valid_verification_cooldown_value_cn_change(self, server, rbac=False, timeout=20): + """Check that we can perform requests without contacting the LDAP server + after successful authentication when the verification_cooldown parameter + is set and the user cn is changed. + """ + + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + new_user = None + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "2" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with rbac_roles("ldap_role") as roles: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user cn in LDAP"): + new_user = change_user_cn_in_ldap(user, "testVCD2") + + with Then("when I try to login again with the old user cn it should work"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("when I sleep for 2 seconds and try to log in, it should fail"): + time.sleep(2) + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message) + + finally: + with Finally("I make sure LDAP user is deleted"): + if new_user is not None: + delete_user_from_ldap(new_user, exitcode=None) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown("1.0") +) +def valid_verification_cooldown_value_password_change(self, server, rbac=False, timeout=20): + """Check that we can perform requests without contacting the LDAP server + after successful authentication when the verification_cooldown parameter + is set and the user password is changed. + """ + + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "2" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with rbac_roles("ldap_role") as roles: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") + + with Then("when I try to login again with the old password it should work"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("when I sleep for 2 seconds and try to log in, it should fail"): + time.sleep(2) + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message) + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown("1.0") +) +def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False, timeout=20): + """Check that we can perform requests without contacting the LDAP server + after successful authentication when the verification_cooldown parameter + is set, even when the LDAP server is offline. + """ + + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "2" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add a new user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with rbac_roles("ldap_role") as roles: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + try: + with And("then I stop the ldap server"): + self.context.ldap_node.stop() + + with Then("when I try to login again with the server offline it should work"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("when I sleep for 2 seconds and try to log in, it should fail"): + time.sleep(2) + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message) + + finally: + with Finally("I start the ldap server back up"): + self.context.ldap_node.start() + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestOutline +def repeat_requests(self, server, iterations, vcd_value, rbac=False): + """Run repeated requests from some user to the LDAP server. + """ + + user = None + + with Given(f"I have an LDAP configuration that sets verification_cooldown parameter to {vcd_value} sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": vcd_value + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with And("I add a new user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with rbac_roles("ldap_role") as roles: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with When(f"I login and execute some query {iterations} times"): + start_time = time.time() + r = self.context.node.command(f"time for i in {{1..{iterations}}}; do clickhouse client -q \"SELECT 1\" --user {user['cn']} --password {user['userpassword']} > /dev/null; done") + end_time = time.time() + + return end_time - start_time + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Performance("1.0") +) +def verification_cooldown_performance(self, server, rbac=False, iterations=5000): + """Check that login performance is better when the verification cooldown + parameter is set to a positive value when comparing to the case when + the verification cooldown parameter is turned off. + """ + + vcd_time = 0 + no_vcd_time = 0 + + with Example(f"Repeated requests with verification cooldown parameter set to 600 seconds, {iterations} iterations"): + vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="600", rbac=rbac) + metric("login_with_vcd_value_600", units="seconds", value=vcd_time) + + with Example(f"Repeated requests with verification cooldown parameter set to 0 seconds, {iterations} iterations"): + no_vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="0", rbac=rbac) + metric("login_with_vcd_value_0", units="seconds", value=no_vcd_time) + + with Then("The performance with verification cooldown parameter set is better than the performance with no verification cooldown parameter."): + assert no_vcd_time > vcd_time, error() + + with And("Log the performance improvement as a percentage."): + metric("percentage_improvement", units="%", value=100*(no_vcd_time - vcd_time)/vcd_time) + +@TestOutline +def check_verification_cooldown_reset_on_core_server_parameter_change(self, server, + parameter_name, parameter_value, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after one of the core server + parameters is changed in the LDAP server configuration. + """ + config_d_dir="/etc/clickhouse-server/config.d" + config_file="ldap_servers.xml" + error_message = "DB::Exception: {user}: Authentication failed: password is incorrect or there is no user with such name" + error_exitcode = 4 + user = None + config=None + updated_config=None + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 600 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "600" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + with And("LDAP authenticated user"): + users = [ + {"cn": f"testVCD_0", "userpassword": "testVCD_0"}, + {"cn": f"testVCD_1", "userpassword": "testVCD_1"} + ] + + with And("I create LDAP servers configuration file"): + config = create_ldap_servers_config_content(servers, config_d_dir, config_file) + + with ldap_users(*users) as users: + with ldap_servers(servers=None, restart=False, config=config): + with rbac_roles("ldap_role") as roles: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with When("I login and execute a query"): + for user in users: + with By(f"as user {user['cn']}"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user password in LDAP"): + for user in users: + with By(f"for user {user['cn']}"): + change_user_password_in_ldap(user, "newpassword") + + with And(f"I change the server {parameter_name} core parameter", description=f"{parameter_value}"): + servers["openldap1"][parameter_name] = parameter_value + + with And("I create an updated the config file that has a different server host name"): + updated_config = create_ldap_servers_config_content(servers, config_d_dir, config_file) + + with modify_config(updated_config, restart=False): + with Then("when I try to log in it should fail as cache should have been reset"): + for user in users: + with By(f"as user {user['cn']}"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=error_exitcode, message=error_message.format(user=user["cn"])) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") +) +def verification_cooldown_reset_on_server_host_parameter_change(self, server, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after server host name + is changed in the LDAP server configuration. + """ + check_verification_cooldown_reset_on_core_server_parameter_change(server=server, + parameter_name="host", parameter_value="openldap2", rbac=rbac) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") +) +def verification_cooldown_reset_on_server_port_parameter_change(self, server, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after server port is changed in the + LDAP server configuration. + """ + check_verification_cooldown_reset_on_core_server_parameter_change(server=server, + parameter_name="port", parameter_value="9006", rbac=rbac) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") +) +def verification_cooldown_reset_on_server_auth_dn_prefix_parameter_change(self, server, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after server auth_dn_prefix + is changed in the LDAP server configuration. + """ + check_verification_cooldown_reset_on_core_server_parameter_change(server=server, + parameter_name="auth_dn_prefix", parameter_value="cxx=", rbac=rbac) + +@TestScenario +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") +) +def verification_cooldown_reset_on_server_auth_dn_suffix_parameter_change(self, server, rbac=False): + """Check that the LDAP login cache is reset for all the LDAP authentication users + when verification_cooldown parameter is set after server auth_dn_suffix + is changed in the LDAP server configuration. + """ + check_verification_cooldown_reset_on_core_server_parameter_change(server=server, + parameter_name="auth_dn_suffix", + parameter_value=",ou=company,dc=users,dc=com", rbac=rbac) + +@TestScenario +@Name("verification cooldown reset when invalid password is provided") +@Tags("verification_cooldown") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_InvalidPassword("1.0") +) +def scenario(self, server, rbac=False): + """Check that cached bind requests for the user are discarded when + the user provides invalid login credentials. + """ + user = None + error_exitcode = 4 + error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" + + with Given("I have an LDAP configuration that sets verification_cooldown parameter to 600 sec"): + servers = { "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "600" + }} + + self.context.ldap_node = self.context.cluster.node(server) + + try: + with Given("I add a new user to LDAP"): + user = {"cn": "testVCD", "userpassword": "testVCD"} + user = add_user_to_ldap(**user) + + with ldap_servers(servers): + with rbac_roles("ldap_role") as roles: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + + with When("I login and execute a query"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") + + with Then("When I try to log in with the cached password it should work"): + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with And("When I try to log in with an incorrect password it should fail"): + login_and_execute_query(username=user["cn"], password="incorrect", exitcode=error_exitcode, + message=error_message) + + with And("When I try to log in with the cached password it should fail"): + login_and_execute_query(username=user["cn"], password="incorrect", exitcode=error_exitcode, + message=error_message) + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Lookup_Priority("2.0") @@ -734,7 +1187,6 @@ def user_lookup_priority(self, server): with When("I try to login as 'ldap' user defined only in LDAP it should work"): login_and_execute_query(**users["ldap"]) - @TestOutline(Feature) @Name("user authentications") @Requirements( @@ -755,8 +1207,11 @@ def feature(self, servers=None, server=None, node="clickhouse1"): with ldap_servers(servers): with rbac_roles("ldap_role") as roles: with ldap_external_user_directory(server=server, roles=roles, restart=True): - for scenario in loads(current_module(), Scenario, filter=~has.tag("custom config")): - Scenario(test=scenario, flags=TE)(server=server) + for scenario in loads(current_module(), Scenario, filter=~has.tag("custom config") & ~has.tag("verification_cooldown")): + Scenario(test=scenario)(server=server) for scenario in loads(current_module(), Scenario, filter=has.tag("custom config")): - Scenario(test=scenario, flags=TE)(server=server) + Scenario(test=scenario)(server=server) + + for scenario in loads(current_module(), Scenario, filter=has.tag("verification_cooldown")): + Scenario(test=scenario)(server=server) diff --git a/tests/testflows/ldap/external_user_directory/tests/common.py b/tests/testflows/ldap/external_user_directory/tests/common.py index 38b53ca6e9f..6d8a97e8611 100644 --- a/tests/testflows/ldap/external_user_directory/tests/common.py +++ b/tests/testflows/ldap/external_user_directory/tests/common.py @@ -5,10 +5,11 @@ from contextlib import contextmanager import testflows.settings as settings from testflows.core import * from testflows.asserts import error -from ldap.authentication.tests.common import getuid, Config, ldap_servers, add_config, restart +from ldap.authentication.tests.common import getuid, Config, ldap_servers, add_config, modify_config, restart from ldap.authentication.tests.common import xmltree, xml_indent, xml_append, xml_with_utf8 from ldap.authentication.tests.common import ldap_user, ldap_users, add_user_to_ldap, delete_user_from_ldap from ldap.authentication.tests.common import change_user_password_in_ldap, change_user_cn_in_ldap +from ldap.authentication.tests.common import create_ldap_servers_config_content from ldap.authentication.tests.common import randomword def join(tasks, timeout): diff --git a/tests/testflows/ldap/external_user_directory/tests/server_config.py b/tests/testflows/ldap/external_user_directory/tests/server_config.py index 2512a4d88de..617c0ee32e5 100644 --- a/tests/testflows/ldap/external_user_directory/tests/server_config.py +++ b/tests/testflows/ldap/external_user_directory/tests/server_config.py @@ -99,7 +99,6 @@ def invalid_port(self): }] login(servers, "openldap1", *users) - @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), @@ -232,6 +231,36 @@ def auth_dn_value(self): login(servers, "openldap1", user) +@TestOutline(Scenario) +@Examples("invalid_value", [ + ("-1", Name("negative int")), + ("foo", Name("string")), + ("", Name("empty string")), + ("36893488147419103232", Name("overflow with extremely large int value")), + ("-36893488147419103232", Name("overflow with extremely large negative int value")), + ("@#", Name("special characters")) +]) +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown_Invalid("1.0") +) +def invalid_verification_cooldown_value(self, invalid_value, timeout=20): + """Check that server returns an error when LDAP server + verification cooldown parameter is invalid. + """ + + error_message = (" Access(user directories): Could not parse LDAP server" + " \\`openldap1\\`: Poco::Exception. Code: 1000, e.code() = 0," + f" e.displayText() = Syntax error: Not a valid unsigned integer{': ' + invalid_value if invalid_value else invalid_value}") + + with Given("LDAP server configuration that uses a negative integer for the verification_cooldown parameter"): + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": f"{invalid_value}" + }} + + with When("I try to use this configuration then it should not work"): + invalid_server_config(servers, message=error_message, tail=17, timeout=timeout) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Syntax("2.0") From c1662b6a4b66af33082eaac29d0b46eacc06c4b8 Mon Sep 17 00:00:00 2001 From: Tai White Date: Mon, 23 Nov 2020 21:21:30 +0100 Subject: [PATCH 010/284] Added line removed by mistake --- .../ldap/external_user_directory/tests/authentications.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index 8651dd9903b..8229947adf7 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -144,6 +144,7 @@ def parallel_login_with_the_same_user(self, server, timeout=200): join(tasks, timeout) @TestScenario +@Tags("custom config") def login_after_ldap_external_user_directory_is_removed(self, server): """Check that ClickHouse stops authenticating LDAP users after LDAP external user directory is removed. From 47be319dea985b1ef170184c447909ed54b86a0d Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 24 Nov 2020 20:17:58 +0400 Subject: [PATCH 011/284] Refactor the cached ldap info locations Add cached value access synchronization --- src/Access/Authentication.cpp | 62 +++++++++++++++++++++-------------- src/Access/Authentication.h | 9 ++--- src/Access/IAccessStorage.cpp | 2 +- src/Access/User.cpp | 16 +++++++++ src/Access/User.h | 18 ++++++++++ 5 files changed, 75 insertions(+), 32 deletions(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index b55ffad2dfa..2455869f783 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -49,7 +50,7 @@ Authentication::Digest Authentication::getPasswordDoubleSHA1() const } -bool Authentication::isCorrectPassword(const String & password_, const String & user_, const ExternalAuthenticators & external_authenticators) const +bool Authentication::isCorrectPassword(const User & user_, const String & password_, const ExternalAuthenticators & external_authenticators) const { switch (type) { @@ -83,41 +84,52 @@ bool Authentication::isCorrectPassword(const String & password_, const String & case LDAP_SERVER: { auto ldap_server_params = external_authenticators.getLDAPServerParams(server_name); - ldap_server_params.user = user_; + ldap_server_params.user = user_.getName(); ldap_server_params.password = password_; const auto current_params_hash = ldap_server_params.getCoreHash(); - const auto last_check_period = std::chrono::steady_clock::now() - ldap_last_successful_password_check_timestamp; + auto & ldap_last_successful_password_check_params_hash = user_.cache.ldap_last_successful_password_check_params_hash; + auto & ldap_last_successful_password_check_timestamp = user_.cache.ldap_last_successful_password_check_timestamp; - if ( - // Forbid the initial values explicitly. - ldap_last_successful_password_check_params_hash != 0 && - ldap_last_successful_password_check_timestamp != std::chrono::steady_clock::time_point{} && - - // Check if the caching is enabled at all. - ldap_server_params.verification_cooldown > std::chrono::seconds{0} && - - // Check if we can "reuse" the result of the previous successful password verification. - current_params_hash == ldap_last_successful_password_check_params_hash && - last_check_period >= std::chrono::seconds{0} && - last_check_period <= ldap_server_params.verification_cooldown - ) { - return true; + std::scoped_lock lock(user_.cache.mutex); + const auto last_check_period = std::chrono::steady_clock::now() - ldap_last_successful_password_check_timestamp; + + if ( + // Check if the caching is enabled at all. + ldap_server_params.verification_cooldown > std::chrono::seconds{0} && + + // Forbid the initial values explicitly. + ldap_last_successful_password_check_params_hash != 0 && + ldap_last_successful_password_check_timestamp != std::chrono::steady_clock::time_point{} && + + // Check if we can "reuse" the result of the previous successful password verification. + current_params_hash == ldap_last_successful_password_check_params_hash && + last_check_period >= std::chrono::seconds{0} && + last_check_period <= ldap_server_params.verification_cooldown + ) + { + return true; + } } LDAPSimpleAuthClient ldap_client(ldap_server_params); const auto result = ldap_client.check(); + const auto current_check_timestamp = std::chrono::steady_clock::now(); - if (result) { - ldap_last_successful_password_check_params_hash = current_params_hash; - ldap_last_successful_password_check_timestamp = std::chrono::steady_clock::now(); - } - else - { - ldap_last_successful_password_check_params_hash = 0; - ldap_last_successful_password_check_timestamp = std::chrono::steady_clock::time_point{}; + std::scoped_lock lock(user_.cache.mutex); + + if (result) + { + ldap_last_successful_password_check_params_hash = current_params_hash; + ldap_last_successful_password_check_timestamp = current_check_timestamp; + } + else + { + ldap_last_successful_password_check_params_hash = 0; + ldap_last_successful_password_check_timestamp = std::chrono::steady_clock::time_point{}; + } } return result; diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h index 65bbcd94720..abbd43555b1 100644 --- a/src/Access/Authentication.h +++ b/src/Access/Authentication.h @@ -19,6 +19,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; } +struct User; class ExternalAuthenticators; /// Authentication type and encrypted password for checking when an user logins. @@ -89,8 +90,8 @@ public: void setServerName(const String & server_name_); /// Checks if the provided password is correct. Returns false if not. - /// User name and external authenticators' info are used only by some specific authentication type (e.g., LDAP_SERVER). - bool isCorrectPassword(const String & password_, const String & user_, const ExternalAuthenticators & external_authenticators) const; + /// User instance and external authenticators' info are used only by some specific authentication type (e.g., LDAP_SERVER). + bool isCorrectPassword(const User & user_, const String & password_, const ExternalAuthenticators & external_authenticators) const; friend bool operator ==(const Authentication & lhs, const Authentication & rhs) { return (lhs.type == rhs.type) && (lhs.password_hash == rhs.password_hash); } friend bool operator !=(const Authentication & lhs, const Authentication & rhs) { return !(lhs == rhs); } @@ -104,11 +105,7 @@ private: Type type = Type::NO_PASSWORD; Digest password_hash; - - // Used and maintained only for LDAP. String server_name; - mutable std::size_t ldap_last_successful_password_check_params_hash = 0; - mutable std::chrono::steady_clock::time_point ldap_last_successful_password_check_timestamp; }; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 58821e7de4b..01516017d89 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -463,7 +463,7 @@ UUID IAccessStorage::loginImpl( bool IAccessStorage::isPasswordCorrectImpl(const User & user, const String & password, const ExternalAuthenticators & external_authenticators) const { - return user.authentication.isCorrectPassword(password, user.getName(), external_authenticators); + return user.authentication.isCorrectPassword(user, password, external_authenticators); } diff --git a/src/Access/User.cpp b/src/Access/User.cpp index f57ec7c1359..b5ba098d0bc 100644 --- a/src/Access/User.cpp +++ b/src/Access/User.cpp @@ -14,4 +14,20 @@ bool User::equal(const IAccessEntity & other) const && (settings == other_user.settings); } +UserEtcCache & UserEtcCache::operator= (const UserEtcCache & other) +{ + std::scoped_lock lock(mutex, other.mutex); + ldap_last_successful_password_check_params_hash = other.ldap_last_successful_password_check_params_hash; + ldap_last_successful_password_check_timestamp = other.ldap_last_successful_password_check_timestamp; + return *this; +} + +UserEtcCache & UserEtcCache::operator= (const UserEtcCache && other) +{ + std::scoped_lock lock(mutex, other.mutex); + ldap_last_successful_password_check_params_hash = std::move(other.ldap_last_successful_password_check_params_hash); + ldap_last_successful_password_check_timestamp = std::move(other.ldap_last_successful_password_check_timestamp); + return *this; +} + } diff --git a/src/Access/User.h b/src/Access/User.h index 13f1e532015..0ad0068794d 100644 --- a/src/Access/User.h +++ b/src/Access/User.h @@ -8,9 +8,26 @@ #include #include +#include +#include namespace DB { + +/** Various cached data bound to a User instance. Access to any member must be synchronized via 'mutex' member. + */ +struct UserEtcCache { + mutable std::recursive_mutex mutex; + std::size_t ldap_last_successful_password_check_params_hash = 0; + std::chrono::steady_clock::time_point ldap_last_successful_password_check_timestamp; + + explicit UserEtcCache() = default; + explicit UserEtcCache(const UserEtcCache & other) { (*this) = other; } + explicit UserEtcCache(const UserEtcCache && other) { (*this) = std::move(other); } + UserEtcCache & operator= (const UserEtcCache & other); + UserEtcCache & operator= (const UserEtcCache && other); +}; + /** User and ACL. */ struct User : public IAccessEntity @@ -21,6 +38,7 @@ struct User : public IAccessEntity GrantedRoles granted_roles; RolesOrUsersSet default_roles = RolesOrUsersSet::AllTag{}; SettingsProfileElements settings; + mutable UserEtcCache cache; bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } From d12f59388a0c0dee3f5cb2201797a15f62e8b79e Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 24 Nov 2020 22:48:15 +0400 Subject: [PATCH 012/284] Compilation fix --- src/Access/User.cpp | 2 +- src/Access/User.h | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Access/User.cpp b/src/Access/User.cpp index b5ba098d0bc..9b705fbdb5e 100644 --- a/src/Access/User.cpp +++ b/src/Access/User.cpp @@ -22,7 +22,7 @@ UserEtcCache & UserEtcCache::operator= (const UserEtcCache & other) return *this; } -UserEtcCache & UserEtcCache::operator= (const UserEtcCache && other) +UserEtcCache & UserEtcCache::operator= (UserEtcCache && other) { std::scoped_lock lock(mutex, other.mutex); ldap_last_successful_password_check_params_hash = std::move(other.ldap_last_successful_password_check_params_hash); diff --git a/src/Access/User.h b/src/Access/User.h index 0ad0068794d..ed3366b4712 100644 --- a/src/Access/User.h +++ b/src/Access/User.h @@ -16,16 +16,17 @@ namespace DB /** Various cached data bound to a User instance. Access to any member must be synchronized via 'mutex' member. */ -struct UserEtcCache { +struct UserEtcCache +{ mutable std::recursive_mutex mutex; std::size_t ldap_last_successful_password_check_params_hash = 0; std::chrono::steady_clock::time_point ldap_last_successful_password_check_timestamp; explicit UserEtcCache() = default; explicit UserEtcCache(const UserEtcCache & other) { (*this) = other; } - explicit UserEtcCache(const UserEtcCache && other) { (*this) = std::move(other); } + explicit UserEtcCache(UserEtcCache && other) { (*this) = std::move(other); } UserEtcCache & operator= (const UserEtcCache & other); - UserEtcCache & operator= (const UserEtcCache && other); + UserEtcCache & operator= (UserEtcCache && other); }; /** User and ACL. From cd8e7981e08dace480fd340646e2504301d5d96c Mon Sep 17 00:00:00 2001 From: vdimir Date: Sun, 29 Nov 2020 20:54:46 +0300 Subject: [PATCH 013/284] Speedup applyCIDRMask for IPv6 with compile-time generated mask array --- src/Common/IPv6ToBinary.cpp | 33 +++++++++++++++++++++++++++++++ src/Common/IPv6ToBinary.h | 5 +++++ src/Functions/FunctionsCoding.h | 35 ++++++++++++++++++++++----------- 3 files changed, 61 insertions(+), 12 deletions(-) diff --git a/src/Common/IPv6ToBinary.cpp b/src/Common/IPv6ToBinary.cpp index 1fd2e3312f6..da213bcb5a9 100644 --- a/src/Common/IPv6ToBinary.cpp +++ b/src/Common/IPv6ToBinary.cpp @@ -2,12 +2,18 @@ #include #include +#include + #include namespace DB { +constexpr size_t IPV6_MASKS_COUNT = 256; + +using RawMaskArray = std::array; + void IPv6ToRawBinary(const Poco::Net::IPAddress & address, char * res) { if (Poco::Net::IPAddress::IPv6 == address.family()) @@ -33,4 +39,31 @@ std::array IPv6ToBinary(const Poco::Net::IPAddress & address) return res; } +static constexpr RawMaskArray generateBitMask(size_t prefix) { + if (prefix >= 128) + prefix = 128; + RawMaskArray arr{0}; + size_t i = 0; + for (; prefix >= 8; ++i, prefix -= 8) + arr[i] = 0xff; + if (prefix > 0) + arr[i++] = ~(0xff >> prefix); + while (i < 16) + arr[i++] = 0x00; + return arr; +} + +static constexpr std::array generateBitMasks() { + std::array arr{}; + for (size_t i = 0; i < IPV6_MASKS_COUNT; ++i) + arr[i] = generateBitMask(i); + return arr; +} + +const uint8_t * getCIDRMaskIPv6(UInt8 prefix_len) +{ + static constexpr std::array IPV6_RAW_MASK_ARRAY = generateBitMasks(); + return IPV6_RAW_MASK_ARRAY[prefix_len].data(); +} + } diff --git a/src/Common/IPv6ToBinary.h b/src/Common/IPv6ToBinary.h index 2d0d4a20ecb..2e47238aeba 100644 --- a/src/Common/IPv6ToBinary.h +++ b/src/Common/IPv6ToBinary.h @@ -14,4 +14,9 @@ void IPv6ToRawBinary(const Poco::Net::IPAddress & address, char * res); /// Convert IP address to 16-byte array with IPv6 data (big endian). If it's an IPv4, map it to IPv6. std::array IPv6ToBinary(const Poco::Net::IPAddress & address); +/// Returns pointer to 16-byte array containing mask with first `prefix_len` bits set to `1` and `128 - prefix_len` to `0`. +/// Pointer is valid during all program execution time and doesn't require freeing. +/// Values of prefix_len greater than 128 interpreted as 128 exactly. +const uint8_t * getCIDRMaskIPv6(UInt8 prefix_len); + } diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index ac3262f5131..e07df450206 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -1,7 +1,8 @@ #pragma once -#include #include +#include +#include #include #include #include @@ -1617,20 +1618,28 @@ public: class FunctionIPv6CIDRToRange : public IFunction { private: - /// TODO Inefficient. + +#if defined(__SSE2__) + + #include + + static inline void applyCIDRMask(const UInt8 * __restrict src, UInt8 * __restrict dst_lower, UInt8 * __restrict dst_upper, UInt8 bits_to_keep) + { + __m128i mask = _mm_loadu_si128(reinterpret_cast(getCIDRMaskIPv6(bits_to_keep))); + __m128i lower = _mm_and_si128(_mm_loadu_si128(reinterpret_cast(src)), mask); + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst_lower), lower); + + __m128i inv_mask = _mm_xor_si128(mask, _mm_cmpeq_epi32(_mm_setzero_si128(), _mm_setzero_si128())); + __m128i upper = _mm_or_si128(lower, inv_mask); + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst_upper), upper); + } + +#else + /// NOTE IPv6 is stored in memory in big endian format that makes some difficulties. static void applyCIDRMask(const UInt8 * __restrict src, UInt8 * __restrict dst_lower, UInt8 * __restrict dst_upper, UInt8 bits_to_keep) { - UInt8 mask[16]{}; - - UInt8 bytes_to_keep = bits_to_keep / 8; - UInt8 bits_to_keep_in_last_byte = bits_to_keep % 8; - - for (size_t i = 0; i < bits_to_keep / 8; ++i) - mask[i] = 0xFFU; - - if (bits_to_keep_in_last_byte) - mask[bytes_to_keep] = 0xFFU << (8 - bits_to_keep_in_last_byte); + const auto * mask = getCIDRMaskIPv6(bits_to_keep); for (size_t i = 0; i < 16; ++i) { @@ -1639,6 +1648,8 @@ private: } } +#endif + public: static constexpr auto name = "IPv6CIDRToRange"; static FunctionPtr create(const Context &) { return std::make_shared(); } From 1aaff75d9ae1b38e74b5e14708bb2168d408301c Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 1 Dec 2020 20:38:49 +0300 Subject: [PATCH 014/284] Fix style in IPv6ToBinary.cpp --- src/Common/IPv6ToBinary.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Common/IPv6ToBinary.cpp b/src/Common/IPv6ToBinary.cpp index da213bcb5a9..94b831fcc35 100644 --- a/src/Common/IPv6ToBinary.cpp +++ b/src/Common/IPv6ToBinary.cpp @@ -39,7 +39,8 @@ std::array IPv6ToBinary(const Poco::Net::IPAddress & address) return res; } -static constexpr RawMaskArray generateBitMask(size_t prefix) { +static constexpr RawMaskArray generateBitMask(size_t prefix) +{ if (prefix >= 128) prefix = 128; RawMaskArray arr{0}; @@ -53,7 +54,8 @@ static constexpr RawMaskArray generateBitMask(size_t prefix) { return arr; } -static constexpr std::array generateBitMasks() { +static constexpr std::array generateBitMasks() +{ std::array arr{}; for (size_t i = 0; i < IPV6_MASKS_COUNT; ++i) arr[i] = generateBitMask(i); From 9ce010e82cbdef46cfbd87d13c8f82cc7cfa4cc7 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 1 Dec 2020 22:12:11 +0300 Subject: [PATCH 015/284] Add comment for IPV6_MASKS_COUNT --- src/Common/IPv6ToBinary.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Common/IPv6ToBinary.cpp b/src/Common/IPv6ToBinary.cpp index 94b831fcc35..3c004a5a84e 100644 --- a/src/Common/IPv6ToBinary.cpp +++ b/src/Common/IPv6ToBinary.cpp @@ -10,6 +10,8 @@ namespace DB { +/// Result array could be indexed with all possible uint8 values without extra check. +/// For values greater than 128 we will store same value as for 128 (all bits set). constexpr size_t IPV6_MASKS_COUNT = 256; using RawMaskArray = std::array; From d1c7833f28976ebb62bea002889a2a4facf0f51f Mon Sep 17 00:00:00 2001 From: Daria Mozhaeva Date: Wed, 9 Dec 2020 17:03:38 +0300 Subject: [PATCH 016/284] Edit and translate. --- docs/en/operations/settings/settings.md | 2 +- docs/en/operations/system-tables/errors.md | 4 +-- .../sql-reference/functions/hash-functions.md | 4 +-- docs/ru/operations/settings/settings.md | 17 ++++++++- docs/ru/operations/system-tables/errors.md | 23 ++++++++++++ docs/ru/sql-reference/data-types/date.md | 35 +++++++++++++++++++ .../sql-reference/functions/hash-functions.md | 7 ++-- .../functions/other-functions.md | 20 +++++++++++ 8 files changed, 104 insertions(+), 8 deletions(-) create mode 100644 docs/ru/operations/system-tables/errors.md diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index c5e44d4c464..01e15f772ac 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -726,7 +726,7 @@ log_queries=1 ## log_queries_min_query_duration_ms {#settings-log-queries-min-query-duration-ms} -Minimal time for the query to run to get to the following tables: +Minimum query execution time to navigate to the following tables: - `system.query_log` - `system.query_thread_log` diff --git a/docs/en/operations/system-tables/errors.md b/docs/en/operations/system-tables/errors.md index 53e8a397217..1366855b5ce 100644 --- a/docs/en/operations/system-tables/errors.md +++ b/docs/en/operations/system-tables/errors.md @@ -1,12 +1,12 @@ # system.errors {#system_tables-errors} -Contains error codes with number of times they have been triggered. +Contains error codes with the number of times they have been triggered. Columns: - `name` ([String](../../sql-reference/data-types/string.md)) — name of the error (`errorCodeToName`). - `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — code number of the error. -- `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) - number of times this error has been happened. +- `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) - the number of times this error has been happened. **Example** diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index 3594737c18a..9394426b20b 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -157,14 +157,14 @@ Levels are the same as in URLHierarchy. This function is specific to Yandex.Metr ## farmHash64 {#farmhash64} -Produces a 64-bit [FarmHash](https://github.com/google/farmhash) or Fingerprint value. Prefer `farmFingerprint64` for a stable and portable value. +Produces a 64-bit [FarmHash](https://github.com/google/farmhash) or Fingerprint value. `farmFingerprint64` is preferred for a stable and portable value. ``` sql farmFingerprint64(par1, ...) farmHash64(par1, ...) ``` -These functions use the `Fingerprint64` and `Hash64` method respectively from all [available methods](https://github.com/google/farmhash/blob/master/src/farmhash.h). +These functions use the `Fingerprint64` and `Hash64` methods respectively from all [available methods](https://github.com/google/farmhash/blob/master/src/farmhash.h). **Parameters** diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 0de29112d1b..c027c7c170b 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -412,7 +412,7 @@ INSERT INTO table_with_enum_column_for_tsv_insert FORMAT TSV 102 2; - `'basic'` — используется базовый парсер. - ClickHouse может парсить только базовый формат `YYYY-MM-DD HH:MM:SS`. Например, `'2019-08-20 10:18:56'`. + ClickHouse может парсить только базовый формат `YYYY-MM-DD HH:MM:SS` или `YYYY-MM-DD`. Например, `'2019-08-20 10:18:56'` или `2019-08-20`. Значение по умолчанию: `'basic'`. @@ -691,6 +691,21 @@ ClickHouse использует этот параметр при чтении д log_queries=1 ``` +## log_queries_min_query_duration_ms {#settings-log-queries-min-query-duration-ms} + +Минимальное время выполнения запроса для перехода к следующим таблицам: + +- `system.query_log` +- `system.query_thread_log` + +В лог попадут только запросы следующего типа: + +- `QUERY_FINISH` +- `EXCEPTION_WHILE_PROCESSING` + +- Тип: milliseconds +- Значение по умолчанию: 0 (для любого запроса) + ## log_queries_min_type {#settings-log-queries-min-type} Задаёт минимальный уровень логирования в `query_log`. diff --git a/docs/ru/operations/system-tables/errors.md b/docs/ru/operations/system-tables/errors.md new file mode 100644 index 00000000000..43c5e0852f0 --- /dev/null +++ b/docs/ru/operations/system-tables/errors.md @@ -0,0 +1,23 @@ +# system.errors {#system_tables-errors} + +Содержит коды ошибок с указанием количества срабатываний. + +Столбцы: + +- `name` ([String](../../sql-reference/data-types/string.md)) — название ошибки (`errorCodeToName`). +- `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — номер кода ошибки. +- `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) - сколько раз возникала эта ошибка. + +**Пример** + +``` sql +SELECT * +FROM system.errors +WHERE value > 0 +ORDER BY code ASC +LIMIT 1 + +┌─name─────────────┬─code─┬─value─┐ +│ CANNOT_OPEN_FILE │ 76 │ 1 │ +└──────────────────┴──────┴───────┘ +``` diff --git a/docs/ru/sql-reference/data-types/date.md b/docs/ru/sql-reference/data-types/date.md index 9bcae2c1d72..e2162f22ff5 100644 --- a/docs/ru/sql-reference/data-types/date.md +++ b/docs/ru/sql-reference/data-types/date.md @@ -9,4 +9,39 @@ toc_title: Date Дата хранится без учёта часового пояса. +## Примеры {#examples} + +**1.** Создание таблицы со столбцом типа `DateTime` и добавление в неё данных: + +``` sql +CREATE TABLE dt +( + `timestamp` Date, + `event_id` UInt8 +) +ENGINE = TinyLog; +``` + +``` sql +INSERT INTO dt Values (1546300800, 1), ('2019-01-01', 2); +``` + +``` sql +SELECT * FROM dt; +``` + +``` text +┌──timestamp─┬─event_id─┐ +│ 2019-01-01 │ 1 │ +│ 2019-01-01 │ 2 │ +└────────────┴──────────┘ +``` + +## Смотрите также {#see-also} + +- [Функции для работы с датой и временем](../../sql-reference/functions/date-time-functions.md) +- [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime) +- [Тип данных `DateTime`](../../sql-reference/data-types/datetime.md) + + [Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/date/) diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md index 92fc69227f4..d9c02ef9883 100644 --- a/docs/ru/sql-reference/functions/hash-functions.md +++ b/docs/ru/sql-reference/functions/hash-functions.md @@ -153,15 +153,18 @@ SELECT groupBitXor(cityHash64(*)) FROM table `URLHash(s, N)` - вычислить хэш от строки до N-го уровня в иерархии URL, без одного завершающего символа `/`, `?` или `#` на конце, если там такой есть. Уровни аналогичные URLHierarchy. Функция специфична для Яндекс.Метрики. +## farmFingerprint64 {#farmfingerprint64} + ## farmHash64 {#farmhash64} -Генерирует 64-х битное значение [FarmHash](https://github.com/google/farmhash). +Создает 64-битное значение [FarmHash](https://github.com/google/farmhash) или значение. `farmFingerprint64` предпочтительнее для стабильных и перемещаемых значений. ``` sql +farmFingerprint64(par1, ...) farmHash64(par1, ...) ``` -Из всех [доступных методов](https://github.com/google/farmhash/blob/master/src/farmhash.h) функция использует `Hash64`. +Эти функции используют методы `Fingerprint64` и `Hash64` из всех [доступных методов](https://github.com/google/farmhash/blob/master/src/farmhash.h). **Параметры** diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 91bf0f5b3a0..587bc497da8 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -1589,4 +1589,24 @@ SELECT countDigits(toDecimal32(1, 9)), countDigits(toDecimal32(-1, 9)), 10 10 19 19 39 39 ``` +## errorCodeToName {#error-code-to-name} + +**Возвращаемое значение** + +- Название переменной для кода ошибки. + +Тип: [LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md). + +**Синтаксис** + +``` sql +errorCodeToName(1) +``` + +Результат: + +``` text +UNSUPPORTED_METHOD +``` + [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/other_functions/) From 843a26cbca218c12527328352cee6142e24912b1 Mon Sep 17 00:00:00 2001 From: Daria Mozhaeva Date: Wed, 9 Dec 2020 18:50:58 +0300 Subject: [PATCH 017/284] Edit and translate v2. --- docs/en/operations/system-tables/errors.md | 2 +- docs/ru/operations/settings/settings.md | 4 ++-- docs/ru/operations/system-tables/errors.md | 2 +- docs/ru/sql-reference/functions/hash-functions.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/operations/system-tables/errors.md b/docs/en/operations/system-tables/errors.md index 1366855b5ce..ec874efd711 100644 --- a/docs/en/operations/system-tables/errors.md +++ b/docs/en/operations/system-tables/errors.md @@ -6,7 +6,7 @@ Columns: - `name` ([String](../../sql-reference/data-types/string.md)) — name of the error (`errorCodeToName`). - `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — code number of the error. -- `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) - the number of times this error has been happened. +- `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — the number of times this error has been happened. **Example** diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index c027c7c170b..e7355935687 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -408,11 +408,11 @@ INSERT INTO table_with_enum_column_for_tsv_insert FORMAT TSV 102 2; - `'best_effort'` — включает расширенный парсинг. - ClickHouse может парсить базовый формат `YYYY-MM-DD HH:MM:SS` и все форматы [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601). Например, `'2018-06-08T01:02:03.000Z'`. +ClickHouse может парсить базовый формат `YYYY-MM-DD HH:MM:SS` и все форматы [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601). Например, `'2018-06-08T01:02:03.000Z'`. - `'basic'` — используется базовый парсер. - ClickHouse может парсить только базовый формат `YYYY-MM-DD HH:MM:SS` или `YYYY-MM-DD`. Например, `'2019-08-20 10:18:56'` или `2019-08-20`. +ClickHouse может парсить только базовый формат `YYYY-MM-DD HH:MM:SS` или `YYYY-MM-DD`. Например, `'2019-08-20 10:18:56'` или `2019-08-20`. Значение по умолчанию: `'basic'`. diff --git a/docs/ru/operations/system-tables/errors.md b/docs/ru/operations/system-tables/errors.md index 43c5e0852f0..321920e8535 100644 --- a/docs/ru/operations/system-tables/errors.md +++ b/docs/ru/operations/system-tables/errors.md @@ -6,7 +6,7 @@ - `name` ([String](../../sql-reference/data-types/string.md)) — название ошибки (`errorCodeToName`). - `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — номер кода ошибки. -- `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) - сколько раз возникала эта ошибка. +- `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — число, возникающих ошибок. **Пример** diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md index d9c02ef9883..5dfdff91c31 100644 --- a/docs/ru/sql-reference/functions/hash-functions.md +++ b/docs/ru/sql-reference/functions/hash-functions.md @@ -157,7 +157,7 @@ SELECT groupBitXor(cityHash64(*)) FROM table ## farmHash64 {#farmhash64} -Создает 64-битное значение [FarmHash](https://github.com/google/farmhash) или значение. `farmFingerprint64` предпочтительнее для стабильных и перемещаемых значений. +Создает 64-битное значение [FarmHash](https://github.com/google/farmhash) или значение Fingerprint. `farmFingerprint64` предпочтительнее для стабильных и перемещаемых значений. ``` sql farmFingerprint64(par1, ...) From c2f1885b123ad4833c422ec21de32847661ceb40 Mon Sep 17 00:00:00 2001 From: damozhaeva <68770561+damozhaeva@users.noreply.github.com> Date: Fri, 11 Dec 2020 11:17:40 +0300 Subject: [PATCH 018/284] Update docs/ru/operations/system-tables/errors.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/operations/system-tables/errors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/errors.md b/docs/ru/operations/system-tables/errors.md index 321920e8535..3a824c8c834 100644 --- a/docs/ru/operations/system-tables/errors.md +++ b/docs/ru/operations/system-tables/errors.md @@ -6,7 +6,7 @@ - `name` ([String](../../sql-reference/data-types/string.md)) — название ошибки (`errorCodeToName`). - `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — номер кода ошибки. -- `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — число, возникающих ошибок. +- `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — количество ошибок. **Пример** From 8b2feaeca24b15415a707daba6494b06cb780292 Mon Sep 17 00:00:00 2001 From: spongedc Date: Mon, 14 Dec 2020 22:37:25 +0800 Subject: [PATCH 019/284] Support SHOW CREATE VIEW Syntax --- src/Common/ErrorCodes.cpp | 1 + src/Databases/IDatabase.h | 1 + src/Interpreters/InterpreterFactory.cpp | 4 +++ .../InterpreterShowCreateQuery.cpp | 9 ++++++- src/Parsers/ASTCreateQuery.h | 2 ++ src/Parsers/ParserTablePropertiesQuery.cpp | 20 +++++++++----- src/Parsers/ParserTablePropertiesQuery.h | 2 +- src/Parsers/TablePropertiesQueriesASTs.h | 9 +++++++ .../01602_show_create_view.reference | 3 +++ .../0_stateless/01602_show_create_view.sql | 26 +++++++++++++++++++ 10 files changed, 69 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/01602_show_create_view.reference create mode 100644 tests/queries/0_stateless/01602_show_create_view.sql diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 1e381808d16..328d0a364ff 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -529,6 +529,7 @@ M(560, ZSTD_ENCODER_FAILED) \ M(561, ZSTD_DECODER_FAILED) \ M(562, TLD_LIST_NOT_FOUND) \ + M(563, NOT_VIEW) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index fadec5fe7a9..fb2a9652d4b 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -30,6 +30,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; extern const int CANNOT_GET_CREATE_TABLE_QUERY; extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY; + extern const int NOT_VIEW; } class IDatabaseTablesIterator diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 30992905f5d..156c3c1ba51 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -160,6 +160,10 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, Context & { return std::make_unique(query, context); } + else if (query->as()) + { + return std::make_unique(query, context); + } else if (query->as()) { return std::make_unique(query, context); diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp index 907523ce94b..bcf6cc09473 100644 --- a/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -43,12 +43,19 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() { ASTPtr create_query; ASTQueryWithTableAndOutput * show_query; - if ((show_query = query_ptr->as())) + if ((show_query = query_ptr->as()) || + (show_query = query_ptr->as())) { auto resolve_table_type = show_query->temporary ? Context::ResolveExternal : Context::ResolveOrdinary; auto table_id = context.resolveStorageID(*show_query, resolve_table_type); context.checkAccess(AccessType::SHOW_COLUMNS, table_id); create_query = DatabaseCatalog::instance().getDatabase(table_id.database_name)->getCreateTableQuery(table_id.table_name, context); + if (query_ptr->as()) + { + auto & ast_create_query = create_query->as(); + if (!ast_create_query.isView()) + throw Exception("'" + ast_create_query.database + "." + ast_create_query.table +"' is not VIEW", ErrorCodes::NOT_VIEW); + } } else if ((show_query = query_ptr->as())) { diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index 7b2deb99698..67ad9a89392 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -91,6 +91,8 @@ public: return removeOnCluster(clone(), new_database); } + bool isView() { return is_view || is_materialized_view || is_live_view; } + protected: void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; diff --git a/src/Parsers/ParserTablePropertiesQuery.cpp b/src/Parsers/ParserTablePropertiesQuery.cpp index 2ee85a3330d..fcb7c8ef7a4 100644 --- a/src/Parsers/ParserTablePropertiesQuery.cpp +++ b/src/Parsers/ParserTablePropertiesQuery.cpp @@ -21,6 +21,7 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & ParserKeyword s_create("CREATE"); ParserKeyword s_database("DATABASE"); ParserKeyword s_table("TABLE"); + ParserKeyword s_view("VIEW"); ParserKeyword s_dictionary("DICTIONARY"); ParserToken s_dot(TokenType::Dot); ParserIdentifier name_p; @@ -30,6 +31,7 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & std::shared_ptr query; bool parse_only_database_name = false; + bool parse_show_create_view = false; bool temporary = false; if (s_exists.ignore(pos, expected)) @@ -56,6 +58,11 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & } else if (s_dictionary.checkWithoutMoving(pos, expected)) query = std::make_shared(); + else if (s_view.ignore(pos, expected)) + { + query = std::make_shared(); + parse_show_create_view = true; + } else query = std::make_shared(); } @@ -71,15 +78,16 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & } else { - if (temporary || s_temporary.ignore(pos, expected)) - query->temporary = true; - - if (!s_table.ignore(pos, expected)) - s_dictionary.ignore(pos, expected); + if (!parse_show_create_view) + { + if (temporary || s_temporary.ignore(pos, expected)) + query->temporary = true; + if (!s_table.ignore(pos, expected)) + s_dictionary.ignore(pos, expected); + } if (!name_p.parse(pos, table, expected)) return false; - if (s_dot.ignore(pos, expected)) { database = table; diff --git a/src/Parsers/ParserTablePropertiesQuery.h b/src/Parsers/ParserTablePropertiesQuery.h index 5de4c45db88..8d2c26d34ab 100644 --- a/src/Parsers/ParserTablePropertiesQuery.h +++ b/src/Parsers/ParserTablePropertiesQuery.h @@ -7,7 +7,7 @@ namespace DB { -/** Query (EXISTS | SHOW CREATE) [TABLE|DICTIONARY] [db.]name [FORMAT format] +/** Query (EXISTS | SHOW CREATE) [DATABASE|TABLE|DICTIONARY] [db.]name [FORMAT format] */ class ParserTablePropertiesQuery : public IParserBase { diff --git a/src/Parsers/TablePropertiesQueriesASTs.h b/src/Parsers/TablePropertiesQueriesASTs.h index 6a8e3b2ce83..cbb9b2a382c 100644 --- a/src/Parsers/TablePropertiesQueriesASTs.h +++ b/src/Parsers/TablePropertiesQueriesASTs.h @@ -29,6 +29,14 @@ struct ASTShowCreateTableQueryIDAndQueryNames static constexpr auto QueryTemporary = "SHOW CREATE TEMPORARY TABLE"; }; +struct ASTShowCreateViewQueryIDAndQueryNames +{ + static constexpr auto ID = "ShowCreateViewQuery"; + static constexpr auto Query = "SHOW CREATE VIEW"; + /// No temporary view are supported, just for parsing + static constexpr auto QueryTemporary = ""; +}; + struct ASTShowCreateDatabaseQueryIDAndQueryNames { static constexpr auto ID = "ShowCreateDatabaseQuery"; @@ -54,6 +62,7 @@ struct ASTDescribeQueryExistsQueryIDAndQueryNames using ASTExistsTableQuery = ASTQueryWithTableAndOutputImpl; using ASTExistsDictionaryQuery = ASTQueryWithTableAndOutputImpl; using ASTShowCreateTableQuery = ASTQueryWithTableAndOutputImpl; +using ASTShowCreateViewQuery = ASTQueryWithTableAndOutputImpl; using ASTShowCreateDictionaryQuery = ASTQueryWithTableAndOutputImpl; class ASTShowCreateDatabaseQuery : public ASTQueryWithTableAndOutputImpl diff --git a/tests/queries/0_stateless/01602_show_create_view.reference b/tests/queries/0_stateless/01602_show_create_view.reference new file mode 100644 index 00000000000..2c2ba13bef3 --- /dev/null +++ b/tests/queries/0_stateless/01602_show_create_view.reference @@ -0,0 +1,3 @@ +CREATE VIEW test_1602.v\n(\n `EventDate` DateTime,\n `CounterID` UInt32,\n `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl +CREATE MATERIALIZED VIEW test_1602.vv\n(\n `EventDate` DateTime,\n `CounterID` UInt32,\n `UserID` UInt32\n)\nENGINE = MergeTree\nPARTITION BY toYYYYMM(EventDate)\nORDER BY (CounterID, EventDate, intHash32(UserID))\nSETTINGS index_granularity = 8192 AS\nSELECT *\nFROM test_1602.tbl +CREATE LIVE VIEW test_1602.vvv\n(\n `EventDate` DateTime,\n `CounterID` UInt32,\n `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl diff --git a/tests/queries/0_stateless/01602_show_create_view.sql b/tests/queries/0_stateless/01602_show_create_view.sql new file mode 100644 index 00000000000..0a4ecd4ceec --- /dev/null +++ b/tests/queries/0_stateless/01602_show_create_view.sql @@ -0,0 +1,26 @@ +DROP DATABASE IF EXISTS test_1602; + +CREATE DATABASE test_1602; + +CREATE TABLE test_1602.tbl (`EventDate` DateTime, `CounterID` UInt32, `UserID` UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SETTINGS index_granularity = 8192; + +CREATE VIEW test_1602.v AS SELECT * FROM test_1602.tbl; + +CREATE MATERIALIZED VIEW test_1602.vv (`EventDate` DateTime, `CounterID` UInt32, `UserID` UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SETTINGS index_granularity = 8192 AS SELECT * FROM test_1602.tbl; + + +SET allow_experimental_live_view=1; + +CREATE LIVE VIEW test_1602.vvv AS SELECT * FROM test_1602.tbl; + +SHOW CREATE VIEW test_1602.v; + +SHOW CREATE VIEW test_1602.vv; + +SHOW CREATE VIEW test_1602.vvv; + +SHOW CREATE VIEW test_1602.not_exist_view; -- { serverError 390 } + +SHOW CREATE VIEW test_1602.tbl; -- { serverError 563 } + +DROP DATABASE IF EXISTS test_1602; From 90cc37ae561f90085601f3bf437a543d85f155e3 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Mon, 14 Dec 2020 23:23:29 +0300 Subject: [PATCH 020/284] Document execute_merges_on_single_replica_time_threshold setting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Задокументировал настройку execute_merges_on_single_replica_time_threshold. --- .../mergetree-family/replication.md | 2 ++ docs/en/operations/settings/settings.md | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 625869a3cb8..4758ff321e8 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -113,6 +113,8 @@ You can have any number of replicas of the same data. Yandex.Metrica uses double The system monitors data synchronicity on replicas and is able to recover after a failure. Failover is automatic (for small differences in data) or semi-automatic (when data differs too much, which may indicate a configuration error). +You can use a special logic for performing merges (for more information, see the documentation for [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) setting). + ## Creating Replicated Tables {#creating-replicated-tables} The `Replicated` prefix is added to the table engine name. For example:`ReplicatedMergeTree`. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index c5e44d4c464..3b2c9a67999 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2410,7 +2410,6 @@ Result: {"number":"1"} {"number":"2"} ``` - ======= ## allow_nullable_key {#allow-nullable-key} @@ -2423,4 +2422,19 @@ Possible values: Default value: `0`. +## execute_merges_on_single_replica_time_threshold {#execute-merges-on-single-replica-time-threshold} + +Enables special logic to perform merges on replicas. + +Possible values: + +- Positive integer (in seconds). +- 0 — Special merge logic is not executed. + +Default value: `0`. + +**Special logic to perform merges** + +Selects one replica to perform the merge on. Sets the time threshold from the start of the merge. Other replicas wait for the merge to finish, then download the result. If the time threshold passes and the selected replica does not perform merges, then the merge is performed manually. + [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) From d9a1b05910ebd73cabd2711cc43aabdca71483cf Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Mon, 14 Dec 2020 23:37:03 +0300 Subject: [PATCH 021/284] Update settings.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Убрал лишние символы в заголовке настройки allow_nullable_key. --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 3b2c9a67999..c60cd2c2c52 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2410,7 +2410,7 @@ Result: {"number":"1"} {"number":"2"} ``` -======= + ## allow_nullable_key {#allow-nullable-key} Allows using of the [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable)-typed values in a sorting and a primary key for [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engines-mergetree) tables. From e0addac6fc3409ff350f806814a6a9d9fe9118d1 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 18 May 2020 13:00:22 +0300 Subject: [PATCH 022/284] save changes --- .../ParallelParsingBlockInputStream.h | 1 - src/Dictionaries/XDBCDictionarySource.cpp | 5 +- src/Formats/FormatFactory.cpp | 55 ++-- src/Formats/FormatFactory.h | 14 +- src/Interpreters/Context.cpp | 3 +- src/Processors/Executors/PipelineExecutor.cpp | 3 + .../Impl/ParallelParsingBlockInputFormat.cpp | 202 ++++++++++++++ .../Impl/ParallelParsingBlockInputFormat.h | 259 ++++++++++++++++++ src/Storages/MergeTree/MergeTreeData.cpp | 4 +- src/Storages/StorageFile.cpp | 10 +- src/Storages/StorageHDFS.cpp | 4 +- src/Storages/StorageS3.cpp | 5 +- src/Storages/StorageURL.cpp | 9 +- 13 files changed, 536 insertions(+), 38 deletions(-) create mode 100644 src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.cpp create mode 100644 src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.h diff --git a/src/DataStreams/ParallelParsingBlockInputStream.h b/src/DataStreams/ParallelParsingBlockInputStream.h index c882acd9ddd..147c86a4cf7 100644 --- a/src/DataStreams/ParallelParsingBlockInputStream.h +++ b/src/DataStreams/ParallelParsingBlockInputStream.h @@ -158,7 +158,6 @@ private: // constructor, which is absent for atomics that are inside ProcessingUnit. std::deque processing_units; - void scheduleParserThreadForUnitWithNumber(size_t ticket_number); void finishAndWait(); diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index 89df4b606fe..aa27f4efc25 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -47,8 +48,8 @@ namespace : name(name_) { read_buf = std::make_unique(uri, Poco::Net::HTTPRequest::HTTP_POST, callback, timeouts); - reader - = FormatFactory::instance().getInput(IXDBCBridgeHelper::DEFAULT_FORMAT, *read_buf, sample_block, context, max_block_size); + auto format = FormatFactory::instance().getInput(IXDBCBridgeHelper::DEFAULT_FORMAT, *read_buf, sample_block, context, max_block_size); + reader = std::make_shared(format); } Block getHeader() const override { return reader->getHeader(); } diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index da63151613e..7c591af8537 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -6,15 +6,17 @@ #include #include #include +#include +#include #include #include #include #include #include -#include #include #include -#include +#include +#include #include #if !defined(ARCADIA_BUILD) @@ -131,7 +133,7 @@ FormatSettings getFormatSettings(const Context & context, const Settings & settings); -BlockInputStreamPtr FormatFactory::getInput( +InputFormatPtr FormatFactory::getInput( const String & name, ReadBuffer & buf, const Block & sample, @@ -140,19 +142,23 @@ BlockInputStreamPtr FormatFactory::getInput( const std::optional & _format_settings) const { if (name == "Native") - return std::make_shared(buf, sample, 0); + return std::make_shared(sample, buf); auto format_settings = _format_settings ? *_format_settings : getFormatSettings(context); if (!getCreators(name).input_processor_creator) { - const auto & input_getter = getCreators(name).input_creator; - if (!input_getter) - throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); - - return input_getter(buf, sample, max_block_size, {}, format_settings); +// const auto & input_getter = getCreators(name).input_creator; +// if (!input_getter) +// throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); +// +// const Settings & settings = context.getSettingsRef(); +// FormatSettings format_settings = getInputFormatSetting(settings, context); +// +// return input_getter(buf, sample, max_block_size, callback ? callback : ReadCallback(), format_settings); + throw; } const Settings & settings = context.getSettingsRef(); @@ -188,19 +194,32 @@ BlockInputStreamPtr FormatFactory::getInput( row_input_format_params.max_execution_time = settings.max_execution_time; row_input_format_params.timeout_overflow_mode = settings.timeout_overflow_mode; - auto input_creator_params = - ParallelParsingBlockInputStream::InputCreatorParams{sample, - row_input_format_params, format_settings}; - ParallelParsingBlockInputStream::Params params{buf, input_getter, - input_creator_params, file_segmentation_engine, + + auto parser_creator = std::bind( + input_getter.target(), + std::placeholders::_1, sample, row_input_format_params, format_settings); + +// auto anime = parser_creator(buf) + auto boruto = input_getter(buf, sample, row_input_format_params, format_settings); +// auto naruto = input_getter.target()(buf, sample, row_input_format_params, format_settings); + + auto naruto = + [sample, row_input_format_params, format_settings, input_getter] + (ReadBuffer & input) + {return input_getter(input, sample, row_input_format_params, format_settings);}; + + auto aaa = naruto(buf); + + ParallelParsingBlockInputFormat::Params params{buf, sample, + naruto, file_segmentation_engine, settings.max_threads, settings.min_chunk_bytes_for_parallel_parsing}; - return std::make_shared(params); + return std::make_shared(params); } - auto format = getInputFormat(name, buf, sample, context, max_block_size, - format_settings); - return std::make_shared(std::move(format)); + + auto format = getInputFormat(name, buf, sample, context, max_block_size, std::move(callback)); + return format; } diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index 0fe6f19f0b7..d191597fe25 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -79,11 +79,13 @@ private: WriteCallback callback, const FormatSettings & settings)>; - using InputProcessorCreator = std::function; + using InputProcessorCreatorFunc = InputFormatPtr( + ReadBuffer & buf, + const Block & header, + const RowInputFormatParams & params, + const FormatSettings & settings); + + using InputProcessorCreator = std::function; using OutputProcessorCreator = std::function #include #include +#include #include #include #include @@ -2072,7 +2073,7 @@ void Context::checkPartitionCanBeDropped(const String & database, const String & BlockInputStreamPtr Context::getInputFormat(const String & name, ReadBuffer & buf, const Block & sample, UInt64 max_block_size) const { - return FormatFactory::instance().getInput(name, buf, sample, *this, max_block_size); + return std::make_shared(FormatFactory::instance().getInput(name, buf, sample, *this, max_block_size)); } BlockOutputStreamPtr Context::getOutputFormat(const String & name, WriteBuffer & buf, const Block & sample) const diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 517e07a3ba4..e7911f9b367 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -464,6 +464,9 @@ void PipelineExecutor::finalizeExecution() if (!all_processors_finished) throw Exception("Pipeline stuck. Current state:\n" + dumpPipeline(), ErrorCodes::LOGICAL_ERROR); + + WriteBufferFromOStream out(std::cout); + printPipeline(processors, out); } void PipelineExecutor::wakeUpExecutor(size_t thread_num) diff --git a/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.cpp new file mode 100644 index 00000000000..6f63504d227 --- /dev/null +++ b/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.cpp @@ -0,0 +1,202 @@ +#include + +namespace DB +{ + +void ParallelParsingBlockInputFormat::segmentatorThreadFunction() +{ + setThreadName("Segmentator"); + try + { + while (!parsing_finished) + { + const auto current_unit_number = segmentator_ticket_number % processing_units.size(); + auto & unit = processing_units[current_unit_number]; + + { + std::unique_lock lock(mutex); + segmentator_condvar.wait(lock, + [&]{ return unit.status == READY_TO_INSERT || parsing_finished; }); + } + + if (parsing_finished) + { + break; + } + + assert(unit.status == READY_TO_INSERT); + + // Segmentating the original input. + unit.segment.resize(0); + + const bool have_more_data = file_segmentation_engine(in, unit.segment, min_chunk_bytes); + + unit.is_last = !have_more_data; + unit.status = READY_TO_PARSE; + scheduleParserThreadForUnitWithNumber(segmentator_ticket_number); + ++segmentator_ticket_number; + + if (!have_more_data) + { + break; + } + } + } + catch (...) + { + onBackgroundException(); + } +} + +void ParallelParsingBlockInputFormat::parserThreadFunction(size_t current_ticket_number) +{ + try + { + setThreadName("ChunkParser"); + + const auto current_unit_number = current_ticket_number % processing_units.size(); + auto & unit = processing_units[current_unit_number]; + + /* + * This is kind of suspicious -- the input_process_creator contract with + * respect to multithreaded use is not clear, but we hope that it is + * just a 'normal' factory class that doesn't have any state, and so we + * can use it from multiple threads simultaneously. + */ + ReadBuffer read_buffer(unit.segment.data(), unit.segment.size(), 0); + InputFormatPtr input_format = internal_parser_creator(read_buffer); + InternalParser parser(input_format); + + unit.chunk_ext.chunk.clear(); + unit.chunk_ext.block_missing_values.clear(); + + // We don't know how many blocks will be. So we have to read them all + // until an empty block occured. + Chunk chunk; + while (!parsing_finished && (chunk = parser.getChunk()) != Chunk()) + { + unit.chunk_ext.chunk.emplace_back(std::move(chunk)); + unit.chunk_ext.block_missing_values.emplace_back(parser.getMissingValues()); + } + + // We suppose we will get at least some blocks for a non-empty buffer, + // except at the end of file. Also see a matching assert in readImpl(). + assert(unit.is_last || !unit.chunk_ext.chunk.empty()); + + std::lock_guard lock(mutex); + unit.status = READY_TO_READ; + reader_condvar.notify_all(); + } + catch (...) + { + onBackgroundException(); + } +} + + +void ParallelParsingBlockInputFormat::onBackgroundException() +{ + tryLogCurrentException(__PRETTY_FUNCTION__); + + std::unique_lock lock(mutex); + if (!background_exception) + { + background_exception = std::current_exception(); + } + parsing_finished = true; + reader_condvar.notify_all(); + segmentator_condvar.notify_all(); +} + +Chunk ParallelParsingBlockInputFormat::generate() +{ + if (isCancelled() || parsing_finished) + { + /** + * Check for background exception and rethrow it before we return. + */ + std::unique_lock lock(mutex); + if (background_exception) + { + lock.unlock(); + onCancel(); + std::rethrow_exception(background_exception); + } + + return {}; + } + + const auto current_unit_number = reader_ticket_number % processing_units.size(); + auto & unit = processing_units[current_unit_number]; + + if (!next_block_in_current_unit.has_value()) + { + // We have read out all the Blocks from the previous Processing Unit, + // wait for the current one to become ready. + std::unique_lock lock(mutex); + reader_condvar.wait(lock, [&](){ return unit.status == READY_TO_READ || parsing_finished; }); + + if (parsing_finished) + { + /** + * Check for background exception and rethrow it before we return. + */ + if (background_exception) + { + lock.unlock(); + cancel(); + std::rethrow_exception(background_exception); + } + + return {}; + } + + assert(unit.status == READY_TO_READ); + next_block_in_current_unit = 0; + } + + if (unit.chunk_ext.chunk.empty()) + { + /* + * Can we get zero blocks for an entire segment, when the format parser + * skips it entire content and does not create any blocks? Probably not, + * but if we ever do, we should add a loop around the above if, to skip + * these. Also see a matching assert in the parser thread. + */ + assert(unit.is_last); + parsing_finished = true; + return {}; + } + + assert(next_block_in_current_unit.value() < unit.chunk_ext.chunk.size()); + + Chunk res = std::move(unit.chunk_ext.chunk.at(*next_block_in_current_unit)); + last_block_missing_values = std::move(unit.chunk_ext.block_missing_values[*next_block_in_current_unit]); + + next_block_in_current_unit.value() += 1; + + if (*next_block_in_current_unit == unit.chunk_ext.chunk.size()) + { + // parsing_finished reading this Processing Unit, move to the next one. + next_block_in_current_unit.reset(); + ++reader_ticket_number; + + if (unit.is_last) + { + // It it was the last unit, we're parsing_finished. + parsing_finished = true; + } + else + { + // Pass the unit back to the segmentator. + std::unique_lock lock(mutex); + unit.status = READY_TO_INSERT; + segmentator_condvar.notify_all(); + } + } + + return res; +} + + +} diff --git a/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.h new file mode 100644 index 00000000000..3900880ae1b --- /dev/null +++ b/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.h @@ -0,0 +1,259 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +class Context; + +/** + * ORDER-PRESERVING parallel parsing of data formats. + * It splits original data into chunks. Then each chunk is parsed by different thread. + * The number of chunks equals to the number or parser threads. + * The size of chunk is equal to min_chunk_bytes_for_parallel_parsing setting. + * + * This stream has three kinds of threads: one segmentator, multiple parsers, + * and one reader thread -- that is, the one from which readImpl() is called. + * They operate one after another on parts of data called "processing units". + * One unit consists of buffer with raw data from file, filled by segmentator + * thread. This raw data is then parsed by a parser thread to form a number of + * Blocks. These Blocks are returned to the parent stream from readImpl(). + * After being read out, a processing unit is reused, to save on allocating + * memory for the raw buffer. The processing units are organized into a circular + * array to facilitate reuse and to apply backpressure on the segmentator thread + * -- after it runs out of processing units, it has to wait for the reader to + * read out the previous blocks. + * The outline of what the threads do is as follows: + * segmentator thread: + * 1) wait for the next processing unit to become empty + * 2) fill it with a part of input file + * 3) start a parser thread + * 4) repeat until eof + * parser thread: + * 1) parse the given raw buffer without any synchronization + * 2) signal that the given unit is ready to read + * 3) finish + * readImpl(): + * 1) wait for the next processing unit to become ready to read + * 2) take the blocks from the processing unit to return them to the caller + * 3) signal that the processing unit is empty + * 4) repeat until it encounters unit that is marked as "past_the_end" + * All threads must also check for cancel/eof/exception flags. + */ +class ParallelParsingBlockInputFormat : public IInputFormat +{ +public: + /* Used to recreate parser on every new data piece. */ + using InternalParserCreator = std::function; + + struct Params + { + ReadBuffer & in; + Block header; + InternalParserCreator internal_parser_creator; + FormatFactory::FileSegmentationEngine file_segmentation_engine; + size_t max_threads; + size_t min_chunk_bytes; + }; + + explicit ParallelParsingBlockInputFormat(Params params) + : IInputFormat(std::move(params.header), params.in) + , internal_parser_creator(params.internal_parser_creator) + , file_segmentation_engine(params.file_segmentation_engine) + , min_chunk_bytes(params.min_chunk_bytes) + // Subtract one thread that we use for segmentation and one for + // reading. After that, must have at least two threads left for + // parsing. See the assertion below. + , pool(params.max_threads) + { + // One unit for each thread, including segmentator and reader, plus a + // couple more units so that the segmentation thread doesn't spuriously + // bump into reader thread on wraparound. + processing_units.resize(params.max_threads + 2); + + segmentator_thread = ThreadFromGlobalPool([this] { segmentatorThreadFunction(); }); + } + + void resetParser() override final + { + throw Exception("resetParser() is not allowed for " + getName(), ErrorCodes::LOGICAL_ERROR); + } + + const BlockMissingValues & getMissingValues() const override final + { + return last_block_missing_values; + } + + String getName() const override final { return "ParallelParsingBlockInputFormat"; } + +protected: + + Chunk generate() override final; + + void onCancel() override final + { + /* + * The format parsers themselves are not being cancelled here, so we'll + * have to wait until they process the current block. Given that the + * chunk size is on the order of megabytes, this should't be too long. + * We can't call IInputFormat->cancel here, because the parser object is + * local to the parser thread, and we don't want to introduce any + * synchronization between parser threads and the other threads to get + * better performance. An ideal solution would be to add a callback to + * IInputFormat that checks whether it was cancelled. + */ + + finishAndWait(); + } + +private: + + class InternalParser + { + public: + explicit InternalParser(const InputFormatPtr & input_format_) + : input_format(input_format_) + , port(input_format->getPort().getHeader(), input_format.get()) + { + connect(input_format->getPort(), port); + port.setNeeded(); + } + + Chunk getChunk() + { + while (true) + { + IProcessor::Status status = input_format->prepare(); + switch (status) + { + case IProcessor::Status::Ready: + input_format->work(); + break; + + case IProcessor::Status::Finished: + return {}; + + case IProcessor::Status::PortFull: + return port.pull(); + + case IProcessor::Status::NeedData: break; + case IProcessor::Status::Async: break; + case IProcessor::Status::Wait: break; + case IProcessor::Status::ExpandPipeline: + throw Exception("One of the parsers returned status " + IProcessor::statusToName(status) + + " during parallel parsing", ErrorCodes::LOGICAL_ERROR); + } + } + } + + const BlockMissingValues & getMissingValues() const { return input_format->getMissingValues(); } + + private: + const InputFormatPtr & input_format; + InputPort port; + }; + + const InternalParserCreator internal_parser_creator; + // Function to segment the file. Then "parsers" will parse that segments. + FormatFactory::FileSegmentationEngine file_segmentation_engine; + const size_t min_chunk_bytes; + + BlockMissingValues last_block_missing_values; + + //Non-atomic because it is used in one thread. + std::optional next_block_in_current_unit; + size_t segmentator_ticket_number{0}; + size_t reader_ticket_number{0}; + + std::mutex mutex; + std::condition_variable reader_condvar; + std::condition_variable segmentator_condvar; + + std::atomic parsing_finished; + + // There are multiple "parsers", that's why we use thread pool. + ThreadPool pool; + // Reading and segmentating the file + ThreadFromGlobalPool segmentator_thread; + + enum ProcessingUnitStatus + { + READY_TO_INSERT, + READY_TO_PARSE, + READY_TO_READ + }; + + struct ChunkExt + { + std::vector chunk; + std::vector block_missing_values; + }; + + struct ProcessingUnit + { + explicit ProcessingUnit() + : status(ProcessingUnitStatus::READY_TO_INSERT) + { + } + + ChunkExt chunk_ext; + Memory<> segment; + std::atomic status; + bool is_last{false}; + }; + + std::exception_ptr background_exception = nullptr; + + // We use deque instead of vector, because it does not require a move + // constructor, which is absent for atomics that are inside ProcessingUnit. + std::deque processing_units; + + + void scheduleParserThreadForUnitWithNumber(size_t ticket_number) + { + pool.scheduleOrThrowOnError([this, ticket_number] { parserThreadFunction(ticket_number); }); + } + + void finishAndWait() + { + parsing_finished = true; + + { + std::unique_lock lock(mutex); + segmentator_condvar.notify_all(); + reader_condvar.notify_all(); + } + + if (segmentator_thread.joinable()) + segmentator_thread.join(); + + try + { + pool.wait(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + + void segmentatorThreadFunction(); + void parserThreadFunction(size_t current_ticket_number); + + // Save/log a background exception, set termination flag, wake up all + // threads. This function is used by segmentator and parsed threads. + // readImpl() is called from the main thread, so the exception handling + // is different. + void onBackgroundException(); +}; + +} diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c3a599665bb..4acfe34d47f 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -2903,7 +2904,8 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, const Context ReadBufferFromMemory right_paren_buf(")", 1); ConcatReadBuffer buf({&left_paren_buf, &fields_buf, &right_paren_buf}); - auto input_stream = FormatFactory::instance().getInput("Values", buf, metadata_snapshot->getPartitionKey().sample_block, context, context.getSettingsRef().max_block_size); + auto format = FormatFactory::instance().getInput("Values", buf, partition_key_sample, context, context.getSettingsRef().max_block_size); + auto input_stream = std::make_shared(format); auto block = input_stream->read(); if (!block || !block.rows()) diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 3ad9384ecfb..477c557bbd0 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -25,7 +25,6 @@ #include #include -#include #include #include @@ -34,6 +33,7 @@ #include #include #include +#include #include namespace fs = std::filesystem; @@ -291,6 +291,7 @@ public: Chunk generate() override { + std::cout << StackTrace().toString() << std::endl; while (!finished_generate) { /// Open file lazily on first read. This is needed to avoid too many open files from different streams. @@ -332,8 +333,11 @@ public: *read_buf, metadata_snapshot->getSampleBlock(), context, max_block_size, storage->format_settings); - if (columns_description.hasDefaults()) - reader = std::make_shared(reader, columns_description, context); + + reader = std::make_shared(format); + + if (!column_defaults.empty()) + reader = std::make_shared(reader, column_defaults, context); reader->readPrefix(); } diff --git a/src/Storages/StorageHDFS.cpp b/src/Storages/StorageHDFS.cpp index 0a4729b1c06..2946d45eb15 100644 --- a/src/Storages/StorageHDFS.cpp +++ b/src/Storages/StorageHDFS.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -122,7 +123,8 @@ public: auto compression = chooseCompressionMethod(path, compression_method); auto read_buf = wrapReadBufferWithCompressionMethod(std::make_unique(current_path), compression); - auto input_stream = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size); + auto input_format = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size); + auto input_stream = std::make_shared(input_format); reader = std::make_shared>(input_stream, std::move(read_buf)); reader->readPrefix(); diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 0465cfdffba..efd04e53557 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -33,6 +34,7 @@ #include #include +#include #include @@ -82,7 +84,8 @@ namespace , file_path(bucket + "/" + key) { read_buf = wrapReadBufferWithCompressionMethod(std::make_unique(client, bucket, key), compression_method); - reader = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size); + auto input_format = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size); + reader = std::make_shared(input_format); if (columns.hasDefaults()) reader = std::make_shared(reader, columns, context); diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 00903abee59..9533892d07a 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -105,10 +106,10 @@ namespace context.getRemoteHostFilter()), compression_method); - reader = FormatFactory::instance().getInput(format, *read_buf, - sample_block, context, max_block_size, format_settings); - reader = std::make_shared(reader, - columns, context); + + auto input_format = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size); + reader = std::make_shared(input_format); + reader = std::make_shared(reader, column_defaults, context); } String getName() const override From f40f3ced2a6ad8ad6bf29b9fa73269a6f302b939 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 10 Jun 2020 15:02:34 +0300 Subject: [PATCH 023/284] fix JSONEachRowArray --- src/Formats/FormatFactory.cpp | 25 +++------ src/Formats/JSONEachRowUtils.cpp | 9 +++ src/Processors/Executors/PipelineExecutor.cpp | 3 - .../Formats/Impl/IReadBufferPrepareAndEndUp.h | 56 +++++++++++++++++++ .../Impl/JSONEachRowRowInputFormat.cpp | 23 +------- .../Formats/Impl/JSONEachRowRowInputFormat.h | 6 +- .../Impl/ParallelParsingBlockInputFormat.cpp | 20 +++++++ .../Impl/ParallelParsingBlockInputFormat.h | 16 +++++- src/Storages/MergeTree/MergeTreeData.cpp | 6 +- src/Storages/StorageFile.cpp | 1 - 10 files changed, 117 insertions(+), 48 deletions(-) create mode 100644 src/Processors/Formats/Impl/IReadBufferPrepareAndEndUp.h diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 7c591af8537..9fba199d0c1 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -150,6 +150,7 @@ InputFormatPtr FormatFactory::getInput( if (!getCreators(name).input_processor_creator) { + // const auto & input_getter = getCreators(name).input_creator; // if (!input_getter) // throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); @@ -194,26 +195,14 @@ InputFormatPtr FormatFactory::getInput( row_input_format_params.max_execution_time = settings.max_execution_time; row_input_format_params.timeout_overflow_mode = settings.timeout_overflow_mode; + /// Const reference is copied to lambda. + auto parser_creator = [input_getter, sample, row_input_format_params, format_settings] + (ReadBuffer & input) -> InputFormatPtr + { return input_getter(input, sample, row_input_format_params, format_settings); }; - auto parser_creator = std::bind( - input_getter.target(), - std::placeholders::_1, sample, row_input_format_params, format_settings); -// auto anime = parser_creator(buf) - auto boruto = input_getter(buf, sample, row_input_format_params, format_settings); -// auto naruto = input_getter.target()(buf, sample, row_input_format_params, format_settings); - - auto naruto = - [sample, row_input_format_params, format_settings, input_getter] - (ReadBuffer & input) - {return input_getter(input, sample, row_input_format_params, format_settings);}; - - auto aaa = naruto(buf); - - ParallelParsingBlockInputFormat::Params params{buf, sample, - naruto, file_segmentation_engine, - settings.max_threads, - settings.min_chunk_bytes_for_parallel_parsing}; + ParallelParsingBlockInputFormat::Params params{ + buf, sample, parser_creator, file_segmentation_engine, name, settings.max_threads, settings.min_chunk_bytes_for_parallel_parsing}; return std::make_shared(params); } diff --git a/src/Formats/JSONEachRowUtils.cpp b/src/Formats/JSONEachRowUtils.cpp index a1d9b4a5fff..44472a36e16 100644 --- a/src/Formats/JSONEachRowUtils.cpp +++ b/src/Formats/JSONEachRowUtils.cpp @@ -9,6 +9,14 @@ bool fileSegmentationEngineJSONEachRowImpl(ReadBuffer & in, DB::Memory<> & memor skipWhitespaceIfAny(in); char * pos = in.position(); + + /// In case that independent JSONs are splitted by comma we skip that comma. + if (pos && *pos == ',') + { + ++in.position(); + ++pos; + } + size_t balance = 0; bool quotes = false; @@ -61,6 +69,7 @@ bool fileSegmentationEngineJSONEachRowImpl(ReadBuffer & in, DB::Memory<> & memor } saveUpToPosition(in, memory, pos); + assert(*memory.data() == '{'); return loadAtPosition(in, memory, pos); } diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index e7911f9b367..517e07a3ba4 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -464,9 +464,6 @@ void PipelineExecutor::finalizeExecution() if (!all_processors_finished) throw Exception("Pipeline stuck. Current state:\n" + dumpPipeline(), ErrorCodes::LOGICAL_ERROR); - - WriteBufferFromOStream out(std::cout); - printPipeline(processors, out); } void PipelineExecutor::wakeUpExecutor(size_t thread_num) diff --git a/src/Processors/Formats/Impl/IReadBufferPrepareAndEndUp.h b/src/Processors/Formats/Impl/IReadBufferPrepareAndEndUp.h new file mode 100644 index 00000000000..33d8bbf9cd6 --- /dev/null +++ b/src/Processors/Formats/Impl/IReadBufferPrepareAndEndUp.h @@ -0,0 +1,56 @@ +#pragma once + +#include + +namespace DB +{ + +class IReadBufferPrepareAndEndUp +{ +public: + virtual ~IReadBufferPrepareAndEndUp() {} + + virtual void prepareReadBuffer(ReadBuffer & buffer) = 0; + + virtual void endUpReadBuffer(ReadBuffer & buffer) = 0; +}; + +using IReadBufferPrepareAndEndUpPtr = std::shared_ptr; + +class JSONEachRowPrepareAndEndUp : public IReadBufferPrepareAndEndUp +{ +public: + void prepareReadBuffer(ReadBuffer & buffer) override + { + /// In this format, BOM at beginning of stream cannot be confused with value, so it is safe to skip it. + skipBOMIfExists(in); + + skipWhitespaceIfAny(in); + if (!in.eof() && *in.position() == '[') + { + ++in.position(); + data_in_square_brackets = true; + } + } + + void endUpReadBuffer(ReadBuffer & buffer) override + { + skipWhitespaceIfAny(buffer); + if (data_in_square_brackets) + { + assertChar(']', buffer); + skipWhitespaceIfAny(buffer); + } + if (!buffer.eof() && *buffer.position() == ';') + { + ++buffer.position(); + skipWhitespaceIfAny(buffer); + } + assertEOF(buffer); + } + +private: + bool data_in_square_brackets{false}; +}; + +} diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp index 234839b41f5..dfb5eb79b4b 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp @@ -303,31 +303,12 @@ void JSONEachRowRowInputFormat::resetParser() void JSONEachRowRowInputFormat::readPrefix() { - /// In this format, BOM at beginning of stream cannot be confused with value, so it is safe to skip it. - skipBOMIfExists(in); - - skipWhitespaceIfAny(in); - if (!in.eof() && *in.position() == '[') - { - ++in.position(); - data_in_square_brackets = true; - } + prepare_and_end_up.prepareReadBuffer(in); } void JSONEachRowRowInputFormat::readSuffix() { - skipWhitespaceIfAny(in); - if (data_in_square_brackets) - { - assertChar(']', in); - skipWhitespaceIfAny(in); - } - if (!in.eof() && *in.position() == ';') - { - ++in.position(); - skipWhitespaceIfAny(in); - } - assertEOF(in); + prepare_and_end_up.endUpReadBuffer(in); } diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h index 29a6ce6ecb8..e121d77d7a6 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h @@ -4,7 +4,7 @@ #include #include #include - +#include "IReadBufferPrepareAndEndUp.h" namespace DB { @@ -81,7 +81,11 @@ private: bool allow_new_rows = true; + bool yield_strings; + + /// Used when readSuffix() or readPrefix() are called. + JSONEachRowPrepareAndEndUp prepare_and_end_up; }; } diff --git a/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.cpp index 6f63504d227..ad7a160ce11 100644 --- a/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB { @@ -64,6 +65,10 @@ void ParallelParsingBlockInputFormat::parserThreadFunction(size_t current_ticket * can use it from multiple threads simultaneously. */ ReadBuffer read_buffer(unit.segment.data(), unit.segment.size(), 0); + + if (current_ticket_number == 0) + prepareReadBuffer(read_buffer); + InputFormatPtr input_format = internal_parser_creator(read_buffer); InternalParser parser(input_format); @@ -83,6 +88,9 @@ void ParallelParsingBlockInputFormat::parserThreadFunction(size_t current_ticket // except at the end of file. Also see a matching assert in readImpl(). assert(unit.is_last || !unit.chunk_ext.chunk.empty()); + if (unit.is_last) + endUpReadBuffer(read_buffer); + std::lock_guard lock(mutex); unit.status = READY_TO_READ; reader_condvar.notify_all(); @@ -198,5 +206,17 @@ Chunk ParallelParsingBlockInputFormat::generate() return res; } +void ParallelParsingBlockInputFormat::prepareReadBuffer(ReadBuffer & buffer) +{ + if (prepare_and_end_up_ptr) + prepare_and_end_up_ptr->prepareReadBuffer(buffer); +} + + +void ParallelParsingBlockInputFormat::endUpReadBuffer(ReadBuffer & buffer) +{ + if (prepare_and_end_up_ptr) + prepare_and_end_up_ptr->endUpReadBuffer(buffer); +} } diff --git a/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.h index 3900880ae1b..1a0cd2c537a 100644 --- a/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.h @@ -10,6 +10,7 @@ #include #include #include +#include "IReadBufferPrepareAndEndUp.h" namespace DB { @@ -61,6 +62,7 @@ public: Block header; InternalParserCreator internal_parser_creator; FormatFactory::FileSegmentationEngine file_segmentation_engine; + String format_name; size_t max_threads; size_t min_chunk_bytes; }; @@ -69,6 +71,7 @@ public: : IInputFormat(std::move(params.header), params.in) , internal_parser_creator(params.internal_parser_creator) , file_segmentation_engine(params.file_segmentation_engine) + , format_name(params.format_name) , min_chunk_bytes(params.min_chunk_bytes) // Subtract one thread that we use for segmentation and one for // reading. After that, must have at least two threads left for @@ -80,6 +83,10 @@ public: // bump into reader thread on wraparound. processing_units.resize(params.max_threads + 2); + /// To skip '[' and ']'. + if (format_name == "JSONEachRow") + prepare_and_end_up_ptr = std::make_shared(); + segmentator_thread = ThreadFromGlobalPool([this] { segmentatorThreadFunction(); }); } @@ -165,6 +172,7 @@ private: const InternalParserCreator internal_parser_creator; // Function to segment the file. Then "parsers" will parse that segments. FormatFactory::FileSegmentationEngine file_segmentation_engine; + const String format_name; const size_t min_chunk_bytes; BlockMissingValues last_block_missing_values; @@ -178,7 +186,7 @@ private: std::condition_variable reader_condvar; std::condition_variable segmentator_condvar; - std::atomic parsing_finished; + std::atomic parsing_finished{false}; // There are multiple "parsers", that's why we use thread pool. ThreadPool pool; @@ -254,6 +262,12 @@ private: // readImpl() is called from the main thread, so the exception handling // is different. void onBackgroundException(); + + IReadBufferPrepareAndEndUpPtr prepare_and_end_up_ptr; + + void prepareReadBuffer(ReadBuffer & buffer); + + void endUpReadBuffer(ReadBuffer & buffer); }; } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 4acfe34d47f..9804abb2247 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -60,7 +60,7 @@ #include #include #include - +#include namespace ProfileEvents { @@ -2904,8 +2904,8 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, const Context ReadBufferFromMemory right_paren_buf(")", 1); ConcatReadBuffer buf({&left_paren_buf, &fields_buf, &right_paren_buf}); - auto format = FormatFactory::instance().getInput("Values", buf, partition_key_sample, context, context.getSettingsRef().max_block_size); - auto input_stream = std::make_shared(format); + auto input_format = FormatFactory::instance().getInput("Values", buf, getPartitionKey().sample_block, context, context.getSettingsRef().max_block_size); + auto input_stream = std::make_shared(input_format); auto block = input_stream->read(); if (!block || !block.rows()) diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 477c557bbd0..736089b666f 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -291,7 +291,6 @@ public: Chunk generate() override { - std::cout << StackTrace().toString() << std::endl; while (!finished_generate) { /// Open file lazily on first read. This is needed to avoid too many open files from different streams. From 0a508c7b8ab6f6685089d941a6594e76b0ef2763 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 11 Jun 2020 03:36:57 +0300 Subject: [PATCH 024/284] save --- src/Common/Arena.h | 56 +++---- src/Common/ThreadPool.h | 1 - src/Formats/FormatFactory.cpp | 6 +- src/IO/WriteBufferFromArena.h | 2 +- src/IO/WriteHelpers.h | 1 + .../Formats/Impl/CSVRowOutputFormat.cpp | 1 + .../Impl/JSONEachRowRowOutputFormat.cpp | 1 + .../Impl/ParallelFormattingOutputFormat.h | 147 ++++++++++++++++++ ...mat.cpp => ParallelParsingInputFormat.cpp} | 26 ++-- ...tFormat.h => ParallelParsingInputFormat.h} | 23 ++- 10 files changed, 213 insertions(+), 51 deletions(-) create mode 100644 src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h rename src/Processors/Formats/Impl/{ParallelParsingBlockInputFormat.cpp => ParallelParsingInputFormat.cpp} (88%) rename src/Processors/Formats/Impl/{ParallelParsingBlockInputFormat.h => ParallelParsingInputFormat.h} (94%) diff --git a/src/Common/Arena.h b/src/Common/Arena.h index 44a9b444ff2..2a05a257381 100644 --- a/src/Common/Arena.h +++ b/src/Common/Arena.h @@ -15,7 +15,7 @@ namespace ProfileEvents { - extern const Event ArenaAllocChunks; + extern const Event ArenaAllocMemoryChunks; extern const Event ArenaAllocBytes; } @@ -28,7 +28,7 @@ namespace DB * - put lot of strings inside pool, keep their addresses; * - addresses remain valid during lifetime of pool; * - at destruction of pool, all memory is freed; - * - memory is allocated and freed by large chunks; + * - memory is allocated and freed by large MemoryChunks; * - freeing parts of data is not possible (but look at ArenaWithFreeLists if you need); */ class Arena : private boost::noncopyable @@ -37,29 +37,31 @@ private: /// Padding allows to use 'memcpySmallAllowReadWriteOverflow15' instead of 'memcpy'. static constexpr size_t pad_right = 15; - /// Contiguous chunk of memory and pointer to free space inside it. Member of single-linked list. - struct alignas(16) Chunk : private Allocator /// empty base optimization + /// Contiguous MemoryChunk of memory and pointer to free space inside it. Member of single-linked list. + struct alignas(16) MemoryChunk : private Allocator /// empty base optimization { char * begin; char * pos; char * end; /// does not include padding. - Chunk * prev; + MemoryChunk * prev; + MemoryChunk * next; - Chunk(size_t size_, Chunk * prev_) + MemoryChunk(size_t size_, MemoryChunk * prev_) { - ProfileEvents::increment(ProfileEvents::ArenaAllocChunks); + ProfileEvents::increment(ProfileEvents::ArenaAllocMemoryChunks); ProfileEvents::increment(ProfileEvents::ArenaAllocBytes, size_); begin = reinterpret_cast(Allocator::alloc(size_)); pos = begin; end = begin + size_ - pad_right; prev = prev_; + prev->next = this; ASAN_POISON_MEMORY_REGION(begin, size_); } - ~Chunk() + ~MemoryChunk() { /// We must unpoison the memory before returning to the allocator, /// because the allocator might not have asan integration, and the @@ -80,8 +82,8 @@ private: size_t growth_factor; size_t linear_growth_threshold; - /// Last contiguous chunk of memory. - Chunk * head; + /// Last contiguous MemoryChunk of memory. + MemoryChunk * head; size_t size_in_bytes; static size_t roundUpToPageSize(size_t s) @@ -89,7 +91,7 @@ private: return (s + 4096 - 1) / 4096 * 4096; } - /// If chunks size is less than 'linear_growth_threshold', then use exponential growth, otherwise - linear growth + /// If MemoryChunks size is less than 'linear_growth_threshold', then use exponential growth, otherwise - linear growth /// (to not allocate too much excessive memory). size_t nextSize(size_t min_next_size) const { @@ -103,7 +105,7 @@ private: { // allocContinue() combined with linear growth results in quadratic // behavior: we append the data by small amounts, and when it - // doesn't fit, we create a new chunk and copy all the previous data + // doesn't fit, we create a new MemoryChunk and copy all the previous data // into it. The number of times we do this is directly proportional // to the total size of data that is going to be serialized. To make // the copying happen less often, round the next size up to the @@ -116,10 +118,10 @@ private: return roundUpToPageSize(size_after_grow); } - /// Add next contiguous chunk of memory with size not less than specified. - void NO_INLINE addChunk(size_t min_size) + /// Add next contiguous MemoryChunk of memory with size not less than specified. + void NO_INLINE addMemoryChunk(size_t min_size) { - head = new Chunk(nextSize(min_size + pad_right), head); + head = new MemoryChunk(nextSize(min_size + pad_right), head); size_in_bytes += head->size(); } @@ -129,7 +131,7 @@ private: public: Arena(size_t initial_size_ = 4096, size_t growth_factor_ = 2, size_t linear_growth_threshold_ = 128 * 1024 * 1024) : growth_factor(growth_factor_), linear_growth_threshold(linear_growth_threshold_), - head(new Chunk(initial_size_, nullptr)), size_in_bytes(head->size()) + head(new MemoryChunk(initial_size_, nullptr)), size_in_bytes(head->size()) { } @@ -142,7 +144,7 @@ public: char * alloc(size_t size) { if (unlikely(head->pos + size > head->end)) - addChunk(size); + addMemoryChunk(size); char * res = head->pos; head->pos += size; @@ -167,7 +169,7 @@ public: return res; } - addChunk(size + alignment); + addMemoryChunk(size + alignment); } while (true); } @@ -192,8 +194,8 @@ public: /** Begin or expand a contiguous range of memory. * 'range_start' is the start of range. If nullptr, a new range is * allocated. - * If there is no space in the current chunk to expand the range, - * the entire range is copied to a new, bigger memory chunk, and the value + * If there is no space in the current MemoryChunk to expand the range, + * the entire range is copied to a new, bigger memory MemoryChunk, and the value * of 'range_start' is updated. * If the optional 'start_alignment' is specified, the start of range is * kept aligned to this value. @@ -207,7 +209,7 @@ public: /* * Allocating zero bytes doesn't make much sense. Also, a zero-sized * range might break the invariant that the range begins at least before - * the current chunk end. + * the current MemoryChunk end. */ assert(additional_bytes > 0); @@ -226,19 +228,19 @@ public: // This method only works for extending the last allocation. For lack of // original size, check a weaker condition: that 'begin' is at least in - // the current Chunk. + // the current MemoryChunk. assert(range_start >= head->begin); assert(range_start < head->end); if (head->pos + additional_bytes <= head->end) { - // The new size fits into the last chunk, so just alloc the + // The new size fits into the last MemoryChunk, so just alloc the // additional size. We can alloc without alignment here, because it // only applies to the start of the range, and we don't change it. return alloc(additional_bytes); } - // New range doesn't fit into this chunk, will copy to a new one. + // New range doesn't fit into this MemoryChunk, will copy to a new one. // // Note: among other things, this method is used to provide a hack-ish // implementation of realloc over Arenas in ArenaAllocators. It wastes a @@ -299,16 +301,16 @@ public: return res; } - /// Size of chunks in bytes. + /// Size of MemoryChunks in bytes. size_t size() const { return size_in_bytes; } - /// Bad method, don't use it -- the chunks are not your business, the entire + /// Bad method, don't use it -- the MemoryChunks are not your business, the entire /// purpose of the arena code is to manage them for you, so if you find /// yourself having to use this method, probably you're doing something wrong. - size_t remainingSpaceInCurrentChunk() const + size_t remainingSpaceInCurrentMemoryChunk() const { return head->remaining(); } diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h index 8dd6cbbe02c..27b76865ab9 100644 --- a/src/Common/ThreadPool.h +++ b/src/Common/ThreadPool.h @@ -13,7 +13,6 @@ #include #include - /** Very simple thread pool similar to boost::threadpool. * Advantages: * - catches exceptions and rethrows on wait. diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 9fba199d0c1..6fde6bee430 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #if !defined(ARCADIA_BUILD) @@ -201,9 +201,9 @@ InputFormatPtr FormatFactory::getInput( { return input_getter(input, sample, row_input_format_params, format_settings); }; - ParallelParsingBlockInputFormat::Params params{ + ParallelParsingInputFormat::Params params{ buf, sample, parser_creator, file_segmentation_engine, name, settings.max_threads, settings.min_chunk_bytes_for_parallel_parsing}; - return std::make_shared(params); + return std::make_shared(params); } diff --git a/src/IO/WriteBufferFromArena.h b/src/IO/WriteBufferFromArena.h index ee09240e9c7..b5fd9fac5a3 100644 --- a/src/IO/WriteBufferFromArena.h +++ b/src/IO/WriteBufferFromArena.h @@ -35,7 +35,7 @@ private: /// tear down the entire WriteBuffer thing and implement it again, /// properly. size_t continuation_size = std::max(size_t(1), - std::max(count(), arena.remainingSpaceInCurrentChunk())); + std::max(count(), arena.remainingSpaceInCurrentMemoryChunk())); /// allocContinue method will possibly move memory region to new place and modify "begin" pointer. diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index e5c6f5b4ab8..4d070577e84 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -557,6 +557,7 @@ void writeProbablyBackQuotedStringMySQL(const StringRef & s, WriteBuffer & buf); template void writeCSVString(const char * begin, const char * end, WriteBuffer & buf) { + std::cout << StackTrace().toString() << std::endl; writeChar(quote, buf); const char * pos = begin; diff --git a/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp b/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp index 2d6a49ccb6f..4546f83d6cc 100644 --- a/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp @@ -11,6 +11,7 @@ namespace DB CSVRowOutputFormat::CSVRowOutputFormat(WriteBuffer & out_, const Block & header_, bool with_names_, const RowOutputFormatParams & params_, const FormatSettings & format_settings_) : IRowOutputFormat(header_, out_, params_), with_names(with_names_), format_settings(format_settings_) { + std::cout << StackTrace().toString() << std::endl; const auto & sample = getPort(PortKind::Main).getHeader(); size_t columns = sample.columns(); data_types.resize(columns); diff --git a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp index 15d8a843f41..6a87feab315 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp @@ -16,6 +16,7 @@ JSONEachRowRowOutputFormat::JSONEachRowRowOutputFormat( : IRowOutputFormat(header_, out_, params_), settings(settings_) { + std::cout << StackTrace().toString() << std::endl; const auto & sample = getPort(PortKind::Main).getHeader(); size_t columns = sample.columns(); fields.resize(columns); diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h new file mode 100644 index 00000000000..52b5475420d --- /dev/null +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -0,0 +1,147 @@ +#pragma once + +#include +#include +#include + +#include +#include + +#include + +namespace DB +{ + +const size_t min_chunk_bytes_for_parallel_formatting = 1024; + +class ParallelFormattingOutputFormat : public IOutputFormat +{ +public: + + struct Params + { + const Block & header; + WriteBuffer & out; + }; + + explicit ParallelFormattingOutputFormat(Params params) + : IOutputFormat(params.header, params.out) + , original_write_buffer(params.out) + { + + } + + String getName() const override final { return "ParallelFormattingOutputFormat"; } + +protected: + void consume(Chunk chunk) override final + { + if (chunk.empty()) + formatting_finished = true; + + const auto current_unit_number = writer_unit_number % processing_units.size(); + + auto & unit = processing_units[current_unit_number]; + + { + std::unique_lock lock(mutex); + writer_condvar.wait(lock, + [&]{ return unit.status == READY_TO_INSERT || formatting_finished; }); + } + + assert(unit.status == READY_TO_INSERT); + + unit.chunk = std::move(chunk); + /// TODO: Allocate new arena for current chunk + + + /// TODO: Submit task to ThreadPool + } + +private: + + WriteBuffer & original_write_buffer; + + enum ProcessingUnitStatus + { + READY_TO_INSERT, + READY_TO_FORMAT, + READY_TO_READ + }; + + struct ProcessingUnit + { + Chunk chunk; + Arena arena; + ProcessingUnitStatus status; + }; + + + + std::deque processing_units; + + std::mutex mutex; + std::atomic_bool formatting_finished{false}; + + + std::atomic_size_t collector_unit_number{0}; + std::atomic_size_t writer_unit_number{0}; + + std::condition_variable collector_condvar; + std::condition_variable writer_condvar; + + void сollectorThreadFunction() + { + setThreadName("Collector"); + + while (!formatting_finished) + { + const auto current_unit_number = collector_unit_number % processing_units.size(); + + auto & unit = processing_units[current_unit_number]; + + { + std::unique_lock lock(mutex); + collector_condvar.wait(lock, + [&]{ return unit.status == READY_TO_READ || formatting_finished; }); + } + + if (formatting_finished) + { + break; + } + + assert(unit.status == READY_TO_READ); + + /// TODO: Arena is singly linked list, and it is simply a list of chunks. + /// We have to write them all into original WriteBuffer. + char * arena_begin = nullptr; + size_t arena_size = 0; + /// Do main work here. + original_write_buffer.write(arena_begin, arena_size); + + /// How to drop this arena? + + + std::lock_guard lock(mutex); + unit.status = READY_TO_INSERT; + writer_condvar.notify_all(); + } + + } + + + void formatterThreadFunction(size_t current_unit_number) + { + setThreadName("Formatter"); + + auto & unit = processing_units[current_unit_number]; + + const char * arena_begin = nullptr; + WriteBufferFromArena out(unit.arena, arena_begin); + + /// TODO: create parser and parse current chunk to arena + } +}; + +} diff --git a/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp similarity index 88% rename from src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.cpp rename to src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index ad7a160ce11..898c103cb24 100644 --- a/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -1,10 +1,10 @@ -#include +#include #include namespace DB { -void ParallelParsingBlockInputFormat::segmentatorThreadFunction() +void ParallelParsingInputFormat::segmentatorThreadFunction() { setThreadName("Segmentator"); try @@ -49,7 +49,7 @@ void ParallelParsingBlockInputFormat::segmentatorThreadFunction() } } -void ParallelParsingBlockInputFormat::parserThreadFunction(size_t current_ticket_number) +void ParallelParsingInputFormat::parserThreadFunction(size_t current_ticket_number) { try { @@ -58,6 +58,8 @@ void ParallelParsingBlockInputFormat::parserThreadFunction(size_t current_ticket const auto current_unit_number = current_ticket_number % processing_units.size(); auto & unit = processing_units[current_unit_number]; + assert(unit.segment.size() > 0); + /* * This is kind of suspicious -- the input_process_creator contract with * respect to multithreaded use is not clear, but we hope that it is @@ -86,7 +88,7 @@ void ParallelParsingBlockInputFormat::parserThreadFunction(size_t current_ticket // We suppose we will get at least some blocks for a non-empty buffer, // except at the end of file. Also see a matching assert in readImpl(). - assert(unit.is_last || !unit.chunk_ext.chunk.empty()); + assert(unit.is_last || !unit.chunk_ext.chunk.empty() || parsing_finished); if (unit.is_last) endUpReadBuffer(read_buffer); @@ -102,7 +104,7 @@ void ParallelParsingBlockInputFormat::parserThreadFunction(size_t current_ticket } -void ParallelParsingBlockInputFormat::onBackgroundException() +void ParallelParsingInputFormat::onBackgroundException() { tryLogCurrentException(__PRETTY_FUNCTION__); @@ -116,7 +118,7 @@ void ParallelParsingBlockInputFormat::onBackgroundException() segmentator_condvar.notify_all(); } -Chunk ParallelParsingBlockInputFormat::generate() +Chunk ParallelParsingInputFormat::generate() { if (isCancelled() || parsing_finished) { @@ -206,17 +208,17 @@ Chunk ParallelParsingBlockInputFormat::generate() return res; } -void ParallelParsingBlockInputFormat::prepareReadBuffer(ReadBuffer & buffer) +void ParallelParsingInputFormat::prepareReadBuffer(ReadBuffer & buffer) { - if (prepare_and_end_up_ptr) - prepare_and_end_up_ptr->prepareReadBuffer(buffer); + if (prepare_and_end_up_map.count(format_name)) + prepare_and_end_up_map[format_name]->prepareReadBuffer(buffer); } -void ParallelParsingBlockInputFormat::endUpReadBuffer(ReadBuffer & buffer) +void ParallelParsingInputFormat::endUpReadBuffer(ReadBuffer & buffer) { - if (prepare_and_end_up_ptr) - prepare_and_end_up_ptr->endUpReadBuffer(buffer); + if (prepare_and_end_up_map.count(format_name)) + prepare_and_end_up_map[format_name]->endUpReadBuffer(buffer); } } diff --git a/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h similarity index 94% rename from src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.h rename to src/Processors/Formats/Impl/ParallelParsingInputFormat.h index 1a0cd2c537a..35cf7047d3c 100644 --- a/src/Processors/Formats/Impl/ParallelParsingBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include "IReadBufferPrepareAndEndUp.h" @@ -50,7 +49,7 @@ class Context; * 4) repeat until it encounters unit that is marked as "past_the_end" * All threads must also check for cancel/eof/exception flags. */ -class ParallelParsingBlockInputFormat : public IInputFormat +class ParallelParsingInputFormat : public IInputFormat { public: /* Used to recreate parser on every new data piece. */ @@ -67,7 +66,7 @@ public: size_t min_chunk_bytes; }; - explicit ParallelParsingBlockInputFormat(Params params) + explicit ParallelParsingInputFormat(Params params) : IInputFormat(std::move(params.header), params.in) , internal_parser_creator(params.internal_parser_creator) , file_segmentation_engine(params.file_segmentation_engine) @@ -83,13 +82,16 @@ public: // bump into reader thread on wraparound. processing_units.resize(params.max_threads + 2); - /// To skip '[' and ']'. - if (format_name == "JSONEachRow") - prepare_and_end_up_ptr = std::make_shared(); + initializePrepareEndUpMap(); segmentator_thread = ThreadFromGlobalPool([this] { segmentatorThreadFunction(); }); } + ~ParallelParsingInputFormat() override + { + finishAndWait(); + } + void resetParser() override final { throw Exception("resetParser() is not allowed for " + getName(), ErrorCodes::LOGICAL_ERROR); @@ -263,7 +265,14 @@ private: // is different. void onBackgroundException(); - IReadBufferPrepareAndEndUpPtr prepare_and_end_up_ptr; + /// To store objects which will prepare and end up ReadBuffer for each format. + std::unordered_map prepare_and_end_up_map; + + void initializePrepareEndUpMap() + { + /// To skip '[' and ']'. + prepare_and_end_up_map.insert({"JSONEachRow", std::make_shared()}); + } void prepareReadBuffer(ReadBuffer & buffer); From 9f127a46c7ca8374f73ce5550b30416f0760fc25 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 12 Jun 2020 02:00:49 +0300 Subject: [PATCH 025/284] first try --- src/Common/Arena.h | 6 +- src/Formats/FormatFactory.cpp | 37 ++- src/IO/BufferWithOwnMemory.h | 25 ++ src/IO/WriteHelpers.h | 1 - src/Processors/Formats/IOutputFormat.h | 3 + .../Formats/Impl/CSVRowOutputFormat.cpp | 1 - .../Impl/JSONEachRowRowOutputFormat.cpp | 1 - .../Impl/ParallelFormattingOutputFormat.h | 222 ++++++++++++++---- 8 files changed, 238 insertions(+), 58 deletions(-) diff --git a/src/Common/Arena.h b/src/Common/Arena.h index 2a05a257381..92de4e98ca5 100644 --- a/src/Common/Arena.h +++ b/src/Common/Arena.h @@ -15,7 +15,7 @@ namespace ProfileEvents { - extern const Event ArenaAllocMemoryChunks; + extern const Event ArenaAllocChunks; extern const Event ArenaAllocBytes; } @@ -45,18 +45,16 @@ private: char * end; /// does not include padding. MemoryChunk * prev; - MemoryChunk * next; MemoryChunk(size_t size_, MemoryChunk * prev_) { - ProfileEvents::increment(ProfileEvents::ArenaAllocMemoryChunks); + ProfileEvents::increment(ProfileEvents::ArenaAllocChunks); ProfileEvents::increment(ProfileEvents::ArenaAllocBytes, size_); begin = reinterpret_cast(Allocator::alloc(size_)); pos = begin; end = begin + size_ - pad_right; prev = prev_; - prev->next = this; ASAN_POISON_MEMORY_REGION(begin, size_); } diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 6fde6bee430..450f0463d6c 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -17,6 +16,7 @@ #include #include #include +#include #include #if !defined(ARCADIA_BUILD) @@ -233,10 +233,37 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, sample); } - auto format = getOutputFormat(name, buf, sample, context, std::move(callback), - format_settings); - return std::make_shared( - std::make_shared(format), sample); + + bool parallel_formatting = true; + + if (parallel_formatting && name == "CSV") + { + const auto & output_getter = getCreators(name).output_processor_creator; + if (!output_getter) + throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); + + const Settings & settings = context.getSettingsRef(); + FormatSettings format_settings = getOutputFormatSetting(settings, context); + + /** TODO: Materialization is needed, because formats can use the functions `IDataType`, + * which only work with full columns. + */ + auto formatter_creator = [output_getter, sample, callback, format_settings] + (WriteBuffer & output) -> OutputFormatPtr + { return output_getter(output, sample, std::move(callback), format_settings);}; + + /// Enable auto-flush for streaming mode. Currently it is needed by INSERT WATCH query. +// if (format_settings.enable_streaming) +// format->setAutoFlush(); + + ParallelFormattingOutputFormat::Params params{buf, sample, formatter_creator}; + auto format = std::make_shared(params); + return std::make_shared(std::make_shared(format), sample); + } + + + auto format = getOutputFormat(name, buf, sample, context, std::move(callback)); + return std::make_shared(std::make_shared(format), sample); } diff --git a/src/IO/BufferWithOwnMemory.h b/src/IO/BufferWithOwnMemory.h index 713c9b12cd1..b21c8cb5d17 100644 --- a/src/IO/BufferWithOwnMemory.h +++ b/src/IO/BufferWithOwnMemory.h @@ -149,4 +149,29 @@ public: }; +/** Buffer that could write data to external memory which came from outside + * Template parameter: ReadBuffer or WriteBuffer + */ +template +class BufferWithOutsideMemory : public Base +{ +protected: + Memory<> & memory; +public: + + BufferWithOutsideMemory(Memory<> & memory_) + : Base(nullptr, 0), memory(memory_) + { + Base::set(memory.data(), memory.size()); + Base::padded = false; + } +private: + void nextImpl() override final + { + const size_t prev_size = memory.size(); + memory.resize(2 * prev_size); + Base::set(memory.data(), memory.size()); + } +}; + } diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 4d070577e84..e5c6f5b4ab8 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -557,7 +557,6 @@ void writeProbablyBackQuotedStringMySQL(const StringRef & s, WriteBuffer & buf); template void writeCSVString(const char * begin, const char * end, WriteBuffer & buf) { - std::cout << StackTrace().toString() << std::endl; writeChar(quote, buf); const char * pos = begin; diff --git a/src/Processors/Formats/IOutputFormat.h b/src/Processors/Formats/IOutputFormat.h index 67c307df2aa..8f5d137f08f 100644 --- a/src/Processors/Formats/IOutputFormat.h +++ b/src/Processors/Formats/IOutputFormat.h @@ -39,12 +39,15 @@ protected: RowsBeforeLimitCounterPtr rows_before_limit_counter; + friend class ParallelFormattingOutputFormat; + virtual void consume(Chunk) = 0; virtual void consumeTotals(Chunk) {} virtual void consumeExtremes(Chunk) {} virtual void finalize() {} public: + IOutputFormat(const Block & header_, WriteBuffer & out_); Status prepare() override; diff --git a/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp b/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp index 4546f83d6cc..2d6a49ccb6f 100644 --- a/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp @@ -11,7 +11,6 @@ namespace DB CSVRowOutputFormat::CSVRowOutputFormat(WriteBuffer & out_, const Block & header_, bool with_names_, const RowOutputFormatParams & params_, const FormatSettings & format_settings_) : IRowOutputFormat(header_, out_, params_), with_names(with_names_), format_settings(format_settings_) { - std::cout << StackTrace().toString() << std::endl; const auto & sample = getPort(PortKind::Main).getHeader(); size_t columns = sample.columns(); data_types.resize(columns); diff --git a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp index 6a87feab315..15d8a843f41 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp @@ -16,7 +16,6 @@ JSONEachRowRowOutputFormat::JSONEachRowRowOutputFormat( : IRowOutputFormat(header_, out_, params_), settings(settings_) { - std::cout << StackTrace().toString() << std::endl; const auto & sample = getPort(PortKind::Main).getHeader(); size_t columns = sample.columns(); fields.resize(columns); diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 52b5475420d..0a8fe511471 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -3,32 +3,50 @@ #include #include #include +#include #include #include #include +#include +#include +#include namespace DB { -const size_t min_chunk_bytes_for_parallel_formatting = 1024; +const size_t min_chunk_bytes_for_parallel_formatting_ = 1024; +const size_t max_threads_for_parallel_formatting_ = 4; class ParallelFormattingOutputFormat : public IOutputFormat { public: + /* Used to recreate formatter on every new data piece. */ + using InternalFormatterCreator = std::function; struct Params { - const Block & header; - WriteBuffer & out; + WriteBuffer & out_; + const Block & header_; + InternalFormatterCreator internal_formatter_creator_; }; explicit ParallelFormattingOutputFormat(Params params) - : IOutputFormat(params.header, params.out) - , original_write_buffer(params.out) - { + : IOutputFormat(params.header_, params.out_) + , internal_formatter_creator(params.internal_formatter_creator_) + , pool(max_threads_for_parallel_formatting_) + { +// std::cout << "ParallelFormattingOutputFormat::constructor" << std::endl; + processing_units.resize(max_threads_for_parallel_formatting_ + 2); + + collector_thread = ThreadFromGlobalPool([this] { сollectorThreadFunction(); }); + } + + ~ParallelFormattingOutputFormat() override + { + finishAndWait(); } String getName() const override final { return "ParallelFormattingOutputFormat"; } @@ -36,31 +54,46 @@ public: protected: void consume(Chunk chunk) override final { + + std::cout << StackTrace().toString() << std::endl; + if (chunk.empty()) + { +// std::cout << "Empty chunk" << std::endl; formatting_finished = true; + } + const auto current_unit_number = writer_unit_number % processing_units.size(); auto & unit = processing_units[current_unit_number]; +// std::cout << "Consume " << current_unit_number << " before wait" << std::endl; + { std::unique_lock lock(mutex); writer_condvar.wait(lock, [&]{ return unit.status == READY_TO_INSERT || formatting_finished; }); } +// std::cout << "Consume " << current_unit_number << " after wait" << std::endl; + assert(unit.status == READY_TO_INSERT); unit.chunk = std::move(chunk); - /// TODO: Allocate new arena for current chunk + /// Resize memory without deallocate + unit.segment.resize(0); + unit.status = READY_TO_FORMAT; - /// TODO: Submit task to ThreadPool + scheduleFormatterThreadForUnitWithNumber(current_unit_number); + + ++writer_unit_number; } private: - WriteBuffer & original_write_buffer; + InternalFormatterCreator internal_formatter_creator; enum ProcessingUnitStatus { @@ -71,63 +104,134 @@ private: struct ProcessingUnit { + explicit ProcessingUnit() + : status(ProcessingUnitStatus::READY_TO_INSERT) + { + } + + std::atomic status; Chunk chunk; - Arena arena; - ProcessingUnitStatus status; + Memory<> segment; }; + // There are multiple "formatters", that's why we use thread pool. + ThreadPool pool; + // Collecting all memory to original ReadBuffer + ThreadFromGlobalPool collector_thread; + std::exception_ptr background_exception = nullptr; std::deque processing_units; std::mutex mutex; std::atomic_bool formatting_finished{false}; - std::atomic_size_t collector_unit_number{0}; std::atomic_size_t writer_unit_number{0}; std::condition_variable collector_condvar; std::condition_variable writer_condvar; + void finishAndWait() + { + + std::cout << "finishAndWait()" << std::endl; + formatting_finished = true; + + { + std::unique_lock lock(mutex); + collector_condvar.notify_all(); + writer_condvar.notify_all(); + } + + if (collector_thread.joinable()) + collector_thread.join(); + + try + { + pool.wait(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + + + void onBackgroundException() + { +// std::cout << "onBackgroundException" << std::endl; + tryLogCurrentException(__PRETTY_FUNCTION__); + + std::unique_lock lock(mutex); + if (!background_exception) + { + background_exception = std::current_exception(); + } + formatting_finished = true; + writer_condvar.notify_all(); + collector_condvar.notify_all(); + } + + void scheduleFormatterThreadForUnitWithNumber(size_t ticket_number) + { + pool.scheduleOrThrowOnError([this, ticket_number] { formatterThreadFunction(ticket_number); }); + } + void сollectorThreadFunction() { setThreadName("Collector"); - while (!formatting_finished) + try { - const auto current_unit_number = collector_unit_number % processing_units.size(); - - auto & unit = processing_units[current_unit_number]; - + while (!formatting_finished) { - std::unique_lock lock(mutex); - collector_condvar.wait(lock, - [&]{ return unit.status == READY_TO_READ || formatting_finished; }); + const auto current_unit_number = collector_unit_number % processing_units.size(); + + auto &unit = processing_units[current_unit_number]; + +// std::cout << "Collector " << current_unit_number << " before wait" << std::endl; + + { + std::unique_lock lock(mutex); + collector_condvar.wait(lock, + [&]{ return unit.status == READY_TO_READ || formatting_finished; }); + } + +// std::cout << "Collector " << current_unit_number << " after wait" << std::endl; + + if (formatting_finished) + { +// std::cout << "formatting finished" << std::endl; + break; + } + + assert(unit.status == READY_TO_READ); + assert(unit.segment.size() > 0); + + for (size_t i=0; i < unit.segment.size(); ++i) + { + std::cout << *(unit.segment.data() + i); + } + std::cout << std::endl; + + + /// Do main work here. + out.write(unit.segment.data(), unit.segment.size()); + + flush(); + + ++collector_unit_number; + + std::lock_guard lock(mutex); + unit.status = READY_TO_INSERT; + writer_condvar.notify_all(); } - - if (formatting_finished) - { - break; - } - - assert(unit.status == READY_TO_READ); - - /// TODO: Arena is singly linked list, and it is simply a list of chunks. - /// We have to write them all into original WriteBuffer. - char * arena_begin = nullptr; - size_t arena_size = 0; - /// Do main work here. - original_write_buffer.write(arena_begin, arena_size); - - /// How to drop this arena? - - - std::lock_guard lock(mutex); - unit.status = READY_TO_INSERT; - writer_condvar.notify_all(); } - + catch (...) + { + onBackgroundException(); + } } @@ -135,12 +239,38 @@ private: { setThreadName("Formatter"); - auto & unit = processing_units[current_unit_number]; + try + { +// std::cout << "Formatter " << current_unit_number << std::endl; - const char * arena_begin = nullptr; - WriteBufferFromArena out(unit.arena, arena_begin); + auto & unit = processing_units[current_unit_number]; + + assert(unit.status = READY_TO_FORMAT); + + unit.segment.resize(10 * DBMS_DEFAULT_BUFFER_SIZE); + + /// TODO: Implement proper nextImpl + BufferWithOutsideMemory out(unit.segment); + + auto formatter = internal_formatter_creator(out); + + formatter->consume(std::move(unit.chunk)); + +// std::cout << "Formatter " << current_unit_number << " before notify" << std::endl; + + { + std::lock_guard lock(mutex); + unit.status = READY_TO_READ; + collector_condvar.notify_all(); + } + +// std::cout << "Formatter " << current_unit_number << " after notify" << std::endl; + } + catch (...) + { + onBackgroundException(); + } - /// TODO: create parser and parse current chunk to arena } }; From 3bc1affd214b9ec06d2b7503892b0d5f80b1e586 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 12 Jun 2020 02:08:14 +0300 Subject: [PATCH 026/284] remove CSV restriction --- programs/client/Client.cpp | 2 ++ src/Formats/FormatFactory.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 92ae37123fa..06ce0a615eb 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1993,6 +1993,8 @@ private: /// Received data block is immediately displayed to the user. block_out_stream->flush(); + std::this_thread::sleep_for(std::chrono::seconds(2)); + /// Restore progress bar after data block. if (clear_progess) writeProgress(); diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 450f0463d6c..07103ae1ed3 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -236,7 +236,7 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, bool parallel_formatting = true; - if (parallel_formatting && name == "CSV") + if (parallel_formatting) { const auto & output_getter = getCreators(name).output_processor_creator; if (!output_getter) From 82888b5c4de060d8b7ad379fa1185ce4a8eb325f Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 12 Jun 2020 15:14:21 +0300 Subject: [PATCH 027/284] fix build --- programs/odbc-bridge/MainHandler.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 9bbffed3e23..78b55bf531e 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -22,6 +22,7 @@ #if USE_ODBC #include +#include #define POCO_SQL_ODBC_CLASS Poco::Data::ODBC #endif @@ -162,8 +163,9 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne auto pool = getPool(connection_string); ReadBufferFromIStream read_buf(request.stream()); - BlockInputStreamPtr input_stream = FormatFactory::instance().getInput(format, read_buf, *sample_block, - context, max_block_size); + auto input_format = FormatFactory::instance().getInput(format, read_buf, *sample_block, + context, max_block_size); + auto input_stream = std::make_shared(input_format); ODBCBlockOutputStream output_stream(pool->get(), db_name, table_name, *sample_block, quoting_style); copyData(*input_stream, output_stream); writeStringBinary("Ok.", out); From edc6267e3f70669813efaccb8113d73af6094cba Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 15 Jun 2020 12:03:08 +0300 Subject: [PATCH 028/284] fix build --- src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 0a8fe511471..7f63202de6e 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -250,9 +250,9 @@ private: unit.segment.resize(10 * DBMS_DEFAULT_BUFFER_SIZE); /// TODO: Implement proper nextImpl - BufferWithOutsideMemory out(unit.segment); + BufferWithOutsideMemory out_buffer(unit.segment); - auto formatter = internal_formatter_creator(out); + auto formatter = internal_formatter_creator(out_buffer); formatter->consume(std::move(unit.chunk)); From f50750dfed498da878d445c61b1f2cc591241c0e Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 15 Jun 2020 13:08:41 +0300 Subject: [PATCH 029/284] remove stacktrace output --- src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 7f63202de6e..a919f1d2b3b 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -55,7 +55,7 @@ protected: void consume(Chunk chunk) override final { - std::cout << StackTrace().toString() << std::endl; +// std::cout << StackTrace().toString() << std::endl; if (chunk.empty()) { From 7d7c73c5fcdc89291faaa3627c388bf73c32f907 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 15 Jun 2020 20:38:30 +0300 Subject: [PATCH 030/284] fix build final --- src/DataStreams/ParallelParsingBlockInputStream.cpp | 1 - src/Processors/Formats/IInputFormat.h | 7 ------- src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp | 6 ++---- .../Formats/Impl/ParallelFormattingOutputFormat.h | 4 ++-- 4 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/DataStreams/ParallelParsingBlockInputStream.cpp b/src/DataStreams/ParallelParsingBlockInputStream.cpp index 19b04d36fc1..8ec1f2bc499 100644 --- a/src/DataStreams/ParallelParsingBlockInputStream.cpp +++ b/src/DataStreams/ParallelParsingBlockInputStream.cpp @@ -170,7 +170,6 @@ void ParallelParsingBlockInputStream::parserThreadFunction(ThreadGroupStatusPtr */ ReadBuffer read_buffer(unit.segment.data(), unit.segment.size(), 0); auto format = input_processor_creator(read_buffer, header, row_input_format_params, format_settings); - format->setCurrentUnitNumber(current_ticket_number); auto parser = std::make_unique(std::move(format)); unit.block_ext.block.clear(); diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index e1537aff6c5..00cb38405cf 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -38,13 +38,6 @@ public: static const BlockMissingValues none; return none; } - - size_t getCurrentUnitNumber() const { return current_unit_number; } - void setCurrentUnitNumber(size_t current_unit_number_) { current_unit_number = current_unit_number_; } - -private: - /// Number of currently parsed chunk (if parallel parsing is enabled) - size_t current_unit_number = 0; }; } diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp index dfb5eb79b4b..ec1deddc94b 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp @@ -245,11 +245,9 @@ bool JSONEachRowRowInputFormat::readRow(MutableColumns & columns, RowReadExtensi /// then seeking to next ;, or \n would trigger reading of an extra row at the end. /// Semicolon is added for convenience as it could be used at end of INSERT query. - bool is_first_row = getCurrentUnitNumber() == 0 && getTotalRows() == 1; - if (!in.eof()) + if (getTotalRows() && !in.eof()) { - /// There may be optional ',' (but not before the first row) - if (!is_first_row && *in.position() == ',') + if (*in.position() == ',') ++in.position(); else if (!data_in_square_brackets && *in.position() == ';') { diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index a919f1d2b3b..45b0ef8e18c 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -41,7 +41,7 @@ public: // std::cout << "ParallelFormattingOutputFormat::constructor" << std::endl; processing_units.resize(max_threads_for_parallel_formatting_ + 2); - collector_thread = ThreadFromGlobalPool([this] { сollectorThreadFunction(); }); + collector_thread = ThreadFromGlobalPool([this] { collectorThreadFunction(); }); } ~ParallelFormattingOutputFormat() override @@ -178,7 +178,7 @@ private: pool.scheduleOrThrowOnError([this, ticket_number] { formatterThreadFunction(ticket_number); }); } - void сollectorThreadFunction() + void collectorThreadFunction() { setThreadName("Collector"); From 0e31424c92201b76c2c63dbbfc550909e9a3c0b6 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 16 Jul 2020 13:48:57 +0300 Subject: [PATCH 031/284] save changes --- programs/client/Client.cpp | 2 - .../Impl/ParallelFormattingOutputFormat.h | 50 +++++++++---------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 06ce0a615eb..92ae37123fa 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1993,8 +1993,6 @@ private: /// Received data block is immediately displayed to the user. block_out_stream->flush(); - std::this_thread::sleep_for(std::chrono::seconds(2)); - /// Restore progress bar after data block. if (clear_progess) writeProgress(); diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 45b0ef8e18c..eed8a38f02c 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -16,8 +16,8 @@ namespace DB { -const size_t min_chunk_bytes_for_parallel_formatting_ = 1024; -const size_t max_threads_for_parallel_formatting_ = 4; +const size_t min_chunk_bytes_for_parallel_formatting = 1024; +const size_t max_threads_for_parallel_formatting = 4; class ParallelFormattingOutputFormat : public IOutputFormat { @@ -27,19 +27,19 @@ public: struct Params { - WriteBuffer & out_; - const Block & header_; - InternalFormatterCreator internal_formatter_creator_; + WriteBuffer & out; + const Block & header; + InternalFormatterCreator internal_formatter_creator; }; explicit ParallelFormattingOutputFormat(Params params) - : IOutputFormat(params.header_, params.out_) - , internal_formatter_creator(params.internal_formatter_creator_) - , pool(max_threads_for_parallel_formatting_) + : IOutputFormat(params.header, params.out) + , internal_formatter_creator(params.internal_formatter_creator) + , pool(max_threads_for_parallel_formatting) { -// std::cout << "ParallelFormattingOutputFormat::constructor" << std::endl; - processing_units.resize(max_threads_for_parallel_formatting_ + 2); + std::cout << "ParallelFormattingOutputFormat::constructor" << std::endl; + processing_units.resize(max_threads_for_parallel_formatting + 2); collector_thread = ThreadFromGlobalPool([this] { collectorThreadFunction(); }); } @@ -54,12 +54,11 @@ public: protected: void consume(Chunk chunk) override final { - -// std::cout << StackTrace().toString() << std::endl; + std::cout << StackTrace().toString() << std::endl; if (chunk.empty()) { -// std::cout << "Empty chunk" << std::endl; + std::cout << "Empty chunk" << std::endl; formatting_finished = true; } @@ -68,7 +67,7 @@ protected: auto & unit = processing_units[current_unit_number]; -// std::cout << "Consume " << current_unit_number << " before wait" << std::endl; + std::cout << "Consume " << current_unit_number << " before wait" << std::endl; { std::unique_lock lock(mutex); @@ -76,7 +75,7 @@ protected: [&]{ return unit.status == READY_TO_INSERT || formatting_finished; }); } -// std::cout << "Consume " << current_unit_number << " after wait" << std::endl; + std::cout << "Consume " << current_unit_number << " after wait" << std::endl; assert(unit.status == READY_TO_INSERT); @@ -136,6 +135,7 @@ private: { std::cout << "finishAndWait()" << std::endl; + std::cout << StackTrace().toString() << std::endl; formatting_finished = true; { @@ -160,7 +160,7 @@ private: void onBackgroundException() { -// std::cout << "onBackgroundException" << std::endl; + std::cout << "onBackgroundException" << std::endl; tryLogCurrentException(__PRETTY_FUNCTION__); std::unique_lock lock(mutex); @@ -190,7 +190,7 @@ private: auto &unit = processing_units[current_unit_number]; -// std::cout << "Collector " << current_unit_number << " before wait" << std::endl; + std::cout << "Collector " << current_unit_number << " before wait" << std::endl; { std::unique_lock lock(mutex); @@ -198,13 +198,13 @@ private: [&]{ return unit.status == READY_TO_READ || formatting_finished; }); } -// std::cout << "Collector " << current_unit_number << " after wait" << std::endl; + std::cout << "Collector " << current_unit_number << " after wait" << std::endl; - if (formatting_finished) - { +// if (formatting_finished) +// { // std::cout << "formatting finished" << std::endl; - break; - } +// break; +// } assert(unit.status == READY_TO_READ); assert(unit.segment.size() > 0); @@ -241,7 +241,7 @@ private: try { -// std::cout << "Formatter " << current_unit_number << std::endl; + std::cout << "Formatter " << current_unit_number << std::endl; auto & unit = processing_units[current_unit_number]; @@ -256,7 +256,7 @@ private: formatter->consume(std::move(unit.chunk)); -// std::cout << "Formatter " << current_unit_number << " before notify" << std::endl; + std::cout << "Formatter " << current_unit_number << " before notify" << std::endl; { std::lock_guard lock(mutex); @@ -264,7 +264,7 @@ private: collector_condvar.notify_all(); } -// std::cout << "Formatter " << current_unit_number << " after notify" << std::endl; + std::cout << "Formatter " << current_unit_number << " after notify" << std::endl; } catch (...) { From 0b4ff5f284bb14f87dccd063c6efc2f7ceea3c7b Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 29 Jul 2020 18:42:02 +0300 Subject: [PATCH 032/284] save --- .../Impl/ParallelFormattingOutputFormat.h | 61 ++++++++++++------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index eed8a38f02c..ba809554dff 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -38,7 +38,7 @@ public: , pool(max_threads_for_parallel_formatting) { - std::cout << "ParallelFormattingOutputFormat::constructor" << std::endl; +// std::cout << "ParallelFormattingOutputFormat::constructor" << std::endl; processing_units.resize(max_threads_for_parallel_formatting + 2); collector_thread = ThreadFromGlobalPool([this] { collectorThreadFunction(); }); @@ -46,19 +46,34 @@ public: ~ParallelFormattingOutputFormat() override { + std::cout << "ParallelFormattingOutputFormat::destructor" << std::endl; finishAndWait(); } String getName() const override final { return "ParallelFormattingOutputFormat"; } + void flush() override final + { + ///FIXME + while(writer_unit_number - collector_unit_number > 1) + { + std::cout << writer_unit_number << ' ' << collector_unit_number << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + IOutputFormat::flush(); + } + protected: void consume(Chunk chunk) override final { - std::cout << StackTrace().toString() << std::endl; +// std::cout << StackTrace().toString() << std::endl; + + std::cout << chunk.dumpStructure() << std::endl; if (chunk.empty()) { - std::cout << "Empty chunk" << std::endl; +// std::cout << "Empty chunk" << std::endl; formatting_finished = true; } @@ -67,7 +82,7 @@ protected: auto & unit = processing_units[current_unit_number]; - std::cout << "Consume " << current_unit_number << " before wait" << std::endl; +// std::cout << "Consume " << current_unit_number << " before wait" << std::endl; { std::unique_lock lock(mutex); @@ -75,7 +90,7 @@ protected: [&]{ return unit.status == READY_TO_INSERT || formatting_finished; }); } - std::cout << "Consume " << current_unit_number << " after wait" << std::endl; +// std::cout << "Consume " << current_unit_number << " after wait" << std::endl; assert(unit.status == READY_TO_INSERT); @@ -134,8 +149,8 @@ private: void finishAndWait() { - std::cout << "finishAndWait()" << std::endl; - std::cout << StackTrace().toString() << std::endl; +// std::cout << "finishAndWait()" << std::endl; +// std::cout << StackTrace().toString() << std::endl; formatting_finished = true; { @@ -160,7 +175,7 @@ private: void onBackgroundException() { - std::cout << "onBackgroundException" << std::endl; +// std::cout << "onBackgroundException" << std::endl; tryLogCurrentException(__PRETTY_FUNCTION__); std::unique_lock lock(mutex); @@ -190,7 +205,7 @@ private: auto &unit = processing_units[current_unit_number]; - std::cout << "Collector " << current_unit_number << " before wait" << std::endl; +// std::cout << "Collector " << current_unit_number << " before wait" << std::endl; { std::unique_lock lock(mutex); @@ -198,22 +213,22 @@ private: [&]{ return unit.status == READY_TO_READ || formatting_finished; }); } - std::cout << "Collector " << current_unit_number << " after wait" << std::endl; +// std::cout << "Collector " << current_unit_number << " after wait" << std::endl; -// if (formatting_finished) -// { -// std::cout << "formatting finished" << std::endl; -// break; -// } + if (formatting_finished) + { + std::cout << "formatting finished" << std::endl; + break; + } assert(unit.status == READY_TO_READ); assert(unit.segment.size() > 0); - for (size_t i=0; i < unit.segment.size(); ++i) - { - std::cout << *(unit.segment.data() + i); - } - std::cout << std::endl; +// for (size_t i=0; i < unit.segment.size(); ++i) +// { +// std::cout << *(unit.segment.data() + i); +// } +// std::cout << std::endl; /// Do main work here. @@ -241,7 +256,7 @@ private: try { - std::cout << "Formatter " << current_unit_number << std::endl; +// std::cout << "Formatter " << current_unit_number << std::endl; auto & unit = processing_units[current_unit_number]; @@ -256,7 +271,7 @@ private: formatter->consume(std::move(unit.chunk)); - std::cout << "Formatter " << current_unit_number << " before notify" << std::endl; +// std::cout << "Formatter " << current_unit_number << " before notify" << std::endl; { std::lock_guard lock(mutex); @@ -264,7 +279,7 @@ private: collector_condvar.notify_all(); } - std::cout << "Formatter " << current_unit_number << " after notify" << std::endl; +// std::cout << "Formatter " << current_unit_number << " after notify" << std::endl; } catch (...) { From 5a47928431cb75b56954bb9d0b1f6b31d51d101b Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 3 Aug 2020 20:38:11 +0300 Subject: [PATCH 033/284] save --- base/CMakeLists.txt | 2 +- programs/CMakeLists.txt | 2 +- src/CMakeLists.txt | 2 +- .../Formats/Impl/ParallelFormattingOutputFormat.h | 1 + src/Storages/MergeTree/MergeTreeData.cpp | 4 +++- src/Storages/StorageFile.cpp | 8 +++----- utils/CMakeLists.txt | 2 +- 7 files changed, 11 insertions(+), 10 deletions(-) diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt index 46bd57eda12..b8a4f144489 100644 --- a/base/CMakeLists.txt +++ b/base/CMakeLists.txt @@ -1,5 +1,5 @@ if (USE_CLANG_TIDY) - set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") + set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH} -fix") endif () add_subdirectory (common) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 8f45bf53f53..8ea020a1278 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -1,5 +1,5 @@ if (USE_CLANG_TIDY) - set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") + set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH} -fix") endif () # The `clickhouse` binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6021065f937..a99a4e17956 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,7 +3,7 @@ if (USE_INCLUDE_WHAT_YOU_USE) endif () if (USE_CLANG_TIDY) - set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") + set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH} -fix") endif () if(COMPILER_PIPE) diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index ba809554dff..43c822ebee2 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -54,6 +54,7 @@ public: void flush() override final { + std::cout << "flush()" << std::endl; ///FIXME while(writer_unit_number - collector_unit_number > 1) { diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 9804abb2247..01632b57989 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include @@ -2904,7 +2905,8 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, const Context ReadBufferFromMemory right_paren_buf(")", 1); ConcatReadBuffer buf({&left_paren_buf, &fields_buf, &right_paren_buf}); - auto input_format = FormatFactory::instance().getInput("Values", buf, getPartitionKey().sample_block, context, context.getSettingsRef().max_block_size); + + auto input_format = FormatFactory::instance().getInput("Values", buf, metadata_snapshot->getPartitionKey().sample_block, context, context.getSettingsRef().max_block_size); auto input_stream = std::make_shared(input_format); auto block = input_stream->read(); diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 736089b666f..caf9c06e0ba 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -326,12 +326,10 @@ public: method = chooseCompressionMethod(current_path, storage->compression_method); } - read_buf = wrapReadBufferWithCompressionMethod( - std::move(nested_buffer), method); - reader = FormatFactory::instance().getInput(storage->format_name, - *read_buf, metadata_snapshot->getSampleBlock(), context, - max_block_size, storage->format_settings); + read_buf = wrapReadBufferWithCompressionMethod(std::move(nested_buffer), method); + auto format = FormatFactory::instance().getInput( + storage->format_name, *read_buf, metadata_snapshot->getSampleBlock(), context, max_block_size); reader = std::make_shared(format); diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index a27a7e9dadc..680554be57d 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,5 +1,5 @@ if (USE_CLANG_TIDY) - set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") + set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH} -fix") endif () if(MAKE_STATIC_LIBRARIES) From 9922324787047a993c68315474c09a36bc51e32b Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 14 Aug 2020 03:34:35 +0300 Subject: [PATCH 034/284] it works --- src/Core/Settings.h | 1 + src/Formats/FormatFactory.cpp | 4 +- src/IO/BufferWithOwnMemory.h | 11 ++- .../Impl/ParallelFormattingOutputFormat.h | 78 ++++++------------- 4 files changed, 37 insertions(+), 57 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 0b4aa65a37b..5502c692865 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -121,6 +121,7 @@ class IColumn; \ M(Bool, input_format_parallel_parsing, true, "Enable parallel parsing for some data formats.", 0) \ M(UInt64, min_chunk_bytes_for_parallel_parsing, (10 * 1024 * 1024), "The minimum chunk size in bytes, which each thread will parse in parallel.", 0) \ + M(Bool, output_format_parallel_formatting, false, "Enable parallel formatting for all data formats.", 0) \ \ M(UInt64, merge_tree_min_rows_for_concurrent_read, (20 * 8192), "If at least as many lines are read from one file, the reading can be parallelized.", 0) \ M(UInt64, merge_tree_min_bytes_for_concurrent_read, (24 * 10 * 1024 * 1024), "If at least as many bytes are read from one file, the reading can be parallelized.", 0) \ diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 07103ae1ed3..a7eb4d9df4e 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -234,7 +234,8 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, } - bool parallel_formatting = true; + const Settings & settings = context.getSettingsRef(); + bool parallel_formatting = settings.output_format_parallel_formatting; if (parallel_formatting) { @@ -242,7 +243,6 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, if (!output_getter) throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); - const Settings & settings = context.getSettingsRef(); FormatSettings format_settings = getOutputFormatSetting(settings, context); /** TODO: Materialization is needed, because formats can use the functions `IDataType`, diff --git a/src/IO/BufferWithOwnMemory.h b/src/IO/BufferWithOwnMemory.h index b21c8cb5d17..396f8a36791 100644 --- a/src/IO/BufferWithOwnMemory.h +++ b/src/IO/BufferWithOwnMemory.h @@ -159,18 +159,25 @@ protected: Memory<> & memory; public: - BufferWithOutsideMemory(Memory<> & memory_) + explicit BufferWithOutsideMemory(Memory<> & memory_) : Base(nullptr, 0), memory(memory_) { Base::set(memory.data(), memory.size()); Base::padded = false; } + + size_t getActualSize() + { + return Base::count(); + } + private: void nextImpl() override final { + std::cout << "nextImpl" << std::endl; const size_t prev_size = memory.size(); memory.resize(2 * prev_size); - Base::set(memory.data(), memory.size()); + Base::set(memory.data() + prev_size, memory.size() - prev_size); } }; diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 43c822ebee2..68109f181b5 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -17,7 +17,7 @@ namespace DB { const size_t min_chunk_bytes_for_parallel_formatting = 1024; -const size_t max_threads_for_parallel_formatting = 4; +const size_t max_threads_for_parallel_formatting = 6; class ParallelFormattingOutputFormat : public IOutputFormat { @@ -38,15 +38,14 @@ public: , pool(max_threads_for_parallel_formatting) { -// std::cout << "ParallelFormattingOutputFormat::constructor" << std::endl; processing_units.resize(max_threads_for_parallel_formatting + 2); - collector_thread = ThreadFromGlobalPool([this] { collectorThreadFunction(); }); + collector_thread = ThreadFromGlobalPool([&] { collectorThreadFunction(); }); } ~ParallelFormattingOutputFormat() override { - std::cout << "ParallelFormattingOutputFormat::destructor" << std::endl; + flush(); finishAndWait(); } @@ -54,45 +53,23 @@ public: void flush() override final { - std::cout << "flush()" << std::endl; - ///FIXME - while(writer_unit_number - collector_unit_number > 1) - { - std::cout << writer_unit_number << ' ' << collector_unit_number << std::endl; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - + waitForFormattingFinished(); IOutputFormat::flush(); } protected: void consume(Chunk chunk) override final { -// std::cout << StackTrace().toString() << std::endl; - - std::cout << chunk.dumpStructure() << std::endl; - - if (chunk.empty()) - { -// std::cout << "Empty chunk" << std::endl; - formatting_finished = true; - } - - const auto current_unit_number = writer_unit_number % processing_units.size(); auto & unit = processing_units[current_unit_number]; -// std::cout << "Consume " << current_unit_number << " before wait" << std::endl; - { std::unique_lock lock(mutex); writer_condvar.wait(lock, [&]{ return unit.status == READY_TO_INSERT || formatting_finished; }); } -// std::cout << "Consume " << current_unit_number << " after wait" << std::endl; - assert(unit.status == READY_TO_INSERT); unit.chunk = std::move(chunk); @@ -127,6 +104,7 @@ private: std::atomic status; Chunk chunk; Memory<> segment; + size_t actual_memory_size{0}; }; // There are multiple "formatters", that's why we use thread pool. @@ -149,9 +127,6 @@ private: void finishAndWait() { - -// std::cout << "finishAndWait()" << std::endl; -// std::cout << StackTrace().toString() << std::endl; formatting_finished = true; { @@ -176,7 +151,6 @@ private: void onBackgroundException() { -// std::cout << "onBackgroundException" << std::endl; tryLogCurrentException(__PRETTY_FUNCTION__); std::unique_lock lock(mutex); @@ -194,6 +168,21 @@ private: pool.scheduleOrThrowOnError([this, ticket_number] { formatterThreadFunction(ticket_number); }); } + void waitForFormattingFinished() + { + ///FIXME + while(hasChunksToWorkWith()) + { + std::cout << writer_unit_number << ' ' << collector_unit_number << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + } + + bool hasChunksToWorkWith() + { + return writer_unit_number - collector_unit_number > 0; + } + void collectorThreadFunction() { setThreadName("Collector"); @@ -206,36 +195,21 @@ private: auto &unit = processing_units[current_unit_number]; -// std::cout << "Collector " << current_unit_number << " before wait" << std::endl; - { std::unique_lock lock(mutex); collector_condvar.wait(lock, [&]{ return unit.status == READY_TO_READ || formatting_finished; }); } -// std::cout << "Collector " << current_unit_number << " after wait" << std::endl; - if (formatting_finished) - { - std::cout << "formatting finished" << std::endl; break; - } assert(unit.status == READY_TO_READ); assert(unit.segment.size() > 0); -// for (size_t i=0; i < unit.segment.size(); ++i) -// { -// std::cout << *(unit.segment.data() + i); -// } -// std::cout << std::endl; - - /// Do main work here. - out.write(unit.segment.data(), unit.segment.size()); - - flush(); + out.write(unit.segment.data(), unit.actual_memory_size); + out.sync(); ++collector_unit_number; @@ -257,13 +231,11 @@ private: try { -// std::cout << "Formatter " << current_unit_number << std::endl; - auto & unit = processing_units[current_unit_number]; assert(unit.status = READY_TO_FORMAT); - unit.segment.resize(10 * DBMS_DEFAULT_BUFFER_SIZE); + unit.segment.resize(DBMS_DEFAULT_BUFFER_SIZE); /// TODO: Implement proper nextImpl BufferWithOutsideMemory out_buffer(unit.segment); @@ -271,8 +243,9 @@ private: auto formatter = internal_formatter_creator(out_buffer); formatter->consume(std::move(unit.chunk)); + formatter->flush(); -// std::cout << "Formatter " << current_unit_number << " before notify" << std::endl; + unit.actual_memory_size = out_buffer.getActualSize(); { std::lock_guard lock(mutex); @@ -280,7 +253,6 @@ private: collector_condvar.notify_all(); } -// std::cout << "Formatter " << current_unit_number << " after notify" << std::endl; } catch (...) { From da435046548c38e2e1e989325bf1bff2ebd98e61 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 14 Aug 2020 03:36:03 +0300 Subject: [PATCH 035/284] better --- src/IO/BufferWithOwnMemory.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/IO/BufferWithOwnMemory.h b/src/IO/BufferWithOwnMemory.h index 396f8a36791..26aeeebb24b 100644 --- a/src/IO/BufferWithOwnMemory.h +++ b/src/IO/BufferWithOwnMemory.h @@ -174,7 +174,6 @@ public: private: void nextImpl() override final { - std::cout << "nextImpl" << std::endl; const size_t prev_size = memory.size(); memory.resize(2 * prev_size); Base::set(memory.data() + prev_size, memory.size() - prev_size); From 161c921dba870dd70da0f3a486974be370b930b4 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 14 Aug 2020 16:41:44 +0300 Subject: [PATCH 036/284] fix build --- programs/odbc-bridge/MainHandler.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 78b55bf531e..c456c354dda 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -1,28 +1,29 @@ #include "MainHandler.h" #include "validateODBCConnectionString.h" -#include -#include -#include #include "ODBCBlockInputStream.h" #include "ODBCBlockOutputStream.h" +#include "getIdentifierQuote.h" +#include +#include #include #include #include #include +#include #include #include #include -#include -#include #include -#include -#include -#include "getIdentifierQuote.h" +#include +#include + +#include +#include + #if USE_ODBC #include -#include #define POCO_SQL_ODBC_CLASS Poco::Data::ODBC #endif From 57705f5b734a955e7bb732e4676b77364c8092bf Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Thu, 24 Sep 2020 21:49:18 +0300 Subject: [PATCH 037/284] delete and fix strange code --- base/CMakeLists.txt | 2 +- programs/CMakeLists.txt | 2 +- src/CMakeLists.txt | 2 +- src/Core/Settings.h | 2 +- src/Formats/FormatFactory.cpp | 29 ++----------------- src/Processors/Formats/IOutputFormat.h | 3 -- .../Formats/Impl/IReadBufferPrepareAndEndUp.h | 8 ++--- .../Impl/ParallelFormattingOutputFormat.h | 8 ++--- 8 files changed, 14 insertions(+), 42 deletions(-) diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt index b8a4f144489..46bd57eda12 100644 --- a/base/CMakeLists.txt +++ b/base/CMakeLists.txt @@ -1,5 +1,5 @@ if (USE_CLANG_TIDY) - set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH} -fix") + set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () add_subdirectory (common) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 8ea020a1278..8f45bf53f53 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -1,5 +1,5 @@ if (USE_CLANG_TIDY) - set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH} -fix") + set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () # The `clickhouse` binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a99a4e17956..6021065f937 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,7 +3,7 @@ if (USE_INCLUDE_WHAT_YOU_USE) endif () if (USE_CLANG_TIDY) - set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH} -fix") + set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () if(COMPILER_PIPE) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 5502c692865..0899e5ab729 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -121,7 +121,7 @@ class IColumn; \ M(Bool, input_format_parallel_parsing, true, "Enable parallel parsing for some data formats.", 0) \ M(UInt64, min_chunk_bytes_for_parallel_parsing, (10 * 1024 * 1024), "The minimum chunk size in bytes, which each thread will parse in parallel.", 0) \ - M(Bool, output_format_parallel_formatting, false, "Enable parallel formatting for all data formats.", 0) \ + M(Bool, output_format_parallel_formatting, false, "Enable parallel formatting for all data formats.", 1) \ \ M(UInt64, merge_tree_min_rows_for_concurrent_read, (20 * 8192), "If at least as many lines are read from one file, the reading can be parallelized.", 0) \ M(UInt64, merge_tree_min_bytes_for_concurrent_read, (24 * 10 * 1024 * 1024), "If at least as many bytes are read from one file, the reading can be parallelized.", 0) \ diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index a7eb4d9df4e..7db6a9be152 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -149,17 +149,7 @@ InputFormatPtr FormatFactory::getInput( if (!getCreators(name).input_processor_creator) { - - -// const auto & input_getter = getCreators(name).input_creator; -// if (!input_getter) -// throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); -// -// const Settings & settings = context.getSettingsRef(); -// FormatSettings format_settings = getInputFormatSetting(settings, context); -// -// return input_getter(buf, sample, max_block_size, callback ? callback : ReadCallback(), format_settings); - throw; + throw Exception("Format " + name + " is not suitable for input (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT;; } const Settings & settings = context.getSettingsRef(); @@ -185,8 +175,6 @@ InputFormatPtr FormatFactory::getInput( if (parallel_parsing) { const auto & input_getter = getCreators(name).input_processor_creator; - if (!input_getter) - throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); RowInputFormatParams row_input_format_params; row_input_format_params.max_block_size = max_block_size; @@ -221,16 +209,7 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, if (!getCreators(name).output_processor_creator) { - const auto & output_getter = getCreators(name).output_creator; - if (!output_getter) - throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); - - /** Materialization is needed, because formats can use the functions `IDataType`, - * which only work with full columns. - */ - return std::make_shared( - output_getter(buf, sample, std::move(callback), format_settings), - sample); + throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); } @@ -240,8 +219,6 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, if (parallel_formatting) { const auto & output_getter = getCreators(name).output_processor_creator; - if (!output_getter) - throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); FormatSettings format_settings = getOutputFormatSetting(settings, context); @@ -256,7 +233,7 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, // if (format_settings.enable_streaming) // format->setAutoFlush(); - ParallelFormattingOutputFormat::Params params{buf, sample, formatter_creator}; + ParallelFormattingOutputFormat::Params params{buf, sample, formatter_creator, settings.max_threads}; auto format = std::make_shared(params); return std::make_shared(std::make_shared(format), sample); } diff --git a/src/Processors/Formats/IOutputFormat.h b/src/Processors/Formats/IOutputFormat.h index 8f5d137f08f..67c307df2aa 100644 --- a/src/Processors/Formats/IOutputFormat.h +++ b/src/Processors/Formats/IOutputFormat.h @@ -39,15 +39,12 @@ protected: RowsBeforeLimitCounterPtr rows_before_limit_counter; - friend class ParallelFormattingOutputFormat; - virtual void consume(Chunk) = 0; virtual void consumeTotals(Chunk) {} virtual void consumeExtremes(Chunk) {} virtual void finalize() {} public: - IOutputFormat(const Block & header_, WriteBuffer & out_); Status prepare() override; diff --git a/src/Processors/Formats/Impl/IReadBufferPrepareAndEndUp.h b/src/Processors/Formats/Impl/IReadBufferPrepareAndEndUp.h index 33d8bbf9cd6..6a3b9c53c08 100644 --- a/src/Processors/Formats/Impl/IReadBufferPrepareAndEndUp.h +++ b/src/Processors/Formats/Impl/IReadBufferPrepareAndEndUp.h @@ -23,12 +23,12 @@ public: void prepareReadBuffer(ReadBuffer & buffer) override { /// In this format, BOM at beginning of stream cannot be confused with value, so it is safe to skip it. - skipBOMIfExists(in); + skipBOMIfExists(buffer); - skipWhitespaceIfAny(in); - if (!in.eof() && *in.position() == '[') + skipWhitespaceIfAny(buffer); + if (!buffer.eof() && *buffer.position() == '[') { - ++in.position(); + ++buffer.position(); data_in_square_brackets = true; } } diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 68109f181b5..5f20e247578 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -16,9 +16,6 @@ namespace DB { -const size_t min_chunk_bytes_for_parallel_formatting = 1024; -const size_t max_threads_for_parallel_formatting = 6; - class ParallelFormattingOutputFormat : public IOutputFormat { public: @@ -30,15 +27,16 @@ public: WriteBuffer & out; const Block & header; InternalFormatterCreator internal_formatter_creator; + const size_t max_thread_for_parallel_formatting; }; explicit ParallelFormattingOutputFormat(Params params) : IOutputFormat(params.header, params.out) , internal_formatter_creator(params.internal_formatter_creator) - , pool(max_threads_for_parallel_formatting) + , pool(params.max_threads_for_parallel_formatting) { - processing_units.resize(max_threads_for_parallel_formatting + 2); + processing_units.resize(params.max_threads_for_parallel_formatting + 2); collector_thread = ThreadFromGlobalPool([&] { collectorThreadFunction(); }); } From 8ff072c7027b9ca32cba8cc5a5a562035f9a5068 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Thu, 24 Sep 2020 22:16:32 +0300 Subject: [PATCH 038/284] better --- src/Core/Settings.h | 2 +- src/Formats/FormatFactory.cpp | 2 +- src/Processors/Formats/IOutputFormat.h | 2 ++ src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 0899e5ab729..0d8e728ae49 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -121,7 +121,7 @@ class IColumn; \ M(Bool, input_format_parallel_parsing, true, "Enable parallel parsing for some data formats.", 0) \ M(UInt64, min_chunk_bytes_for_parallel_parsing, (10 * 1024 * 1024), "The minimum chunk size in bytes, which each thread will parse in parallel.", 0) \ - M(Bool, output_format_parallel_formatting, false, "Enable parallel formatting for all data formats.", 1) \ + M(Bool, output_format_parallel_formatting, true, "Enable parallel formatting for all data formats.", 0) \ \ M(UInt64, merge_tree_min_rows_for_concurrent_read, (20 * 8192), "If at least as many lines are read from one file, the reading can be parallelized.", 0) \ M(UInt64, merge_tree_min_bytes_for_concurrent_read, (24 * 10 * 1024 * 1024), "If at least as many bytes are read from one file, the reading can be parallelized.", 0) \ diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 7db6a9be152..d20cf3bea86 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -149,7 +149,7 @@ InputFormatPtr FormatFactory::getInput( if (!getCreators(name).input_processor_creator) { - throw Exception("Format " + name + " is not suitable for input (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT;; + throw Exception("Format " + name + " is not suitable for input (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); } const Settings & settings = context.getSettingsRef(); diff --git a/src/Processors/Formats/IOutputFormat.h b/src/Processors/Formats/IOutputFormat.h index 67c307df2aa..4c2b3f30070 100644 --- a/src/Processors/Formats/IOutputFormat.h +++ b/src/Processors/Formats/IOutputFormat.h @@ -39,6 +39,8 @@ protected: RowsBeforeLimitCounterPtr rows_before_limit_counter; + friend class ParallelFormattingOutputFormat; + virtual void consume(Chunk) = 0; virtual void consumeTotals(Chunk) {} virtual void consumeExtremes(Chunk) {} diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 5f20e247578..5c415900e3f 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -27,7 +27,7 @@ public: WriteBuffer & out; const Block & header; InternalFormatterCreator internal_formatter_creator; - const size_t max_thread_for_parallel_formatting; + const size_t max_threads_for_parallel_formatting; }; explicit ParallelFormattingOutputFormat(Params params) From c541df4e54ad0de1c67dbb457c654bd913e28ec6 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Wed, 30 Sep 2020 21:04:32 +0300 Subject: [PATCH 039/284] better --- src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 5c415900e3f..a3aee37693e 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -171,7 +171,6 @@ private: ///FIXME while(hasChunksToWorkWith()) { - std::cout << writer_unit_number << ' ' << collector_unit_number << std::endl; std::this_thread::sleep_for(std::chrono::milliseconds(100)); } } @@ -207,7 +206,7 @@ private: /// Do main work here. out.write(unit.segment.data(), unit.actual_memory_size); - out.sync(); + // out.sync(); ++collector_unit_number; From 5b7af8aa9c372c9b516ba17dfda84b512c7f9de1 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Mon, 5 Oct 2020 18:59:29 +0300 Subject: [PATCH 040/284] add extremes and totals --- src/IO/BufferWithOwnMemory.h | 8 +- src/IO/WriteBuffer.h | 1 + .../Impl/ParallelFormattingOutputFormat.h | 105 ++++++++++++++---- 3 files changed, 89 insertions(+), 25 deletions(-) diff --git a/src/IO/BufferWithOwnMemory.h b/src/IO/BufferWithOwnMemory.h index 26aeeebb24b..8b5bb552525 100644 --- a/src/IO/BufferWithOwnMemory.h +++ b/src/IO/BufferWithOwnMemory.h @@ -160,9 +160,9 @@ protected: public: explicit BufferWithOutsideMemory(Memory<> & memory_) - : Base(nullptr, 0), memory(memory_) + : Base(memory_.data(), memory_.size()), memory(memory_) { - Base::set(memory.data(), memory.size()); + Base::set(memory.data(), memory.size(), 0); Base::padded = false; } @@ -175,8 +175,8 @@ private: void nextImpl() override final { const size_t prev_size = memory.size(); - memory.resize(2 * prev_size); - Base::set(memory.data() + prev_size, memory.size() - prev_size); + memory.resize(2 * prev_size + 1); + Base::set(memory.data(), memory.size(), prev_size + 1); } }; diff --git a/src/IO/WriteBuffer.h b/src/IO/WriteBuffer.h index 4fdb814849e..53d0dbe09eb 100644 --- a/src/IO/WriteBuffer.h +++ b/src/IO/WriteBuffer.h @@ -27,6 +27,7 @@ namespace ErrorCodes class WriteBuffer : public BufferBase { public: + using BufferBase::set; WriteBuffer(Position ptr, size_t size) : BufferBase(ptr, size, 0) {} void set(Position ptr, size_t size) { BufferBase::set(ptr, size, 0); } diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index a3aee37693e..2117bd2ee9b 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -47,16 +47,56 @@ public: finishAndWait(); } - String getName() const override final { return "ParallelFormattingOutputFormat"; } + String getName() const override { return "ParallelFormattingOutputFormat"; } - void flush() override final + void flush() override { - waitForFormattingFinished(); - IOutputFormat::flush(); + need_flush = true; } protected: void consume(Chunk chunk) override final + { + addChunk(std::move(chunk), ProcessingUnitType::PLAIN); + } + + void consumeTotals(Chunk totals) override + { + addChunk(std::move(totals), ProcessingUnitType::TOTALS); + } + + void consumeExtremes(Chunk extremes) override + { + addChunk(std::move(extremes), ProcessingUnitType::EXTREMES); + } + + void finalize() override + { + IOutputFormat::finalized = true; + addChunk(Chunk{}, ProcessingUnitType::FINALIZE); + } + +private: + + InternalFormatterCreator internal_formatter_creator; + + enum ProcessingUnitStatus + { + READY_TO_INSERT, + READY_TO_FORMAT, + READY_TO_READ + }; + + + enum class ProcessingUnitType + { + PLAIN, + TOTALS, + EXTREMES, + FINALIZE + }; + + void addChunk(Chunk chunk, ProcessingUnitType type) { const auto current_unit_number = writer_unit_number % processing_units.size(); @@ -75,23 +115,13 @@ protected: /// Resize memory without deallocate unit.segment.resize(0); unit.status = READY_TO_FORMAT; + unit.type = type; scheduleFormatterThreadForUnitWithNumber(current_unit_number); ++writer_unit_number; } -private: - - InternalFormatterCreator internal_formatter_creator; - - enum ProcessingUnitStatus - { - READY_TO_INSERT, - READY_TO_FORMAT, - READY_TO_READ - }; - struct ProcessingUnit { explicit ProcessingUnit() @@ -100,11 +130,15 @@ private: } std::atomic status; + ProcessingUnitType type; Chunk chunk; Memory<> segment; size_t actual_memory_size{0}; + }; + std::atomic_bool need_flush{false}; + // There are multiple "formatters", that's why we use thread pool. ThreadPool pool; // Collecting all memory to original ReadBuffer @@ -190,23 +224,29 @@ private: { const auto current_unit_number = collector_unit_number % processing_units.size(); - auto &unit = processing_units[current_unit_number]; + auto & unit = processing_units[current_unit_number]; { std::unique_lock lock(mutex); collector_condvar.wait(lock, - [&]{ return unit.status == READY_TO_READ || formatting_finished; }); + [&]{ return unit.status == READY_TO_READ; }); } - if (formatting_finished) + if (unit.type == ProcessingUnitType::FINALIZE) break; + if (unit.type == ProcessingUnitType::TOTALS) { + + } + assert(unit.status == READY_TO_READ); assert(unit.segment.size() > 0); /// Do main work here. out.write(unit.segment.data(), unit.actual_memory_size); - // out.sync(); + + if (need_flush.exchange(false)) + IOutputFormat::flush(); ++collector_unit_number; @@ -239,9 +279,32 @@ private: auto formatter = internal_formatter_creator(out_buffer); - formatter->consume(std::move(unit.chunk)); - formatter->flush(); + unit.actual_memory_size = 0; + switch (unit.type) + { + case ProcessingUnitType::PLAIN : + { + formatter->consume(std::move(unit.chunk)); + break; + } + case ProcessingUnitType::TOTALS : + { + formatter->consumeTotals(std::move(unit.chunk)); + break; + } + case ProcessingUnitType::EXTREMES : + { + formatter->consumeExtremes(std::move(unit.chunk)); + break; + } + case ProcessingUnitType::FINALIZE : + { + break; + } + } + + formatter->flush(); unit.actual_memory_size = out_buffer.getActualSize(); { From a1010d708f358b88e55dea2ae0b8cadcdb4f28fd Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Mon, 5 Oct 2020 20:50:01 +0300 Subject: [PATCH 041/284] disable PrettySpaceMonoBlock + writePrefix --- src/Formats/FormatFactory.cpp | 2 +- .../Impl/ParallelFormattingOutputFormat.h | 26 ++++++++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index d20cf3bea86..34558e2d260 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -216,7 +216,7 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, const Settings & settings = context.getSettingsRef(); bool parallel_formatting = settings.output_format_parallel_formatting; - if (parallel_formatting) + if (parallel_formatting && name != "PrettyCompactMonoBlock") { const auto & output_getter = getCreators(name).output_processor_creator; diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 2117bd2ee9b..f5775f80be9 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -54,6 +54,11 @@ public: need_flush = true; } + void doWritePrefix() override + { + addChunk(Chunk{}, ProcessingUnitType::START); + } + protected: void consume(Chunk chunk) override final { @@ -90,6 +95,7 @@ private: enum class ProcessingUnitType { + START, PLAIN, TOTALS, EXTREMES, @@ -232,9 +238,6 @@ private: [&]{ return unit.status == READY_TO_READ; }); } - if (unit.type == ProcessingUnitType::FINALIZE) - break; - if (unit.type == ProcessingUnitType::TOTALS) { } @@ -250,9 +253,14 @@ private: ++collector_unit_number; - std::lock_guard lock(mutex); - unit.status = READY_TO_INSERT; - writer_condvar.notify_all(); + { + std::lock_guard lock(mutex); + unit.status = READY_TO_INSERT; + writer_condvar.notify_all(); + } + + if (unit.type == ProcessingUnitType::FINALIZE) + break; } } catch (...) @@ -283,6 +291,11 @@ private: switch (unit.type) { + case ProcessingUnitType::START : + { + formatter->doWritePrefix(); + break; + } case ProcessingUnitType::PLAIN : { formatter->consume(std::move(unit.chunk)); @@ -300,6 +313,7 @@ private: } case ProcessingUnitType::FINALIZE : { + formatter->doWriteSuffix(); break; } } From 4ff1be6e25fafdb41ae6f90a33858b3097a6722d Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Tue, 6 Oct 2020 17:02:01 +0300 Subject: [PATCH 042/284] better --- src/Formats/FormatFactory.cpp | 12 ++++++++- src/Formats/FormatFactory.h | 3 +++ src/IO/BufferWithOwnMemory.h | 2 +- .../Formats/Impl/CSVRowOutputFormat.cpp | 1 + .../JSONCompactEachRowRowOutputFormat.cpp | 4 +++ .../Impl/JSONEachRowRowOutputFormat.cpp | 2 ++ .../Impl/ParallelFormattingOutputFormat.h | 26 ++++++++++++------- .../Formats/Impl/TSKVRowOutputFormat.cpp | 1 + .../Impl/TabSeparatedRowOutputFormat.cpp | 4 +++ .../Formats/Impl/ValuesRowOutputFormat.cpp | 1 + .../01514_parallel_formatting.reference | 6 +++++ .../0_stateless/01514_parallel_formatting.sql | 19 ++++++++++++++ 12 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 tests/queries/0_stateless/01514_parallel_formatting.reference create mode 100644 tests/queries/0_stateless/01514_parallel_formatting.sql diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 34558e2d260..25ea5fc9c06 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -216,7 +216,7 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, const Settings & settings = context.getSettingsRef(); bool parallel_formatting = settings.output_format_parallel_formatting; - if (parallel_formatting && name != "PrettyCompactMonoBlock") + if (parallel_formatting && getCreators(name).supports_parallel_formatting) { const auto & output_getter = getCreators(name).output_processor_creator; @@ -351,6 +351,16 @@ void FormatFactory::registerFileSegmentationEngine(const String & name, FileSegm target = std::move(file_segmentation_engine); } + +void FormatFactory::markOutputFormatSupportsParallelFormatting(const String & name) +{ + auto & target = dict[name].supports_parallel_formatting; + if (target) + throw Exception("FormatFactory: Output format " + name + " is already marked as supporting parallel formatting.", ErrorCodes::LOGICAL_ERROR); + target = true; +} + + FormatFactory & FormatFactory::instance() { static FormatFactory ret; diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index d191597fe25..ddf557b9dfc 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -100,6 +100,7 @@ private: InputProcessorCreator input_processor_creator; OutputProcessorCreator output_processor_creator; FileSegmentationEngine file_segmentation_engine; + bool supports_parallel_formatting{false}; }; using FormatsDictionary = std::unordered_map; @@ -140,6 +141,8 @@ public: void registerInputFormatProcessor(const String & name, InputProcessorCreator input_creator); void registerOutputFormatProcessor(const String & name, OutputProcessorCreator output_creator); + void markOutputFormatSupportsParallelFormatting(const String & name); + const FormatsDictionary & getAllFormats() const { return dict; diff --git a/src/IO/BufferWithOwnMemory.h b/src/IO/BufferWithOwnMemory.h index 8b5bb552525..104c15a4acc 100644 --- a/src/IO/BufferWithOwnMemory.h +++ b/src/IO/BufferWithOwnMemory.h @@ -176,7 +176,7 @@ private: { const size_t prev_size = memory.size(); memory.resize(2 * prev_size + 1); - Base::set(memory.data(), memory.size(), prev_size + 1); + Base::set(memory.data() + prev_size, memory.size() - prev_size, 0); } }; diff --git a/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp b/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp index 2d6a49ccb6f..90fc768d311 100644 --- a/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp @@ -82,6 +82,7 @@ void registerOutputFormatProcessorCSV(FormatFactory & factory) { return std::make_shared(buf, sample, with_names, params, format_settings); }); + factory.markOutputFormatSupportsParallelFormatting(with_names ? "CSVWithNames" : "CSV"); } } diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp index e12ca966a93..6efecc1b978 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp @@ -108,6 +108,7 @@ void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory) { return std::make_shared(buf, sample, params, format_settings, false, false); }); + factory.markOutputFormatSupportsParallelFormatting("JSONCompactEachRow"); factory.registerOutputFormatProcessor("JSONCompactEachRowWithNamesAndTypes", []( WriteBuffer &buf, @@ -117,6 +118,7 @@ void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory) { return std::make_shared(buf, sample, params, format_settings, true, false); }); + factory.markOutputFormatSupportsParallelFormatting("JSONCompactEachRowWithNamesAndTypes"); factory.registerOutputFormatProcessor("JSONCompactStringsEachRow", []( WriteBuffer & buf, @@ -126,6 +128,7 @@ void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory) { return std::make_shared(buf, sample, params, format_settings, false, true); }); + factory.markOutputFormatSupportsParallelFormatting("JSONCompactStringsEachRow"); factory.registerOutputFormatProcessor("JSONCompactStringsEachRowWithNamesAndTypes", []( WriteBuffer &buf, @@ -135,6 +138,7 @@ void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory) { return std::make_shared(buf, sample, params, format_settings, true, true); }); + factory.markOutputFormatSupportsParallelFormatting("JSONCompactStringsEachRowWithNamesAndTypes"); } diff --git a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp index 15d8a843f41..30cd0660682 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp @@ -138,6 +138,7 @@ void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory) return std::make_shared(buf, sample, params, settings); }); + factory.markOutputFormatSupportsParallelFormatting("JSONEachRow"); factory.registerOutputFormatProcessor("JSONStringsEachRow", []( WriteBuffer & buf, @@ -150,6 +151,7 @@ void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory) return std::make_shared(buf, sample, params, settings); }); + factory.markOutputFormatSupportsParallelFormatting("JSONStringEachRow"); } } diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index f5775f80be9..d926ba4e125 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -82,7 +83,6 @@ protected: } private: - InternalFormatterCreator internal_formatter_creator; enum ProcessingUnitStatus @@ -104,6 +104,7 @@ private: void addChunk(Chunk chunk, ProcessingUnitType type) { + // std::cout << "AddChunk of size " << chunk.getNumRows() << std::endl; const auto current_unit_number = writer_unit_number % processing_units.size(); auto & unit = processing_units[current_unit_number]; @@ -119,7 +120,6 @@ private: unit.chunk = std::move(chunk); /// Resize memory without deallocate - unit.segment.resize(0); unit.status = READY_TO_FORMAT; unit.type = type; @@ -143,6 +143,8 @@ private: }; + std::promise finalizator{}; + std::atomic_bool need_flush{false}; // There are multiple "formatters", that's why we use thread pool. @@ -165,6 +167,9 @@ private: void finishAndWait() { + std::future future_finalizator = finalizator.get_future(); + future_finalizator.get(); + formatting_finished = true; { @@ -230,6 +235,8 @@ private: { const auto current_unit_number = collector_unit_number % processing_units.size(); + // std::cout << "collecting " << current_unit_number << std::endl; + auto & unit = processing_units[current_unit_number]; { @@ -238,13 +245,11 @@ private: [&]{ return unit.status == READY_TO_READ; }); } - if (unit.type == ProcessingUnitType::TOTALS) { - - } - assert(unit.status == READY_TO_READ); assert(unit.segment.size() > 0); + auto copy_if_unit_type = unit.type; + /// Do main work here. out.write(unit.segment.data(), unit.actual_memory_size); @@ -259,8 +264,12 @@ private: writer_condvar.notify_all(); } - if (unit.type == ProcessingUnitType::FINALIZE) + if (copy_if_unit_type == ProcessingUnitType::FINALIZE) + { + finalizator.set_value(true); break; + } + } } catch (...) @@ -280,9 +289,8 @@ private: assert(unit.status = READY_TO_FORMAT); - unit.segment.resize(DBMS_DEFAULT_BUFFER_SIZE); + unit.segment.resize(1); - /// TODO: Implement proper nextImpl BufferWithOutsideMemory out_buffer(unit.segment); auto formatter = internal_formatter_creator(out_buffer); diff --git a/src/Processors/Formats/Impl/TSKVRowOutputFormat.cpp b/src/Processors/Formats/Impl/TSKVRowOutputFormat.cpp index d65ce95313e..149ba3f0a2a 100644 --- a/src/Processors/Formats/Impl/TSKVRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/TSKVRowOutputFormat.cpp @@ -49,6 +49,7 @@ void registerOutputFormatProcessorTSKV(FormatFactory & factory) { return std::make_shared(buf, sample, params, settings); }); + factory.markOutputFormatSupportsParallelFormatting("TSKV"); } } diff --git a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp index da8221b11c5..dd3adfa40eb 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp @@ -85,6 +85,7 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory) { return std::make_shared(buf, sample, false, false, params, settings); }); + factory.markOutputFormatSupportsParallelFormatting(name); } for (const auto * name : {"TabSeparatedRaw", "TSVRaw"}) @@ -97,6 +98,7 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory) { return std::make_shared(buf, sample, false, false, params, settings); }); + factory.markOutputFormatSupportsParallelFormatting(name); } for (const auto * name : {"TabSeparatedWithNames", "TSVWithNames"}) @@ -109,6 +111,7 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory) { return std::make_shared(buf, sample, true, false, params, settings); }); + factory.markOutputFormatSupportsParallelFormatting(name); } for (const auto * name : {"TabSeparatedWithNamesAndTypes", "TSVWithNamesAndTypes"}) @@ -121,6 +124,7 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory) { return std::make_shared(buf, sample, true, true, params, settings); }); + factory.markOutputFormatSupportsParallelFormatting(name); } } diff --git a/src/Processors/Formats/Impl/ValuesRowOutputFormat.cpp b/src/Processors/Formats/Impl/ValuesRowOutputFormat.cpp index 7791e1296e0..7f249200404 100644 --- a/src/Processors/Formats/Impl/ValuesRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ValuesRowOutputFormat.cpp @@ -51,6 +51,7 @@ void registerOutputFormatProcessorValues(FormatFactory & factory) { return std::make_shared(buf, sample, params, settings); }); + factory.markOutputFormatSupportsParallelFormatting("Values"); } } diff --git a/tests/queries/0_stateless/01514_parallel_formatting.reference b/tests/queries/0_stateless/01514_parallel_formatting.reference new file mode 100644 index 00000000000..c9bddf62490 --- /dev/null +++ b/tests/queries/0_stateless/01514_parallel_formatting.reference @@ -0,0 +1,6 @@ +10000000 +10000000 +20000000 +20000000 +30000000 +30000000 diff --git a/tests/queries/0_stateless/01514_parallel_formatting.sql b/tests/queries/0_stateless/01514_parallel_formatting.sql new file mode 100644 index 00000000000..90fa9e4b4b8 --- /dev/null +++ b/tests/queries/0_stateless/01514_parallel_formatting.sql @@ -0,0 +1,19 @@ +drop table if exists tsv; +create table tsv(a int, b int default 7) engine File(TSV); + +insert into tsv(a) select number from numbers(10000000); +select '10000000'; +select count() from tsv; + + +insert into tsv(a) select number from numbers(10000000); +select '20000000'; +select count() from tsv; + + +insert into tsv(a) select number from numbers(10000000); +select '30000000'; +select count() from tsv; + + +drop table tsv; From d0bd4e97c972c4c409048c4e4a1f7c116211038f Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 6 Oct 2020 17:07:37 +0300 Subject: [PATCH 043/284] Update src/Formats/JSONEachRowUtils.cpp Co-authored-by: tavplubix --- src/Formats/JSONEachRowUtils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Formats/JSONEachRowUtils.cpp b/src/Formats/JSONEachRowUtils.cpp index 44472a36e16..f5a2da3c307 100644 --- a/src/Formats/JSONEachRowUtils.cpp +++ b/src/Formats/JSONEachRowUtils.cpp @@ -69,7 +69,7 @@ bool fileSegmentationEngineJSONEachRowImpl(ReadBuffer & in, DB::Memory<> & memor } saveUpToPosition(in, memory, pos); - assert(*memory.data() == '{'); + assert(*memory.data() == '{' || memory.size() == 0); return loadAtPosition(in, memory, pos); } From a89d6bc75a1b602f46e84e996444857adaa1c06c Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Tue, 6 Oct 2020 20:49:57 +0300 Subject: [PATCH 044/284] comments + fixes for parsing --- .../Impl/ParallelFormattingOutputFormat.h | 85 ++++++++----------- .../Impl/ParallelParsingInputFormat.cpp | 27 ++++-- .../Formats/Impl/ParallelParsingInputFormat.h | 32 ++++--- 3 files changed, 72 insertions(+), 72 deletions(-) diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index d926ba4e125..431ec00e764 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -20,9 +21,10 @@ namespace DB class ParallelFormattingOutputFormat : public IOutputFormat { public: - /* Used to recreate formatter on every new data piece. */ + /// Used to recreate formatter on every new data piece. using InternalFormatterCreator = std::function; + /// Struct to simplify constructor. struct Params { WriteBuffer & out; @@ -37,14 +39,17 @@ public: , pool(params.max_threads_for_parallel_formatting) { + /// Just heuristic. We need one thread for collecting, one thread for receiving chunks + /// and n threads for formatting. processing_units.resize(params.max_threads_for_parallel_formatting + 2); - collector_thread = ThreadFromGlobalPool([&] { collectorThreadFunction(); }); } ~ParallelFormattingOutputFormat() override { flush(); + if (!IOutputFormat::finalized) + finalize(); finishAndWait(); } @@ -80,11 +85,13 @@ protected: { IOutputFormat::finalized = true; addChunk(Chunk{}, ProcessingUnitType::FINALIZE); + collector_finished.wait(); } private: InternalFormatterCreator internal_formatter_creator; + /// Status to synchronize multiple threads. enum ProcessingUnitStatus { READY_TO_INSERT, @@ -92,7 +99,7 @@ private: READY_TO_READ }; - + /// Some information about what methods to call from internal parser. enum class ProcessingUnitType { START, @@ -104,22 +111,22 @@ private: void addChunk(Chunk chunk, ProcessingUnitType type) { - // std::cout << "AddChunk of size " << chunk.getNumRows() << std::endl; const auto current_unit_number = writer_unit_number % processing_units.size(); - auto & unit = processing_units[current_unit_number]; { std::unique_lock lock(mutex); writer_condvar.wait(lock, - [&]{ return unit.status == READY_TO_INSERT || formatting_finished; }); + [&]{ return unit.status == READY_TO_INSERT || emergency_stop; }); } + if (emergency_stop) + return; + assert(unit.status == READY_TO_INSERT); - unit.chunk = std::move(chunk); - - /// Resize memory without deallocate + /// Resize memory without deallocation. + unit.segment.resize(0); unit.status = READY_TO_FORMAT; unit.type = type; @@ -140,10 +147,9 @@ private: Chunk chunk; Memory<> segment; size_t actual_memory_size{0}; - }; - std::promise finalizator{}; + Poco::Event collector_finished{}; std::atomic_bool need_flush{false}; @@ -154,10 +160,11 @@ private: std::exception_ptr background_exception = nullptr; + /// We use deque, because ProcessingUnit doesn't have move or copy constructor. std::deque processing_units; std::mutex mutex; - std::atomic_bool formatting_finished{false}; + std::atomic_bool emergency_stop{false}; std::atomic_size_t collector_unit_number{0}; std::atomic_size_t writer_unit_number{0}; @@ -167,10 +174,7 @@ private: void finishAndWait() { - std::future future_finalizator = finalizator.get_future(); - future_finalizator.get(); - - formatting_finished = true; + emergency_stop = true; { std::unique_lock lock(mutex); @@ -201,7 +205,7 @@ private: { background_exception = std::current_exception(); } - formatting_finished = true; + emergency_stop = true; writer_condvar.notify_all(); collector_condvar.notify_all(); } @@ -211,43 +215,29 @@ private: pool.scheduleOrThrowOnError([this, ticket_number] { formatterThreadFunction(ticket_number); }); } - void waitForFormattingFinished() - { - ///FIXME - while(hasChunksToWorkWith()) - { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - } - - bool hasChunksToWorkWith() - { - return writer_unit_number - collector_unit_number > 0; - } - void collectorThreadFunction() { setThreadName("Collector"); try { - while (!formatting_finished) + while (!emergency_stop) { const auto current_unit_number = collector_unit_number % processing_units.size(); - - // std::cout << "collecting " << current_unit_number << std::endl; - auto & unit = processing_units[current_unit_number]; { std::unique_lock lock(mutex); collector_condvar.wait(lock, - [&]{ return unit.status == READY_TO_READ; }); + [&]{ return unit.status == READY_TO_READ || emergency_stop; }); } - assert(unit.status == READY_TO_READ); - assert(unit.segment.size() > 0); + if (emergency_stop) + return; + assert(unit.status == READY_TO_READ); + + /// Use this copy to after notification to stop the execution. auto copy_if_unit_type = unit.type; /// Do main work here. @@ -257,19 +247,19 @@ private: IOutputFormat::flush(); ++collector_unit_number; - + { + /// Notify other threads. std::lock_guard lock(mutex); unit.status = READY_TO_INSERT; writer_condvar.notify_all(); } - + /// We can exit only after writing last piece of to out buffer. if (copy_if_unit_type == ProcessingUnitType::FINALIZE) { - finalizator.set_value(true); + collector_finished.set(); break; } - } } catch (...) @@ -286,17 +276,14 @@ private: try { auto & unit = processing_units[current_unit_number]; - assert(unit.status = READY_TO_FORMAT); - unit.segment.resize(1); - + unit.segment.resize(0); + unit.actual_memory_size = 0; BufferWithOutsideMemory out_buffer(unit.segment); auto formatter = internal_formatter_creator(out_buffer); - unit.actual_memory_size = 0; - switch (unit.type) { case ProcessingUnitType::START : @@ -325,7 +312,7 @@ private: break; } } - + /// Flush all the data to handmade buffer. formatter->flush(); unit.actual_memory_size = out_buffer.getActualSize(); @@ -334,13 +321,11 @@ private: unit.status = READY_TO_READ; collector_condvar.notify_all(); } - } catch (...) { onBackgroundException(); } - } }; diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 898c103cb24..367b1c748f5 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -1,11 +1,21 @@ #include #include +#include +#include +#include namespace DB { -void ParallelParsingInputFormat::segmentatorThreadFunction() +void ParallelParsingInputFormat::segmentatorThreadFunction(ThreadGroupStatusPtr thread_group) { + SCOPE_EXIT( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); + if (thread_group) + CurrentThread::attachTo(thread_group); + setThreadName("Segmentator"); try { @@ -21,9 +31,7 @@ void ParallelParsingInputFormat::segmentatorThreadFunction() } if (parsing_finished) - { break; - } assert(unit.status == READY_TO_INSERT); @@ -38,9 +46,7 @@ void ParallelParsingInputFormat::segmentatorThreadFunction() ++segmentator_ticket_number; if (!have_more_data) - { break; - } } } catch (...) @@ -49,8 +55,15 @@ void ParallelParsingInputFormat::segmentatorThreadFunction() } } -void ParallelParsingInputFormat::parserThreadFunction(size_t current_ticket_number) +void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr thread_group, size_t current_ticket_number) { + SCOPE_EXIT( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); + if (thread_group) + CurrentThread::attachTo(thread_group); + try { setThreadName("ChunkParser"); @@ -58,8 +71,6 @@ void ParallelParsingInputFormat::parserThreadFunction(size_t current_ticket_numb const auto current_unit_number = current_ticket_number % processing_units.size(); auto & unit = processing_units[current_unit_number]; - assert(unit.segment.size() > 0); - /* * This is kind of suspicious -- the input_process_creator contract with * respect to multithreaded use is not clear, but we hope that it is diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index 35cf7047d3c..f8a85536158 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -84,7 +84,8 @@ public: initializePrepareEndUpMap(); - segmentator_thread = ThreadFromGlobalPool([this] { segmentatorThreadFunction(); }); + segmentator_thread = ThreadFromGlobalPool( + &ParallelParsingInputFormat::segmentatorThreadFunction, this, CurrentThread::getGroup()); } ~ParallelParsingInputFormat() override @@ -172,14 +173,14 @@ private: }; const InternalParserCreator internal_parser_creator; - // Function to segment the file. Then "parsers" will parse that segments. + /// Function to segment the file. Then "parsers" will parse that segments. FormatFactory::FileSegmentationEngine file_segmentation_engine; const String format_name; const size_t min_chunk_bytes; BlockMissingValues last_block_missing_values; - //Non-atomic because it is used in one thread. + /// Non-atomic because it is used in one thread. std::optional next_block_in_current_unit; size_t segmentator_ticket_number{0}; size_t reader_ticket_number{0}; @@ -190,9 +191,9 @@ private: std::atomic parsing_finished{false}; - // There are multiple "parsers", that's why we use thread pool. + /// There are multiple "parsers", that's why we use thread pool. ThreadPool pool; - // Reading and segmentating the file + /// Reading and segmentating the file ThreadFromGlobalPool segmentator_thread; enum ProcessingUnitStatus @@ -223,14 +224,17 @@ private: std::exception_ptr background_exception = nullptr; - // We use deque instead of vector, because it does not require a move - // constructor, which is absent for atomics that are inside ProcessingUnit. + /// We use deque instead of vector, because it does not require a move + /// constructor, which is absent for atomics that are inside ProcessingUnit. std::deque processing_units; void scheduleParserThreadForUnitWithNumber(size_t ticket_number) { - pool.scheduleOrThrowOnError([this, ticket_number] { parserThreadFunction(ticket_number); }); + pool.scheduleOrThrowOnError([this, ticket_number, group = CurrentThread::getGroup()]() + { + parserThreadFunction(group, ticket_number); + }); } void finishAndWait() @@ -256,13 +260,13 @@ private: } } - void segmentatorThreadFunction(); - void parserThreadFunction(size_t current_ticket_number); + void segmentatorThreadFunction(ThreadGroupStatusPtr thread_group); + void parserThreadFunction(ThreadGroupStatusPtr thread_group, size_t current_ticket_number); - // Save/log a background exception, set termination flag, wake up all - // threads. This function is used by segmentator and parsed threads. - // readImpl() is called from the main thread, so the exception handling - // is different. + /// Save/log a background exception, set termination flag, wake up all + /// threads. This function is used by segmentator and parsed threads. + /// readImpl() is called from the main thread, so the exception handling + /// is different. void onBackgroundException(); /// To store objects which will prepare and end up ReadBuffer for each format. From 7dbf71cf2307eca7547b4f02c003bd36fc905259 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Tue, 6 Oct 2020 20:50:50 +0300 Subject: [PATCH 045/284] delete stream parsing --- .../ParallelParsingBlockInputStream.cpp | 306 ------------------ .../ParallelParsingBlockInputStream.h | 174 ---------- 2 files changed, 480 deletions(-) delete mode 100644 src/DataStreams/ParallelParsingBlockInputStream.cpp delete mode 100644 src/DataStreams/ParallelParsingBlockInputStream.h diff --git a/src/DataStreams/ParallelParsingBlockInputStream.cpp b/src/DataStreams/ParallelParsingBlockInputStream.cpp deleted file mode 100644 index 8ec1f2bc499..00000000000 --- a/src/DataStreams/ParallelParsingBlockInputStream.cpp +++ /dev/null @@ -1,306 +0,0 @@ -#include -#include -#include -#include -#include - -namespace DB -{ - -ParallelParsingBlockInputStream::ParallelParsingBlockInputStream(const Params & params) - : header(params.input_creator_params.sample), - row_input_format_params(params.input_creator_params.row_input_format_params), - format_settings(params.input_creator_params.settings), - input_processor_creator(params.input_processor_creator), - min_chunk_bytes(params.min_chunk_bytes), - original_buffer(params.read_buffer), - // Subtract one thread that we use for segmentation and one for - // reading. After that, must have at least two threads left for - // parsing. See the assertion below. - pool(std::max(2, static_cast(params.max_threads) - 2)), - file_segmentation_engine(params.file_segmentation_engine) -{ - // See comment above. - assert(params.max_threads >= 4); - - // One unit for each thread, including segmentator and reader, plus a - // couple more units so that the segmentation thread doesn't spuriously - // bump into reader thread on wraparound. - processing_units.resize(params.max_threads + 2); - - segmentator_thread = ThreadFromGlobalPool( - &ParallelParsingBlockInputStream::segmentatorThreadFunction, this, CurrentThread::getGroup()); -} - -ParallelParsingBlockInputStream::~ParallelParsingBlockInputStream() -{ - finishAndWait(); -} - -void ParallelParsingBlockInputStream::cancel(bool kill) -{ - /** - * Can be called multiple times, from different threads. Saturate the - * the kill flag with OR. - */ - if (kill) - is_killed = true; - is_cancelled = true; - - /* - * The format parsers themselves are not being cancelled here, so we'll - * have to wait until they process the current block. Given that the - * chunk size is on the order of megabytes, this shouldn't be too long. - * We can't call IInputFormat->cancel here, because the parser object is - * local to the parser thread, and we don't want to introduce any - * synchronization between parser threads and the other threads to get - * better performance. An ideal solution would be to add a callback to - * IInputFormat that checks whether it was cancelled. - */ - - finishAndWait(); -} - -void ParallelParsingBlockInputStream::scheduleParserThreadForUnitWithNumber(size_t ticket_number) -{ - pool.scheduleOrThrowOnError([this, ticket_number, group = CurrentThread::getGroup()]() - { - parserThreadFunction(group, ticket_number); - }); -} - -void ParallelParsingBlockInputStream::finishAndWait() -{ - finished = true; - - { - std::unique_lock lock(mutex); - segmentator_condvar.notify_all(); - reader_condvar.notify_all(); - } - - if (segmentator_thread.joinable()) - segmentator_thread.join(); - - try - { - pool.wait(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } -} - -void ParallelParsingBlockInputStream::segmentatorThreadFunction(ThreadGroupStatusPtr thread_group) -{ - SCOPE_EXIT( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); - if (thread_group) - CurrentThread::attachTo(thread_group); - - setThreadName("Segmentator"); - - try - { - while (!finished) - { - const auto current_unit_number = segmentator_ticket_number % processing_units.size(); - auto & unit = processing_units[current_unit_number]; - - { - std::unique_lock lock(mutex); - segmentator_condvar.wait(lock, - [&]{ return unit.status == READY_TO_INSERT || finished; }); - } - - if (finished) - { - break; - } - - assert(unit.status == READY_TO_INSERT); - - // Segmentating the original input. - unit.segment.resize(0); - - const bool have_more_data = file_segmentation_engine(original_buffer, - unit.segment, min_chunk_bytes); - - unit.is_last = !have_more_data; - unit.status = READY_TO_PARSE; - scheduleParserThreadForUnitWithNumber(segmentator_ticket_number); - ++segmentator_ticket_number; - - if (!have_more_data) - { - break; - } - } - } - catch (...) - { - onBackgroundException(); - } -} - -void ParallelParsingBlockInputStream::parserThreadFunction(ThreadGroupStatusPtr thread_group, size_t current_ticket_number) -{ - SCOPE_EXIT( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); - if (thread_group) - CurrentThread::attachTo(thread_group); - - setThreadName("ChunkParser"); - - try - { - const auto current_unit_number = current_ticket_number % processing_units.size(); - auto & unit = processing_units[current_unit_number]; - - /* - * This is kind of suspicious -- the input_process_creator contract with - * respect to multithreaded use is not clear, but we hope that it is - * just a 'normal' factory class that doesn't have any state, and so we - * can use it from multiple threads simultaneously. - */ - ReadBuffer read_buffer(unit.segment.data(), unit.segment.size(), 0); - auto format = input_processor_creator(read_buffer, header, row_input_format_params, format_settings); - auto parser = std::make_unique(std::move(format)); - - unit.block_ext.block.clear(); - unit.block_ext.block_missing_values.clear(); - - // We don't know how many blocks will be. So we have to read them all - // until an empty block occurred. - Block block; - while (!finished && (block = parser->read()) != Block()) - { - unit.block_ext.block.emplace_back(block); - unit.block_ext.block_missing_values.emplace_back(parser->getMissingValues()); - } - - // We suppose we will get at least some blocks for a non-empty buffer, - // except at the end of file. Also see a matching assert in readImpl(). - assert(unit.is_last || !unit.block_ext.block.empty()); - - std::unique_lock lock(mutex); - unit.status = READY_TO_READ; - reader_condvar.notify_all(); - } - catch (...) - { - onBackgroundException(); - } -} - -void ParallelParsingBlockInputStream::onBackgroundException() -{ - tryLogCurrentException(__PRETTY_FUNCTION__); - - std::unique_lock lock(mutex); - if (!background_exception) - { - background_exception = std::current_exception(); - } - finished = true; - reader_condvar.notify_all(); - segmentator_condvar.notify_all(); -} - -Block ParallelParsingBlockInputStream::readImpl() -{ - if (isCancelledOrThrowIfKilled() || finished) - { - /** - * Check for background exception and rethrow it before we return. - */ - std::unique_lock lock(mutex); - if (background_exception) - { - lock.unlock(); - cancel(false); - std::rethrow_exception(background_exception); - } - - return Block{}; - } - - const auto current_unit_number = reader_ticket_number % processing_units.size(); - auto & unit = processing_units[current_unit_number]; - - if (!next_block_in_current_unit.has_value()) - { - // We have read out all the Blocks from the previous Processing Unit, - // wait for the current one to become ready. - std::unique_lock lock(mutex); - reader_condvar.wait(lock, [&](){ return unit.status == READY_TO_READ || finished; }); - - if (finished) - { - /** - * Check for background exception and rethrow it before we return. - */ - if (background_exception) - { - lock.unlock(); - cancel(false); - std::rethrow_exception(background_exception); - } - - return Block{}; - } - - assert(unit.status == READY_TO_READ); - next_block_in_current_unit = 0; - } - - if (unit.block_ext.block.empty()) - { - /* - * Can we get zero blocks for an entire segment, when the format parser - * skips it entire content and does not create any blocks? Probably not, - * but if we ever do, we should add a loop around the above if, to skip - * these. Also see a matching assert in the parser thread. - */ - assert(unit.is_last); - finished = true; - return Block{}; - } - - assert(next_block_in_current_unit.value() < unit.block_ext.block.size()); - - Block res = std::move(unit.block_ext.block.at(*next_block_in_current_unit)); - last_block_missing_values = std::move(unit.block_ext.block_missing_values[*next_block_in_current_unit]); - - next_block_in_current_unit.value() += 1; - - if (*next_block_in_current_unit == unit.block_ext.block.size()) - { - // Finished reading this Processing Unit, move to the next one. - next_block_in_current_unit.reset(); - ++reader_ticket_number; - - if (unit.is_last) - { - // It it was the last unit, we're finished. - finished = true; - } - else - { - // Pass the unit back to the segmentator. - std::unique_lock lock(mutex); - unit.status = READY_TO_INSERT; - segmentator_condvar.notify_all(); - } - } - - return res; -} - - -} diff --git a/src/DataStreams/ParallelParsingBlockInputStream.h b/src/DataStreams/ParallelParsingBlockInputStream.h deleted file mode 100644 index 147c86a4cf7..00000000000 --- a/src/DataStreams/ParallelParsingBlockInputStream.h +++ /dev/null @@ -1,174 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -namespace DB -{ - -class ReadBuffer; - -/** - * ORDER-PRESERVING parallel parsing of data formats. - * It splits original data into chunks. Then each chunk is parsed by different thread. - * The number of chunks equals to the number or parser threads. - * The size of chunk is equal to min_chunk_bytes_for_parallel_parsing setting. - * - * This stream has three kinds of threads: one segmentator, multiple parsers, - * and one reader thread -- that is, the one from which readImpl() is called. - * They operate one after another on parts of data called "processing units". - * One unit consists of buffer with raw data from file, filled by segmentator - * thread. This raw data is then parsed by a parser thread to form a number of - * Blocks. These Blocks are returned to the parent stream from readImpl(). - * After being read out, a processing unit is reused, to save on allocating - * memory for the raw buffer. The processing units are organized into a circular - * array to facilitate reuse and to apply backpressure on the segmentator thread - * -- after it runs out of processing units, it has to wait for the reader to - * read out the previous blocks. - * The outline of what the threads do is as follows: - * segmentator thread: - * 1) wait for the next processing unit to become empty - * 2) fill it with a part of input file - * 3) start a parser thread - * 4) repeat until eof - * parser thread: - * 1) parse the given raw buffer without any synchronization - * 2) signal that the given unit is ready to read - * 3) finish - * readImpl(): - * 1) wait for the next processing unit to become ready to read - * 2) take the blocks from the processing unit to return them to the caller - * 3) signal that the processing unit is empty - * 4) repeat until it encounters unit that is marked as "past_the_end" - * All threads must also check for cancel/eof/exception flags. - */ -class ParallelParsingBlockInputStream : public IBlockInputStream -{ -private: - using ReadCallback = std::function; - - using InputProcessorCreator = std::function; -public: - struct InputCreatorParams - { - const Block & sample; - const RowInputFormatParams & row_input_format_params; - const FormatSettings &settings; - }; - - struct Params - { - ReadBuffer & read_buffer; - const InputProcessorCreator & input_processor_creator; - const InputCreatorParams & input_creator_params; - FormatFactory::FileSegmentationEngine file_segmentation_engine; - size_t max_threads; - size_t min_chunk_bytes; - }; - - explicit ParallelParsingBlockInputStream(const Params & params); - ~ParallelParsingBlockInputStream() override; - - String getName() const override { return "ParallelParsing"; } - Block getHeader() const override { return header; } - - void cancel(bool kill) override; - -protected: - // Reader routine - Block readImpl() override; - - const BlockMissingValues & getMissingValues() const override - { - return last_block_missing_values; - } - -private: - const Block header; - const RowInputFormatParams row_input_format_params; - const FormatSettings format_settings; - const InputProcessorCreator input_processor_creator; - - const size_t min_chunk_bytes; - - /* - * This is declared as atomic to avoid UB, because parser threads access it - * without synchronization. - */ - std::atomic finished{false}; - - BlockMissingValues last_block_missing_values; - - // Original ReadBuffer to read from. - ReadBuffer & original_buffer; - - //Non-atomic because it is used in one thread. - std::optional next_block_in_current_unit; - size_t segmentator_ticket_number{0}; - size_t reader_ticket_number{0}; - - std::mutex mutex; - std::condition_variable reader_condvar; - std::condition_variable segmentator_condvar; - - // There are multiple "parsers", that's why we use thread pool. - ThreadPool pool; - // Reading and segmentating the file - ThreadFromGlobalPool segmentator_thread; - - // Function to segment the file. Then "parsers" will parse that segments. - FormatFactory::FileSegmentationEngine file_segmentation_engine; - - enum ProcessingUnitStatus - { - READY_TO_INSERT, - READY_TO_PARSE, - READY_TO_READ - }; - - struct BlockExt - { - std::vector block; - std::vector block_missing_values; - }; - - struct ProcessingUnit - { - explicit ProcessingUnit() - : status(ProcessingUnitStatus::READY_TO_INSERT) - { - } - - BlockExt block_ext; - Memory<> segment; - std::atomic status; - bool is_last{false}; - }; - - std::exception_ptr background_exception = nullptr; - - // We use deque instead of vector, because it does not require a move - // constructor, which is absent for atomics that are inside ProcessingUnit. - std::deque processing_units; - - void scheduleParserThreadForUnitWithNumber(size_t ticket_number); - void finishAndWait(); - - void segmentatorThreadFunction(ThreadGroupStatusPtr thread_group); - void parserThreadFunction(ThreadGroupStatusPtr thread_group, size_t current_ticket_number); - - // Save/log a background exception, set termination flag, wake up all - // threads. This function is used by segmentator and parsed threads. - // readImpl() is called from the main thread, so the exception handling - // is different. - void onBackgroundException(); -}; - -} From 861b3c0fc84447a9e9641f83bc34155688258372 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Tue, 6 Oct 2020 21:23:45 +0300 Subject: [PATCH 046/284] cmake --- utils/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 680554be57d..a27a7e9dadc 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,5 +1,5 @@ if (USE_CLANG_TIDY) - set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH} -fix") + set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () if(MAKE_STATIC_LIBRARIES) From 746b7d0e131cb31af250f90a0cbea9cf8206498d Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Tue, 6 Oct 2020 22:01:55 +0300 Subject: [PATCH 047/284] better test + fix writePrefix in JSONCompact --- .../Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp | 2 +- src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h | 2 +- tests/queries/0_stateless/01514_parallel_formatting.sql | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp index 6efecc1b978..11134499984 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp @@ -68,7 +68,7 @@ void JSONCompactEachRowRowOutputFormat::writeTotals(const Columns & columns, siz writeCString("]\n", out); } -void JSONCompactEachRowRowOutputFormat::writePrefix() +void JSONCompactEachRowRowOutputFormat::doWritePrefix() { if (with_names) { diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h index eb426bec15d..3d4b80247b8 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h @@ -25,7 +25,7 @@ public: String getName() const override { return "JSONCompactEachRowRowOutputFormat"; } - void writePrefix() override; + void doWritePrefix() override; void writeBeforeTotals() override {} void writeTotals(const Columns & columns, size_t row_num) override; diff --git a/tests/queries/0_stateless/01514_parallel_formatting.sql b/tests/queries/0_stateless/01514_parallel_formatting.sql index 90fa9e4b4b8..95a9e19aa1f 100644 --- a/tests/queries/0_stateless/01514_parallel_formatting.sql +++ b/tests/queries/0_stateless/01514_parallel_formatting.sql @@ -1,4 +1,5 @@ drop table if exists tsv; +set output_format_parallel_formatting=1; create table tsv(a int, b int default 7) engine File(TSV); insert into tsv(a) select number from numbers(10000000); From 81be9d77da9e2168d7ba3b36e77befb25d45366a Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Tue, 6 Oct 2020 22:07:27 +0300 Subject: [PATCH 048/284] pvs check --- .../Formats/Impl/ParallelFormattingOutputFormat.h | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 431ec00e764..8697575c302 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -47,7 +47,7 @@ public: ~ParallelFormattingOutputFormat() override { - flush(); + need_flush = true; if (!IOutputFormat::finalized) finalize(); finishAndWait(); @@ -137,13 +137,8 @@ private: struct ProcessingUnit { - explicit ProcessingUnit() - : status(ProcessingUnitStatus::READY_TO_INSERT) - { - } - - std::atomic status; - ProcessingUnitType type; + std::atomic status{ProcessingUnitStatus::READY_TO_INSERT}; + ProcessingUnitType type{ProcessingUnitType::START}; Chunk chunk; Memory<> segment; size_t actual_memory_size{0}; From 1bdfc63ef3e0e3948d8a469d6611471a4f254272 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Wed, 7 Oct 2020 21:16:58 +0300 Subject: [PATCH 049/284] delete PrepareAndEndUpReadBuffer --- src/Formats/FormatFactory.cpp | 2 + src/Formats/JSONEachRowUtils.cpp | 8 --- .../Formats/Impl/IReadBufferPrepareAndEndUp.h | 56 ------------------- .../Impl/JSONEachRowRowInputFormat.cpp | 31 ++++++++-- .../Formats/Impl/JSONEachRowRowInputFormat.h | 5 -- .../Impl/ParallelParsingInputFormat.cpp | 18 ------ .../Formats/Impl/ParallelParsingInputFormat.h | 16 ------ 7 files changed, 28 insertions(+), 108 deletions(-) delete mode 100644 src/Processors/Formats/Impl/IReadBufferPrepareAndEndUp.h diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 25ea5fc9c06..4af9e9df260 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -19,6 +19,8 @@ #include #include +#include + #if !defined(ARCADIA_BUILD) # include #endif diff --git a/src/Formats/JSONEachRowUtils.cpp b/src/Formats/JSONEachRowUtils.cpp index f5a2da3c307..1df87ebd3fa 100644 --- a/src/Formats/JSONEachRowUtils.cpp +++ b/src/Formats/JSONEachRowUtils.cpp @@ -10,13 +10,6 @@ bool fileSegmentationEngineJSONEachRowImpl(ReadBuffer & in, DB::Memory<> & memor char * pos = in.position(); - /// In case that independent JSONs are splitted by comma we skip that comma. - if (pos && *pos == ',') - { - ++in.position(); - ++pos; - } - size_t balance = 0; bool quotes = false; @@ -69,7 +62,6 @@ bool fileSegmentationEngineJSONEachRowImpl(ReadBuffer & in, DB::Memory<> & memor } saveUpToPosition(in, memory, pos); - assert(*memory.data() == '{' || memory.size() == 0); return loadAtPosition(in, memory, pos); } diff --git a/src/Processors/Formats/Impl/IReadBufferPrepareAndEndUp.h b/src/Processors/Formats/Impl/IReadBufferPrepareAndEndUp.h deleted file mode 100644 index 6a3b9c53c08..00000000000 --- a/src/Processors/Formats/Impl/IReadBufferPrepareAndEndUp.h +++ /dev/null @@ -1,56 +0,0 @@ -#pragma once - -#include - -namespace DB -{ - -class IReadBufferPrepareAndEndUp -{ -public: - virtual ~IReadBufferPrepareAndEndUp() {} - - virtual void prepareReadBuffer(ReadBuffer & buffer) = 0; - - virtual void endUpReadBuffer(ReadBuffer & buffer) = 0; -}; - -using IReadBufferPrepareAndEndUpPtr = std::shared_ptr; - -class JSONEachRowPrepareAndEndUp : public IReadBufferPrepareAndEndUp -{ -public: - void prepareReadBuffer(ReadBuffer & buffer) override - { - /// In this format, BOM at beginning of stream cannot be confused with value, so it is safe to skip it. - skipBOMIfExists(buffer); - - skipWhitespaceIfAny(buffer); - if (!buffer.eof() && *buffer.position() == '[') - { - ++buffer.position(); - data_in_square_brackets = true; - } - } - - void endUpReadBuffer(ReadBuffer & buffer) override - { - skipWhitespaceIfAny(buffer); - if (data_in_square_brackets) - { - assertChar(']', buffer); - skipWhitespaceIfAny(buffer); - } - if (!buffer.eof() && *buffer.position() == ';') - { - ++buffer.position(); - skipWhitespaceIfAny(buffer); - } - assertEOF(buffer); - } - -private: - bool data_in_square_brackets{false}; -}; - -} diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp index ec1deddc94b..1fa03cf348c 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp @@ -245,14 +245,16 @@ bool JSONEachRowRowInputFormat::readRow(MutableColumns & columns, RowReadExtensi /// then seeking to next ;, or \n would trigger reading of an extra row at the end. /// Semicolon is added for convenience as it could be used at end of INSERT query. - if (getTotalRows() && !in.eof()) + bool is_first_row = getTotalRows() == 1; + if (!in.eof()) { - if (*in.position() == ',') + /// There may be optional ',' (but not before the first row) + if (!is_first_row && *in.position() == ',') ++in.position(); else if (!data_in_square_brackets && *in.position() == ';') { - /// ';' means the end of query (but it cannot be before ']') return allow_new_rows = false; + /// ';' means the end of query (but it cannot be before ']') } else if (data_in_square_brackets && *in.position() == ']') { @@ -301,12 +303,31 @@ void JSONEachRowRowInputFormat::resetParser() void JSONEachRowRowInputFormat::readPrefix() { - prepare_and_end_up.prepareReadBuffer(in); + /// In this format, BOM at beginning of stream cannot be confused with value, so it is safe to skip it. + skipBOMIfExists(in); + + skipWhitespaceIfAny(in); + if (!in.eof() && *in.position() == '[') + { + ++in.position(); + data_in_square_brackets = true; + } } void JSONEachRowRowInputFormat::readSuffix() { - prepare_and_end_up.endUpReadBuffer(in); + skipWhitespaceIfAny(in); + if (data_in_square_brackets) + { + assertChar(']', in); + skipWhitespaceIfAny(in); + } + if (!in.eof() && *in.position() == ';') + { + ++in.position(); + skipWhitespaceIfAny(in); + } + assertEOF(in); } diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h index e121d77d7a6..4d6c92077b2 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h @@ -4,7 +4,6 @@ #include #include #include -#include "IReadBufferPrepareAndEndUp.h" namespace DB { @@ -81,11 +80,7 @@ private: bool allow_new_rows = true; - bool yield_strings; - - /// Used when readSuffix() or readPrefix() are called. - JSONEachRowPrepareAndEndUp prepare_and_end_up; }; } diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 367b1c748f5..2c4a567b16f 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -79,9 +79,6 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa */ ReadBuffer read_buffer(unit.segment.data(), unit.segment.size(), 0); - if (current_ticket_number == 0) - prepareReadBuffer(read_buffer); - InputFormatPtr input_format = internal_parser_creator(read_buffer); InternalParser parser(input_format); @@ -101,9 +98,6 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa // except at the end of file. Also see a matching assert in readImpl(). assert(unit.is_last || !unit.chunk_ext.chunk.empty() || parsing_finished); - if (unit.is_last) - endUpReadBuffer(read_buffer); - std::lock_guard lock(mutex); unit.status = READY_TO_READ; reader_condvar.notify_all(); @@ -219,17 +213,5 @@ Chunk ParallelParsingInputFormat::generate() return res; } -void ParallelParsingInputFormat::prepareReadBuffer(ReadBuffer & buffer) -{ - if (prepare_and_end_up_map.count(format_name)) - prepare_and_end_up_map[format_name]->prepareReadBuffer(buffer); -} - - -void ParallelParsingInputFormat::endUpReadBuffer(ReadBuffer & buffer) -{ - if (prepare_and_end_up_map.count(format_name)) - prepare_and_end_up_map[format_name]->endUpReadBuffer(buffer); -} } diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index f8a85536158..e53833764dd 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -9,7 +9,6 @@ #include #include #include -#include "IReadBufferPrepareAndEndUp.h" namespace DB { @@ -82,8 +81,6 @@ public: // bump into reader thread on wraparound. processing_units.resize(params.max_threads + 2); - initializePrepareEndUpMap(); - segmentator_thread = ThreadFromGlobalPool( &ParallelParsingInputFormat::segmentatorThreadFunction, this, CurrentThread::getGroup()); } @@ -268,19 +265,6 @@ private: /// readImpl() is called from the main thread, so the exception handling /// is different. void onBackgroundException(); - - /// To store objects which will prepare and end up ReadBuffer for each format. - std::unordered_map prepare_and_end_up_map; - - void initializePrepareEndUpMap() - { - /// To skip '[' and ']'. - prepare_and_end_up_map.insert({"JSONEachRow", std::make_shared()}); - } - - void prepareReadBuffer(ReadBuffer & buffer); - - void endUpReadBuffer(ReadBuffer & buffer); }; } From 48c76613bf5cc685b42f8a06095fa68c65e9ede2 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Thu, 8 Oct 2020 19:06:14 +0300 Subject: [PATCH 050/284] better --- src/Formats/FormatFactory.cpp | 11 ++++---- .../Impl/ParallelFormattingOutputFormat.h | 25 ++++++++++++++----- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 4af9e9df260..857d2194c20 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -218,7 +218,7 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, const Settings & settings = context.getSettingsRef(); bool parallel_formatting = settings.output_format_parallel_formatting; - if (parallel_formatting && getCreators(name).supports_parallel_formatting) + if (parallel_formatting && getCreators(name).supports_parallel_formatting && !settings.allow_experimental_live_view) { const auto & output_getter = getCreators(name).output_processor_creator; @@ -231,12 +231,13 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, (WriteBuffer & output) -> OutputFormatPtr { return output_getter(output, sample, std::move(callback), format_settings);}; - /// Enable auto-flush for streaming mode. Currently it is needed by INSERT WATCH query. -// if (format_settings.enable_streaming) -// format->setAutoFlush(); - ParallelFormattingOutputFormat::Params params{buf, sample, formatter_creator, settings.max_threads}; auto format = std::make_shared(params); + + /// Enable auto-flush for streaming mode. Currently it is needed by INSERT WATCH query. + if (format_settings.enable_streaming) + format->setAutoFlush(); + return std::make_shared(std::make_shared(format), sample); } diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 8697575c302..2287bec1ab3 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -15,6 +15,9 @@ #include #include +#include +#include + namespace DB { @@ -49,7 +52,7 @@ public: { need_flush = true; if (!IOutputFormat::finalized) - finalize(); + std::terminate(); finishAndWait(); } @@ -65,6 +68,12 @@ public: addChunk(Chunk{}, ProcessingUnitType::START); } + void onCancel() override + { + std::cout << "onCancel" << std::endl; + finishAndWait(); + } + protected: void consume(Chunk chunk) override final { @@ -86,6 +95,8 @@ protected: IOutputFormat::finalized = true; addChunk(Chunk{}, ProcessingUnitType::FINALIZE); collector_finished.wait(); + if (background_exception) + std::rethrow_exception(background_exception); } private: @@ -111,6 +122,9 @@ private: void addChunk(Chunk chunk, ProcessingUnitType type) { + if (background_exception) + std::rethrow_exception(background_exception); + const auto current_unit_number = writer_unit_number % processing_units.size(); auto & unit = processing_units[current_unit_number]; @@ -193,8 +207,6 @@ private: void onBackgroundException() { - tryLogCurrentException(__PRETTY_FUNCTION__); - std::unique_lock lock(mutex); if (!background_exception) { @@ -228,7 +240,7 @@ private: } if (emergency_stop) - return; + break; assert(unit.status == READY_TO_READ); @@ -238,7 +250,7 @@ private: /// Do main work here. out.write(unit.segment.data(), unit.actual_memory_size); - if (need_flush.exchange(false)) + if (need_flush.exchange(false) || auto_flush) IOutputFormat::flush(); ++collector_unit_number; @@ -252,13 +264,14 @@ private: /// We can exit only after writing last piece of to out buffer. if (copy_if_unit_type == ProcessingUnitType::FINALIZE) { - collector_finished.set(); break; } } + collector_finished.set(); } catch (...) { + collector_finished.set(); onBackgroundException(); } } From 67766034f285dabe7d7a41cf45a380826776e237 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Thu, 8 Oct 2020 19:43:29 +0300 Subject: [PATCH 051/284] disable progress bar with parallel_formatting --- programs/client/Client.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 92ae37123fa..6e2c11f9ca6 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1359,6 +1359,9 @@ private: if (with_output && with_output->settings_ast) apply_query_settings(*with_output->settings_ast); + if (context.getSettingsRef().output_format_parallel_formatting) + need_render_progress = false; + connection->forceConnected(connection_parameters.timeouts); ASTPtr input_function; @@ -1986,7 +1989,13 @@ private: written_first_block = true; } - bool clear_progess = std_out.offset() > 0; + // const auto & settings = context.getSettingsRef(); + // const auto parallel_formatting = settings.output_format_parallel_formatting; + + bool clear_progess = false; + // if (!parallel_formatting) + // clear_progess = std_out.offset() > 0; + if (clear_progess) clearProgress(); From 9064af6cb616fb16e770dc9d6905d1cd578e0d82 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Thu, 8 Oct 2020 19:46:55 +0300 Subject: [PATCH 052/284] get rid of std::terminate --- src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 2287bec1ab3..e8157708af7 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -52,7 +52,7 @@ public: { need_flush = true; if (!IOutputFormat::finalized) - std::terminate(); + finalize(); finishAndWait(); } @@ -70,7 +70,6 @@ public: void onCancel() override { - std::cout << "onCancel" << std::endl; finishAndWait(); } From 3c683b1d8620dbb8b396b7e89ba06713bfeb4e1f Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Mon, 2 Nov 2020 15:50:25 +0300 Subject: [PATCH 053/284] better --- src/Formats/FormatFactory.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 857d2194c20..5d8068c97f4 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -227,9 +227,12 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, /** TODO: Materialization is needed, because formats can use the functions `IDataType`, * which only work with full columns. */ - auto formatter_creator = [output_getter, sample, callback, format_settings] + + RowOutputFormatParams row_output_format_params{callback, ignore_no_row_delimiter}; + + auto formatter_creator = [output_getter, sample, row_output_format_params, format_settings] (WriteBuffer & output) -> OutputFormatPtr - { return output_getter(output, sample, std::move(callback), format_settings);}; + { return output_getter(output, sample, row_output_format_params, format_settings);}; ///FIXME ParallelFormattingOutputFormat::Params params{buf, sample, formatter_creator, settings.max_threads}; auto format = std::make_shared(params); @@ -242,7 +245,7 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, } - auto format = getOutputFormat(name, buf, sample, context, std::move(callback)); + auto format = getOutputFormat(name, buf, sample, context, std::move(callback), ignore_no_row_delimiter); return std::make_shared(std::make_shared(format), sample); } From f7ac8bf542273f082b74918a66a6e46b42b412a9 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Tue, 8 Dec 2020 01:52:51 +0300 Subject: [PATCH 054/284] rebase and fix tests --- programs/client/Client.cpp | 11 +++-------- src/Formats/FormatFactory.cpp | 12 ++++-------- src/Storages/StorageFile.cpp | 4 ++-- src/Storages/StorageURL.cpp | 4 ++-- .../0_stateless/01246_insert_into_watch_live_view.py | 7 +++++++ 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 6e2c11f9ca6..aa60503311a 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1359,8 +1359,8 @@ private: if (with_output && with_output->settings_ast) apply_query_settings(*with_output->settings_ast); - if (context.getSettingsRef().output_format_parallel_formatting) - need_render_progress = false; + // if (context.getSettingsRef().output_format_parallel_formatting) + // need_render_progress = false; connection->forceConnected(connection_parameters.timeouts); @@ -1989,12 +1989,7 @@ private: written_first_block = true; } - // const auto & settings = context.getSettingsRef(); - // const auto parallel_formatting = settings.output_format_parallel_formatting; - - bool clear_progess = false; - // if (!parallel_formatting) - // clear_progess = std_out.offset() > 0; + bool clear_progess = std_out.offset() > 0; if (clear_progess) clearProgress(); diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 5d8068c97f4..1962cf8218f 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -197,7 +197,7 @@ InputFormatPtr FormatFactory::getInput( } - auto format = getInputFormat(name, buf, sample, context, max_block_size, std::move(callback)); + auto format = getInputFormat(name, buf, sample, context, max_block_size, format_settings); return format; } @@ -222,17 +222,13 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, { const auto & output_getter = getCreators(name).output_processor_creator; - FormatSettings format_settings = getOutputFormatSetting(settings, context); - /** TODO: Materialization is needed, because formats can use the functions `IDataType`, * which only work with full columns. */ - - RowOutputFormatParams row_output_format_params{callback, ignore_no_row_delimiter}; - auto formatter_creator = [output_getter, sample, row_output_format_params, format_settings] + auto formatter_creator = [output_getter, sample, callback, format_settings] (WriteBuffer & output) -> OutputFormatPtr - { return output_getter(output, sample, row_output_format_params, format_settings);}; ///FIXME + { return output_getter(output, sample, {std::move(callback)}, format_settings);}; ParallelFormattingOutputFormat::Params params{buf, sample, formatter_creator, settings.max_threads}; auto format = std::make_shared(params); @@ -245,7 +241,7 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, } - auto format = getOutputFormat(name, buf, sample, context, std::move(callback), ignore_no_row_delimiter); + auto format = getOutputFormat(name, buf, sample, context, std::move(callback), format_settings); return std::make_shared(std::make_shared(format), sample); } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index caf9c06e0ba..57ab17e156d 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -333,8 +333,8 @@ public: reader = std::make_shared(format); - if (!column_defaults.empty()) - reader = std::make_shared(reader, column_defaults, context); + if (columns_description.hasDefaults()) + reader = std::make_shared(reader, columns_description, context); reader->readPrefix(); } diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 9533892d07a..8c69d9c0179 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -107,9 +107,9 @@ namespace compression_method); - auto input_format = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size); + auto input_format = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size, format_settings); reader = std::make_shared(input_format); - reader = std::make_shared(reader, column_defaults, context); + reader = std::make_shared(reader, columns, context); } String getName() const override diff --git a/tests/queries/0_stateless/01246_insert_into_watch_live_view.py b/tests/queries/0_stateless/01246_insert_into_watch_live_view.py index 0f7c6965b7b..f533573bfc7 100755 --- a/tests/queries/0_stateless/01246_insert_into_watch_live_view.py +++ b/tests/queries/0_stateless/01246_insert_into_watch_live_view.py @@ -18,6 +18,13 @@ with client(name='client1>', log=log) as client1, client(name='client2>', log=lo client2.expect(prompt) client3.expect(prompt) + client1.send('SET output_format_parallel_formatting=0') + client1.expect(prompt) + client2.send('SET output_format_parallel_formatting=0') + client2.expect(prompt) + client3.send('SET output_format_parallel_formatting=0') + client3.expect(prompt) + client1.send('SET allow_experimental_live_view = 1') client1.expect(prompt) client3.send('SET allow_experimental_live_view = 1') From 12e624fd9a7ee02e1d638b3e97dfa9a0e70f604e Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Thu, 10 Dec 2020 02:22:53 +0300 Subject: [PATCH 055/284] fix tests --- programs/client/Client.cpp | 7 ++++- src/Formats/FormatFactory.cpp | 29 +++++++++++++++---- src/Formats/FormatFactory.h | 5 ++++ src/Interpreters/Context.cpp | 5 ++++ src/Interpreters/Context.h | 2 ++ src/Interpreters/executeQuery.cpp | 5 +++- .../Impl/ParallelParsingInputFormat.cpp | 7 +++++ .../Formats/Impl/ParallelParsingInputFormat.h | 1 + src/Storages/StorageFile.cpp | 9 ++++-- 9 files changed, 61 insertions(+), 9 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index aa60503311a..9a30c125169 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1932,7 +1932,12 @@ private: if (has_vertical_output_suffix) current_format = "Vertical"; - block_out_stream = context.getOutputFormat(current_format, *out_buf, block); + if (!is_interactive && !need_render_progress) + block_out_stream = context.getOutputFormatParallelIfPossible(current_format, *out_buf, block); + + if (!block_out_stream) + block_out_stream = context.getOutputFormat(current_format, *out_buf, block); + block_out_stream->writePrefix(); } } diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 1962cf8218f..4f465e05e4e 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -176,6 +176,7 @@ InputFormatPtr FormatFactory::getInput( if (parallel_parsing) { + std::cout << "parallel_parsing" << std::endl; const auto & input_getter = getCreators(name).input_processor_creator; RowInputFormatParams row_input_format_params; @@ -185,10 +186,13 @@ InputFormatPtr FormatFactory::getInput( row_input_format_params.max_execution_time = settings.max_execution_time; row_input_format_params.timeout_overflow_mode = settings.timeout_overflow_mode; + std::cout << "format_settings.csv.delimiter " << format_settings.csv.delimiter << std::endl; + /// Const reference is copied to lambda. auto parser_creator = [input_getter, sample, row_input_format_params, format_settings] (ReadBuffer & input) -> InputFormatPtr - { return input_getter(input, sample, row_input_format_params, format_settings); }; + { std::cout << format_settings.csv.delimiter << std::endl; + return input_getter(input, sample, row_input_format_params, format_settings); }; ParallelParsingInputFormat::Params params{ @@ -201,8 +205,7 @@ InputFormatPtr FormatFactory::getInput( return format; } - -BlockOutputStreamPtr FormatFactory::getOutput(const String & name, +BlockOutputStreamPtr FormatFactory::getOutputParallelIfPossible(const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback, const std::optional & _format_settings) const { @@ -214,11 +217,10 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); } - const Settings & settings = context.getSettingsRef(); bool parallel_formatting = settings.output_format_parallel_formatting; - if (parallel_formatting && getCreators(name).supports_parallel_formatting && !settings.allow_experimental_live_view) + if (parallel_formatting && getCreators(name).supports_parallel_formatting && !settings.allow_experimental_live_view && !settings.output_format_json_array_of_rows) { const auto & output_getter = getCreators(name).output_processor_creator; @@ -241,6 +243,23 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, } + return nullptr; +} + + + +BlockOutputStreamPtr FormatFactory::getOutput(const String & name, + WriteBuffer & buf, const Block & sample, const Context & context, + WriteCallback callback, const std::optional & _format_settings) const +{ + auto format_settings = _format_settings + ? *_format_settings : getFormatSettings(context); + + if (!getCreators(name).output_processor_creator) + { + throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); + } + auto format = getOutputFormat(name, buf, sample, context, std::move(callback), format_settings); return std::make_shared(std::make_shared(format), sample); } diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index ddf557b9dfc..fcabdb5a9ae 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -116,6 +116,11 @@ public: UInt64 max_block_size, const std::optional & format_settings = std::nullopt) const; + /// Checks all preconditions. Returns nullptr of parallel formatting cannot be done. + BlockOutputStreamPtr getOutputParallelIfPossible(const String & name, WriteBuffer & buf, + const Block & sample, const Context & context, WriteCallback callback = {}, + const std::optional & format_settings = std::nullopt) const; + BlockOutputStreamPtr getOutput(const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback = {}, const std::optional & format_settings = std::nullopt) const; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index eb56703c61c..e7f0bb4995b 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -2076,6 +2076,11 @@ BlockInputStreamPtr Context::getInputFormat(const String & name, ReadBuffer & bu return std::make_shared(FormatFactory::instance().getInput(name, buf, sample, *this, max_block_size)); } +BlockOutputStreamPtr Context::getOutputFormatParallelIfPossible(const String & name, WriteBuffer & buf, const Block & sample) const +{ + return FormatFactory::instance().getOutputParallelIfPossible(name, buf, sample, *this); +} + BlockOutputStreamPtr Context::getOutputFormat(const String & name, WriteBuffer & buf, const Block & sample) const { return FormatFactory::instance().getOutput(name, buf, sample, *this); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index ccdaf98980d..faa8f606a29 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -412,6 +412,8 @@ public: /// I/O formats. BlockInputStreamPtr getInputFormat(const String & name, ReadBuffer & buf, const Block & sample, UInt64 max_block_size) const; + + BlockOutputStreamPtr getOutputFormatParallelIfPossible(const String & name, WriteBuffer & buf, const Block & sample) const; BlockOutputStreamPtr getOutputFormat(const String & name, WriteBuffer & buf, const Block & sample) const; OutputFormatPtr getOutputFormatProcessor(const String & name, WriteBuffer & buf, const Block & sample) const; diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 633b1e9bd5e..4ff4660fa13 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -945,7 +945,10 @@ void executeQuery( ? getIdentifierName(ast_query_with_output->format) : context.getDefaultFormat(); - BlockOutputStreamPtr out = context.getOutputFormat(format_name, *out_buf, streams.in->getHeader()); + BlockOutputStreamPtr out; + out = context.getOutputFormatParallelIfPossible(format_name, *out_buf, streams.in->getHeader()); + if (!out) + out = context.getOutputFormat(format_name, *out_buf, streams.in->getHeader()); /// Save previous progress callback if any. TODO Do it more conveniently. auto previous_progress_callback = context.getProgressCallback(); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 2c4a567b16f..1f66761be1a 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -79,6 +79,13 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa */ ReadBuffer read_buffer(unit.segment.data(), unit.segment.size(), 0); + std::cout << unit.segment.size() << std::endl; + + for (size_t i = 0; i < unit.segment.size(); ++i) { + std::cout << *(unit.segment.data() + i); + } + std::cout << std::endl; + InputFormatPtr input_format = internal_parser_creator(read_buffer); InternalParser parser(input_format); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index e53833764dd..0690638b97a 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 57ab17e156d..1469b535b51 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -329,7 +329,7 @@ public: read_buf = wrapReadBufferWithCompressionMethod(std::move(nested_buffer), method); auto format = FormatFactory::instance().getInput( - storage->format_name, *read_buf, metadata_snapshot->getSampleBlock(), context, max_block_size); + storage->format_name, *read_buf, metadata_snapshot->getSampleBlock(), context, max_block_size, storage->format_settings); reader = std::make_shared(format); @@ -481,9 +481,14 @@ public: write_buf = wrapWriteBufferWithCompressionMethod(std::move(naked_buffer), compression_method, 3); - writer = FormatFactory::instance().getOutput(storage.format_name, + writer = FormatFactory::instance().getOutputParallelIfPossible(storage.format_name, *write_buf, metadata_snapshot->getSampleBlock(), context, {}, format_settings); + + if (!writer) + writer = FormatFactory::instance().getOutput(storage.format_name, + *write_buf, metadata_snapshot->getSampleBlock(), context, + {}, format_settings); } Block getHeader() const override { return metadata_snapshot->getSampleBlock(); } From 0af342853afd1b72192d678a73dc936ac7dd6fa8 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Thu, 10 Dec 2020 03:26:27 +0300 Subject: [PATCH 056/284] add some logs --- src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index e8157708af7..bd224102bd9 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -46,6 +46,7 @@ public: /// and n threads for formatting. processing_units.resize(params.max_threads_for_parallel_formatting + 2); collector_thread = ThreadFromGlobalPool([&] { collectorThreadFunction(); }); + LOG_TRACE(&Poco::Logger::get("ParallelFormattingOutputFormat"), "Parallel formatting is being used."); } ~ParallelFormattingOutputFormat() override From b5436547ff1a6427c36b07341e90543b51dbc01f Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Tue, 15 Dec 2020 00:56:31 +0300 Subject: [PATCH 057/284] thread tests --- programs/client/Client.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 9a30c125169..79dc7e9e62f 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1994,16 +1994,18 @@ private: written_first_block = true; } - bool clear_progess = std_out.offset() > 0; + bool clear_progress = false; + if (need_render_progress) + clear_progress = std_out.offset() > 0; - if (clear_progess) + if (clear_progress) clearProgress(); /// Received data block is immediately displayed to the user. block_out_stream->flush(); /// Restore progress bar after data block. - if (clear_progess) + if (clear_progress) writeProgress(); } From dc27e0a4fd152c1d1b24cf32af2e080d7fa7cd96 Mon Sep 17 00:00:00 2001 From: spongedc Date: Tue, 15 Dec 2020 20:29:26 +0800 Subject: [PATCH 058/284] Fix error code --- src/Common/ErrorCodes.cpp | 1 - src/Databases/IDatabase.h | 1 - src/Interpreters/InterpreterShowCreateQuery.cpp | 2 +- tests/queries/0_stateless/01602_show_create_view.sql | 5 ++--- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 328d0a364ff..1e381808d16 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -529,7 +529,6 @@ M(560, ZSTD_ENCODER_FAILED) \ M(561, ZSTD_DECODER_FAILED) \ M(562, TLD_LIST_NOT_FOUND) \ - M(563, NOT_VIEW) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index fb2a9652d4b..fadec5fe7a9 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -30,7 +30,6 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; extern const int CANNOT_GET_CREATE_TABLE_QUERY; extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY; - extern const int NOT_VIEW; } class IDatabaseTablesIterator diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp index bcf6cc09473..869573f0d66 100644 --- a/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -54,7 +54,7 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() { auto & ast_create_query = create_query->as(); if (!ast_create_query.isView()) - throw Exception("'" + ast_create_query.database + "." + ast_create_query.table +"' is not VIEW", ErrorCodes::NOT_VIEW); + throw Exception("'" + ast_create_query.database + "." + ast_create_query.table +"' is not VIEW", ErrorCodes::BAD_ARGUMENTS); } } else if ((show_query = query_ptr->as())) diff --git a/tests/queries/0_stateless/01602_show_create_view.sql b/tests/queries/0_stateless/01602_show_create_view.sql index 0a4ecd4ceec..8e049d2d4cd 100644 --- a/tests/queries/0_stateless/01602_show_create_view.sql +++ b/tests/queries/0_stateless/01602_show_create_view.sql @@ -4,8 +4,7 @@ CREATE DATABASE test_1602; CREATE TABLE test_1602.tbl (`EventDate` DateTime, `CounterID` UInt32, `UserID` UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SETTINGS index_granularity = 8192; -CREATE VIEW test_1602.v AS SELECT * FROM test_1602.tbl; - +CREATE VIEW test_1602.v AS SELECT * FROM test_1602.tbl; CREATE MATERIALIZED VIEW test_1602.vv (`EventDate` DateTime, `CounterID` UInt32, `UserID` UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SETTINGS index_granularity = 8192 AS SELECT * FROM test_1602.tbl; @@ -21,6 +20,6 @@ SHOW CREATE VIEW test_1602.vvv; SHOW CREATE VIEW test_1602.not_exist_view; -- { serverError 390 } -SHOW CREATE VIEW test_1602.tbl; -- { serverError 563 } +SHOW CREATE VIEW test_1602.tbl; -- { serverError 36 } DROP DATABASE IF EXISTS test_1602; From 0338b6d0ed5ed72837dd86b7a76505b52e5e342d Mon Sep 17 00:00:00 2001 From: ana-uvarova Date: Tue, 15 Dec 2020 17:54:45 +0300 Subject: [PATCH 059/284] Transferred system.metric_log to another page. --- .../settings.md | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 533fcea5500..3028e12c20f 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -758,6 +758,66 @@ If the table doesn’t exist, ClickHouse will create it. If the structure of the ``` +## system.metric_log {#system_tables-metric_log} + +Contains history of metrics values from tables [`system.metrics`](../../operations/system-tables/metrics.md) and [`system.events`](../../operations/system-tables/events.md#system_tables-events), periodically flushed to disk. + +To turn on metrics history collection on `system.metric_log`, create `/etc/clickhouse-server/config.d/metric_log.xml` with following content: + +``` xml + + + system + metric_log
+ 7500 + 1000 + + +``` + +Columns: +- `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date. +- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time. +- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds resolution. + +**Example** + +``` sql +SELECT * FROM system.metric_log LIMIT 1 FORMAT Vertical; +``` + +``` text +Row 1: +────── +event_date: 2020-09-05 +event_time: 2020-09-05 16:22:33 +event_time_microseconds: 2020-09-05 16:22:33.196807 +milliseconds: 196 +ProfileEvent_Query: 0 +ProfileEvent_SelectQuery: 0 +ProfileEvent_InsertQuery: 0 +ProfileEvent_FailedQuery: 0 +ProfileEvent_FailedSelectQuery: 0 +... +... +CurrentMetric_Revision: 54439 +CurrentMetric_VersionInteger: 20009001 +CurrentMetric_RWLockWaitingReaders: 0 +CurrentMetric_RWLockWaitingWriters: 0 +CurrentMetric_RWLockActiveReaders: 0 +CurrentMetric_RWLockActiveWriters: 0 +CurrentMetric_GlobalThread: 74 +CurrentMetric_GlobalThreadActive: 26 +CurrentMetric_LocalThread: 0 +CurrentMetric_LocalThreadActive: 0 +CurrentMetric_DistributedFilesToInsert: 0 +``` + +**See also** + +- [system.asynchronous_metrics](../../operations/system-tables/asynchronous_metrics.md) — Contains periodically calculated metrics. +- [Monitoring](../../operations/monitoring.md) — Base concepts of ClickHouse monitoring. + ## text_log {#server_configuration_parameters-text_log} Settings for the [text_log](../../operations/system-tables/text_log.md#system_tables-text_log) system table for logging text messages. From b81a932408d5c23313a0682c5bb1ef2d4f8e7769 Mon Sep 17 00:00:00 2001 From: ana-uvarova Date: Tue, 15 Dec 2020 20:49:00 +0300 Subject: [PATCH 060/284] redone --- .../settings.md | 42 +------------------ .../en/operations/system-tables/metric_log.md | 14 +------ 2 files changed, 2 insertions(+), 54 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 3028e12c20f..318b80a23db 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -758,9 +758,7 @@ If the table doesn’t exist, ClickHouse will create it. If the structure of the ``` -## system.metric_log {#system_tables-metric_log} - -Contains history of metrics values from tables [`system.metrics`](../../operations/system-tables/metrics.md) and [`system.events`](../../operations/system-tables/events.md#system_tables-events), periodically flushed to disk. +## metric_log {#metric_log} To turn on metrics history collection on `system.metric_log`, create `/etc/clickhouse-server/config.d/metric_log.xml` with following content: @@ -775,44 +773,6 @@ To turn on metrics history collection on `system.metric_log`, create `/etc/click ``` -Columns: -- `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date. -- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time. -- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds resolution. - -**Example** - -``` sql -SELECT * FROM system.metric_log LIMIT 1 FORMAT Vertical; -``` - -``` text -Row 1: -────── -event_date: 2020-09-05 -event_time: 2020-09-05 16:22:33 -event_time_microseconds: 2020-09-05 16:22:33.196807 -milliseconds: 196 -ProfileEvent_Query: 0 -ProfileEvent_SelectQuery: 0 -ProfileEvent_InsertQuery: 0 -ProfileEvent_FailedQuery: 0 -ProfileEvent_FailedSelectQuery: 0 -... -... -CurrentMetric_Revision: 54439 -CurrentMetric_VersionInteger: 20009001 -CurrentMetric_RWLockWaitingReaders: 0 -CurrentMetric_RWLockWaitingWriters: 0 -CurrentMetric_RWLockActiveReaders: 0 -CurrentMetric_RWLockActiveWriters: 0 -CurrentMetric_GlobalThread: 74 -CurrentMetric_GlobalThreadActive: 26 -CurrentMetric_LocalThread: 0 -CurrentMetric_LocalThreadActive: 0 -CurrentMetric_DistributedFilesToInsert: 0 -``` - **See also** - [system.asynchronous_metrics](../../operations/system-tables/asynchronous_metrics.md) — Contains periodically calculated metrics. diff --git a/docs/en/operations/system-tables/metric_log.md b/docs/en/operations/system-tables/metric_log.md index c6aefc5034f..8cef5dc0931 100644 --- a/docs/en/operations/system-tables/metric_log.md +++ b/docs/en/operations/system-tables/metric_log.md @@ -2,19 +2,6 @@ Contains history of metrics values from tables `system.metrics` and `system.events`, periodically flushed to disk. -To turn on metrics history collection on `system.metric_log`, create `/etc/clickhouse-server/config.d/metric_log.xml` with following content: - -``` xml - - - system - metric_log
- 7500 - 1000 - - -``` - Columns: - `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date. - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time. @@ -55,6 +42,7 @@ CurrentMetric_DistributedFilesToInsert: 0 **See also** +- [metric_log setting](../../operations/server-configuration-parameters/settings.md#metric_log) — What you should add. - [system.asynchronous_metrics](../../operations/system-tables/asynchronous_metrics.md) — Contains periodically calculated metrics. - [system.events](../../operations/system-tables/events.md#system_tables-events) — Contains a number of events that occurred. - [system.metrics](../../operations/system-tables/metrics.md) — Contains instantly calculated metrics. From 4ab2022d24b4979c14e8a647baa6a7dfcb3df091 Mon Sep 17 00:00:00 2001 From: ana-uvarova Date: Tue, 15 Dec 2020 20:57:26 +0300 Subject: [PATCH 061/284] deleted links --- .../operations/server-configuration-parameters/settings.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 318b80a23db..2586295ac0f 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -773,11 +773,6 @@ To turn on metrics history collection on `system.metric_log`, create `/etc/click ``` -**See also** - -- [system.asynchronous_metrics](../../operations/system-tables/asynchronous_metrics.md) — Contains periodically calculated metrics. -- [Monitoring](../../operations/monitoring.md) — Base concepts of ClickHouse monitoring. - ## text_log {#server_configuration_parameters-text_log} Settings for the [text_log](../../operations/system-tables/text_log.md#system_tables-text_log) system table for logging text messages. From e54c90c9e24c37f3f83e2ffd27cb5571fec35cd3 Mon Sep 17 00:00:00 2001 From: ana-uvarova Date: Tue, 15 Dec 2020 21:03:06 +0300 Subject: [PATCH 062/284] right place --- .../settings.md | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 2586295ac0f..1f9302f8be3 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -567,6 +567,21 @@ For more information, see the MergeTreeSettings.h header file. ``` +## metric_log {#metric_log} + +To turn on metrics history collection on `system.metric_log`, create `/etc/clickhouse-server/config.d/metric_log.xml` with following content: + +``` xml + + + system + metric_log
+ 7500 + 1000 + + +``` + ## replicated_merge_tree {#server_configuration_parameters-replicated_merge_tree} Fine tuning for tables in the [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/mergetree.md). @@ -758,21 +773,6 @@ If the table doesn’t exist, ClickHouse will create it. If the structure of the ``` -## metric_log {#metric_log} - -To turn on metrics history collection on `system.metric_log`, create `/etc/clickhouse-server/config.d/metric_log.xml` with following content: - -``` xml - - - system - metric_log
- 7500 - 1000 - - -``` - ## text_log {#server_configuration_parameters-text_log} Settings for the [text_log](../../operations/system-tables/text_log.md#system_tables-text_log) system table for logging text messages. From bf096097cbdeb80555ad03d7592d24e690ea711b Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 15 Dec 2020 21:56:06 +0300 Subject: [PATCH 063/284] Update InterpreterShowCreateQuery.cpp --- src/Interpreters/InterpreterShowCreateQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp index 869573f0d66..e4f30c30cfb 100644 --- a/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -54,7 +54,7 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() { auto & ast_create_query = create_query->as(); if (!ast_create_query.isView()) - throw Exception("'" + ast_create_query.database + "." + ast_create_query.table +"' is not VIEW", ErrorCodes::BAD_ARGUMENTS); + throw Exception(backQuote(ast_create_query.database) + "." + backQuote(ast_create_query.table) + " is not a VIEW", ErrorCodes::BAD_ARGUMENTS); } } else if ((show_query = query_ptr->as())) From 81b0fa298992d42fe4df6a934ae6e90f21eacf51 Mon Sep 17 00:00:00 2001 From: spongedc Date: Wed, 16 Dec 2020 11:19:38 +0800 Subject: [PATCH 064/284] 1. rename is_view to is_ordinary_view 2. add more tests --- src/Interpreters/InterpreterCreateQuery.cpp | 6 +++--- src/Parsers/ASTCreateQuery.h | 2 +- src/Parsers/ParserCreateQuery.cpp | 6 +++--- src/Storages/StorageFactory.cpp | 2 +- tests/queries/0_stateless/01602_show_create_view.sql | 2 ++ 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index d57a14a61dc..8f3d5086b6b 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -593,7 +593,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const if (create.as_table_function) return; - if (create.storage || create.is_view || create.is_materialized_view || create.is_live_view || create.is_dictionary) + if (create.storage || create.is_dictionary || create.isView()) { if (create.temporary && create.storage && create.storage->engine && create.storage->engine->name != "Memory") throw Exception( @@ -748,7 +748,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (create.to_table_id && create.to_table_id.database_name.empty()) create.to_table_id.database_name = current_database; - if (create.select && (create.is_view || create.is_materialized_view || create.is_live_view)) + if (create.select && create.isView()) { AddDefaultDatabaseVisitor visitor(current_database); visitor.visit(*create.select); @@ -884,7 +884,7 @@ BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create) { /// If the query is a CREATE SELECT, insert the data into the table. if (create.select && !create.attach - && !create.is_view && !create.is_live_view && (!create.is_materialized_view || create.is_populate)) + && !create.is_ordinary_view && !create.is_live_view && (!create.is_materialized_view || create.is_populate)) { auto insert = std::make_shared(); insert->table_id = {create.database, create.table, create.uuid}; diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index 67ad9a89392..ee178b74460 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -57,7 +57,7 @@ class ASTCreateQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnC public: bool attach{false}; /// Query ATTACH TABLE, not CREATE TABLE. bool if_not_exists{false}; - bool is_view{false}; + bool is_ordinary_view{false}; bool is_materialized_view{false}; bool is_live_view{false}; bool is_populate{false}; diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index b08646bcf9d..262ddb09398 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -735,7 +735,7 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec String cluster_str; bool attach = false; bool if_not_exists = false; - bool is_view = false; + bool is_ordinary_view = false; bool is_materialized_view = false; bool is_populate = false; bool replace_view = false; @@ -759,7 +759,7 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec is_materialized_view = true; } else - is_view = true; + is_ordinary_view = true; if (!s_view.ignore(pos, expected)) return false; @@ -816,7 +816,7 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->attach = attach; query->if_not_exists = if_not_exists; - query->is_view = is_view; + query->is_ordinary_view = is_ordinary_view; query->is_materialized_view = is_materialized_view; query->is_populate = is_populate; query->replace_view = replace_view; diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index eda9f36010f..93982c2f3a3 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -53,7 +53,7 @@ StoragePtr StorageFactory::get( bool has_engine_args = false; - if (query.is_view) + if (query.is_ordinary_view) { if (query.storage) throw Exception("Specifying ENGINE is not allowed for a View", ErrorCodes::INCORRECT_QUERY); diff --git a/tests/queries/0_stateless/01602_show_create_view.sql b/tests/queries/0_stateless/01602_show_create_view.sql index 8e049d2d4cd..855ccf58562 100644 --- a/tests/queries/0_stateless/01602_show_create_view.sql +++ b/tests/queries/0_stateless/01602_show_create_view.sql @@ -22,4 +22,6 @@ SHOW CREATE VIEW test_1602.not_exist_view; -- { serverError 390 } SHOW CREATE VIEW test_1602.tbl; -- { serverError 36 } +SHOW CREATE TEMPORARY VIEW; -- { serverError 60 } + DROP DATABASE IF EXISTS test_1602; From a0083e23d68a2047323b469e7cc146c25d6fc389 Mon Sep 17 00:00:00 2001 From: spongedc Date: Wed, 16 Dec 2020 12:07:50 +0800 Subject: [PATCH 065/284] Fix compile error --- src/Databases/DatabaseOnDisk.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 4 ++-- src/Parsers/ASTCreateQuery.cpp | 2 +- src/Parsers/ASTCreateQuery.h | 2 +- src/Parsers/New/AST/CreateViewQuery.cpp | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 1e6b4019c4b..7fb91e1d074 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -111,7 +111,7 @@ String getObjectDefinitionFromCreateQuery(const ASTPtr & query) create->replace_view = false; /// For views it is necessary to save the SELECT query itself, for the rest - on the contrary - if (!create->is_view && !create->is_materialized_view && !create->is_live_view) + if (!create->isView()) create->select = nullptr; create->format = nullptr; diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 8f3d5086b6b..0f158eb757d 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -624,7 +624,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const const String qualified_name = backQuoteIfNeed(as_database_name) + "." + backQuoteIfNeed(as_table_name); - if (as_create.is_view) + if (as_create.is_ordinary_view) throw Exception( "Cannot CREATE a table AS " + qualified_name + ", it is a View", ErrorCodes::INCORRECT_QUERY); @@ -1030,7 +1030,7 @@ AccessRightsElements InterpreterCreateQuery::getRequiredAccess() const { required_access.emplace_back(AccessType::CREATE_DICTIONARY, create.database, create.table); } - else if (create.is_view || create.is_materialized_view || create.is_live_view) + else if (create.isView()) { if (create.temporary) required_access.emplace_back(AccessType::CREATE_TEMPORARY_TABLE); diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index 03db54c6957..4cb512fee20 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -231,7 +231,7 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat if (!is_dictionary) { std::string what = "TABLE"; - if (is_view) + if (is_ordinary_view) what = "VIEW"; if (is_materialized_view) what = "MATERIALIZED VIEW"; diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index ee178b74460..4dc895a5774 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -91,7 +91,7 @@ public: return removeOnCluster(clone(), new_database); } - bool isView() { return is_view || is_materialized_view || is_live_view; } + bool isView() const { return is_ordinary_view || is_materialized_view || is_live_view; } protected: void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; diff --git a/src/Parsers/New/AST/CreateViewQuery.cpp b/src/Parsers/New/AST/CreateViewQuery.cpp index df68687eb13..e558a141ca9 100644 --- a/src/Parsers/New/AST/CreateViewQuery.cpp +++ b/src/Parsers/New/AST/CreateViewQuery.cpp @@ -35,7 +35,7 @@ ASTPtr CreateViewQuery::convertToOld() const query->attach = attach; query->replace_view = replace; query->if_not_exists = if_not_exists; - query->is_view = true; + query->is_ordinary_view = true; query->cluster = cluster_name; if (has(SCHEMA)) query->set(query->columns_list, get(SCHEMA)->convertToOld()); From 5a19e3068e1141533d83d34c8512a2976c8bad6f Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 16 Dec 2020 19:52:15 +0300 Subject: [PATCH 066/284] Update settings.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Внес поправки в описание настройки. --- docs/en/operations/settings/settings.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index c60cd2c2c52..f74e8af3401 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2429,12 +2429,12 @@ Enables special logic to perform merges on replicas. Possible values: - Positive integer (in seconds). -- 0 — Special merge logic is not executed. +- 0 — Special merges logic is not executed. Default value: `0`. **Special logic to perform merges** -Selects one replica to perform the merge on. Sets the time threshold from the start of the merge. Other replicas wait for the merge to finish, then download the result. If the time threshold passes and the selected replica does not perform merges, then the merge is performed manually. +Selects one replica to perform the merge on. Sets the time threshold from the start of the merge. Other replicas wait for the merge to finish, then download the result. If the time threshold passes, and the selected replica does not perform merge, then the merge is performed manually. [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) From 77e10884fcbe475c5f1c1bc3f369c6d42a8ab752 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 16 Dec 2020 20:04:16 +0300 Subject: [PATCH 067/284] Document union_default_mode setting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Задокументировал настройку union_default_mode. --- docs/en/operations/settings/settings.md | 79 +++++++++++++++++++ .../statements/select/union-all.md | 5 +- .../statements/select/union-all.md | 6 ++ 3 files changed, 89 insertions(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index c5e44d4c464..97375f86d03 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2423,4 +2423,83 @@ Possible values: Default value: `0`. +## union_default_mode {#union-default-mode} + +Sets a special mode for combining `SELECT` query results using the [UNION](../../sql-reference/statements/select/union-all.md) expression. + +Possible values: + +- `'DISTINCT'` — ClickHouse outputs rows as a result of combining queries removing duplicate rows. +- `'ALL'` — ClickHouse outputs all rows as a result of combining queries including duplicate rows. +- `''` — Clickhouse generates an exception when used with `UNION`. + +Default value: `'DISTINCT'`. + +**Example of using the 'DISTINCT' value** + +Query: + +```sql +SET union_default_mode = 'DISTINCT'; +SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; +``` + +Result: + +```text +┌─1─┐ +│ 1 │ +└───┘ +┌─1─┐ +│ 2 │ +└───┘ +┌─1─┐ +│ 3 │ +└───┘ +``` + +**Example of using the 'ALL' value** + +Query: + +```sql +SET union_default_mode = 'ALL'; +SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; +``` + +Result: + +```text +┌─1─┐ +│ 1 │ +└───┘ +┌─1─┐ +│ 2 │ +└───┘ +┌─1─┐ +│ 2 │ +└───┘ +┌─1─┐ +│ 3 │ +└───┘ +``` + +**Example of using the '' value** + +Query: + +```sql +SET union_default_mode = ''; +SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; +``` + +Result: + +```text +Query id: 8f3755e8-ef76-4d1a-bdb2-7f6fc8a669ec + +Received exception from server (version 20.11.1): +Code: 2005. DB::Exception: Received from localhost:9000. DB::Exception: Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty. +``` + [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/en/sql-reference/statements/select/union-all.md b/docs/en/sql-reference/statements/select/union-all.md index f150efbdc80..85c50450273 100644 --- a/docs/en/sql-reference/statements/select/union-all.md +++ b/docs/en/sql-reference/statements/select/union-all.md @@ -26,12 +26,15 @@ Type casting is performed for unions. For example, if two queries being combined Queries that are parts of `UNION ALL` can’t be enclosed in round brackets. [ORDER BY](../../../sql-reference/statements/select/order-by.md) and [LIMIT](../../../sql-reference/statements/select/limit.md) are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with `UNION ALL` in a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause. # UNION DISTINCT Clause {#union-distinct-clause} + The difference between `UNION ALL` and `UNION DISTINCT` is that `UNION DISTINCT` will do a distinct transform for union result, it is equivalent to `SELECT DISTINCT` from a subquery containing `UNION ALL`. # UNION Clause {#union-clause} -By default, `UNION` has the same behavior as `UNION DISTINCT`, but you can specify union mode by setting `union_default_mode`, values can be 'ALL', 'DISTINCT' or empty string. However, if you use `UNION` with setting `union_default_mode` to empty string, it will throw an exception. +By default, `UNION` has the same behavior as `UNION DISTINCT`, but you can specify union mode by [union_default_mode](../../../operations/settings/settings.md#union-default-mode) setting, values can be `ALL`, `DISTINCT` or an empty string. However, if you use `UNION` with `union_default_mode` setting to empty string, it will throw an exception. ## Implementation Details {#implementation-details} Queries that are parts of `UNION/UNION ALL/UNION DISTINCT` can be run simultaneously, and their results can be mixed together. + +[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/union-all/) diff --git a/docs/ru/sql-reference/statements/select/union-all.md b/docs/ru/sql-reference/statements/select/union-all.md index b9d1f485a7b..86bcf6a53cc 100644 --- a/docs/ru/sql-reference/statements/select/union-all.md +++ b/docs/ru/sql-reference/statements/select/union-all.md @@ -29,6 +29,12 @@ SELECT CounterID, 2 AS table, sum(Sign) AS c Поддерживается только `UNION ALL`. Обычный `UNION` (`UNION DISTINCT`) не поддерживается. Если вам это нужно `UNION DISTINCT`, вы можете написать `SELECT DISTINCT` из подзапроса, содержащего `UNION ALL`. +# Секция UNION {#union-clause} + +По умолчанию, `UNION` ведет себя так же, как и `UNION DISTINCT`. Но вы можете указать режим объединения с помощью настройки [union_default_mode](../../../operations/settings/settings.md#union-default-mode), значениями которой могут быть `ALL`, `DISTINCT` или пустая строка. Однако, если вы используете `UNION` с настройкой `union_default_mode`, значением которой является пустая строка, то сгенерируется исключение. + ## Детали реализации {#implementation-details} Запросы, которые являются частью `UNION ALL` выполняются параллельно, и их результаты могут быть смешаны вместе. + +[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/select/union-all/) From e8093eaaa697be550664101176ba44a0704b1e33 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 16 Dec 2020 23:33:08 +0300 Subject: [PATCH 068/284] Update settings.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Изменил заголовок примера. --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 0f8ee7ee632..dd4ea6697bf 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2520,7 +2520,7 @@ Result: └───┘ ``` -**Example of using the '' value** +**Example of using an empty string as a setting value** Query: From e53281e1763e77016fb463f07916a588afb51b41 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 16 Dec 2020 23:38:28 +0300 Subject: [PATCH 069/284] Update distribution_queue.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Исправил this на these. --- docs/en/operations/system-tables/distribution_queue.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/distribution_queue.md b/docs/en/operations/system-tables/distribution_queue.md index 9068e7e860c..fdc6a134da2 100644 --- a/docs/en/operations/system-tables/distribution_queue.md +++ b/docs/en/operations/system-tables/distribution_queue.md @@ -1,6 +1,6 @@ # system.distribution_queue {#system_tables-distribution_queue} -Contains information about local files that are in the queue to be sent to the shards. This local files contain new parts that are created by inserting new data into the Distributed table in asynchronous mode. +Contains information about local files that are in the queue to be sent to the shards. These local files contain new parts that are created by inserting new data into the Distributed table in asynchronous mode. Columns: From d20da24b0b472a149a6d23d3ad328c090bdb148f Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Thu, 17 Dec 2020 00:56:31 +0300 Subject: [PATCH 070/284] Update settings.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Убрал лишние символы. --- docs/en/operations/settings/settings.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index dd4ea6697bf..2caf4a6f370 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2447,7 +2447,6 @@ Result: {"number":"2"} ``` -======= ## allow_nullable_key {#allow-nullable-key} Allows using of the [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable)-typed values in a sorting and a primary key for [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engines-mergetree) tables. From bcaccab0ff35e5280a014e5a0e899321a7f057c9 Mon Sep 17 00:00:00 2001 From: spongedc Date: Thu, 17 Dec 2020 11:28:18 +0800 Subject: [PATCH 071/284] 1. Fix test error 2. Add more test cases --- .../01083_expressions_in_engine_arguments.sql | 2 +- .../01602_show_create_view.reference | 4 ++++ .../0_stateless/01602_show_create_view.sql | 24 +++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql index 22aa4434f19..e8ea12f9f9f 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql @@ -59,7 +59,7 @@ SHOW CREATE distributed; SHOW CREATE distributed_tf; SHOW CREATE url; SHOW CREATE rich_syntax; -SHOW CREATE view; +SHOW CREATE VIEW view; SHOW CREATE dict; INSERT INTO buffer VALUES (1); diff --git a/tests/queries/0_stateless/01602_show_create_view.reference b/tests/queries/0_stateless/01602_show_create_view.reference index 2c2ba13bef3..5d4bd2cd972 100644 --- a/tests/queries/0_stateless/01602_show_create_view.reference +++ b/tests/queries/0_stateless/01602_show_create_view.reference @@ -1,3 +1,7 @@ CREATE VIEW test_1602.v\n(\n `EventDate` DateTime,\n `CounterID` UInt32,\n `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl CREATE MATERIALIZED VIEW test_1602.vv\n(\n `EventDate` DateTime,\n `CounterID` UInt32,\n `UserID` UInt32\n)\nENGINE = MergeTree\nPARTITION BY toYYYYMM(EventDate)\nORDER BY (CounterID, EventDate, intHash32(UserID))\nSETTINGS index_granularity = 8192 AS\nSELECT *\nFROM test_1602.tbl CREATE LIVE VIEW test_1602.vvv\n(\n `EventDate` DateTime,\n `CounterID` UInt32,\n `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl +CREATE VIEW test_1602.VIEW\n(\n `EventDate` DateTime,\n `CounterID` UInt32,\n `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl +CREATE VIEW test_1602.DATABASE\n(\n `EventDate` DateTime,\n `CounterID` UInt32,\n `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl +CREATE VIEW test_1602.DICTIONARY\n(\n `EventDate` DateTime,\n `CounterID` UInt32,\n `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl +CREATE VIEW test_1602.TABLE\n(\n `EventDate` DateTime,\n `CounterID` UInt32,\n `UserID` UInt32\n) AS\nSELECT *\nFROM test_1602.tbl diff --git a/tests/queries/0_stateless/01602_show_create_view.sql b/tests/queries/0_stateless/01602_show_create_view.sql index 855ccf58562..fd5bf70d470 100644 --- a/tests/queries/0_stateless/01602_show_create_view.sql +++ b/tests/queries/0_stateless/01602_show_create_view.sql @@ -5,8 +5,16 @@ CREATE DATABASE test_1602; CREATE TABLE test_1602.tbl (`EventDate` DateTime, `CounterID` UInt32, `UserID` UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SETTINGS index_granularity = 8192; CREATE VIEW test_1602.v AS SELECT * FROM test_1602.tbl; + +CREATE VIEW test_1602.DATABASE AS SELECT * FROM test_1602.tbl; + +CREATE VIEW test_1602.DICTIONARY AS SELECT * FROM test_1602.tbl; + +CREATE VIEW test_1602.TABLE AS SELECT * FROM test_1602.tbl; + CREATE MATERIALIZED VIEW test_1602.vv (`EventDate` DateTime, `CounterID` UInt32, `UserID` UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SETTINGS index_granularity = 8192 AS SELECT * FROM test_1602.tbl; +CREATE VIEW test_1602.VIEW AS SELECT * FROM test_1602.tbl; SET allow_experimental_live_view=1; @@ -24,4 +32,20 @@ SHOW CREATE VIEW test_1602.tbl; -- { serverError 36 } SHOW CREATE TEMPORARY VIEW; -- { serverError 60 } +SHOW CREATE VIEW; -- { clientError 62 } + +SHOW CREATE DATABASE; -- { clientError 62 } + +SHOW CREATE DICTIONARY; -- { clientError 62 } + +SHOW CREATE TABLE; -- { clientError 62 } + +SHOW CREATE test_1602.VIEW; + +SHOW CREATE test_1602.DATABASE; + +SHOW CREATE test_1602.DICTIONARY; + +SHOW CREATE test_1602.TABLE; + DROP DATABASE IF EXISTS test_1602; From 5425fa4dd2f22d3266f164c695c0ee483686fc67 Mon Sep 17 00:00:00 2001 From: spongedc Date: Thu, 17 Dec 2020 12:27:04 +0800 Subject: [PATCH 072/284] Fix style check fail --- src/Interpreters/InterpreterShowCreateQuery.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp index e4f30c30cfb..10c8339c135 100644 --- a/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes { extern const int SYNTAX_ERROR; extern const int THERE_IS_NO_QUERY; + extern const int BAD_ARGUMENTS; } BlockIO InterpreterShowCreateQuery::execute() From fbf2ac35e843d7afedd074c712792ca6f88614b8 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Thu, 17 Dec 2020 18:14:09 +0300 Subject: [PATCH 073/284] fix tsan --- src/Formats/FormatFactory.cpp | 8 ++------ .../Formats/Impl/{NativeFormat.cpp => NativeFormat.h} | 0 .../Formats/Impl/ParallelParsingInputFormat.cpp | 7 ------- tests/integration/test_storage_kafka/test.py | 2 +- 4 files changed, 3 insertions(+), 14 deletions(-) rename src/Processors/Formats/Impl/{NativeFormat.cpp => NativeFormat.h} (100%) diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 4f465e05e4e..edce5a6bbf9 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -176,7 +176,6 @@ InputFormatPtr FormatFactory::getInput( if (parallel_parsing) { - std::cout << "parallel_parsing" << std::endl; const auto & input_getter = getCreators(name).input_processor_creator; RowInputFormatParams row_input_format_params; @@ -186,13 +185,10 @@ InputFormatPtr FormatFactory::getInput( row_input_format_params.max_execution_time = settings.max_execution_time; row_input_format_params.timeout_overflow_mode = settings.timeout_overflow_mode; - std::cout << "format_settings.csv.delimiter " << format_settings.csv.delimiter << std::endl; - /// Const reference is copied to lambda. auto parser_creator = [input_getter, sample, row_input_format_params, format_settings] (ReadBuffer & input) -> InputFormatPtr - { std::cout << format_settings.csv.delimiter << std::endl; - return input_getter(input, sample, row_input_format_params, format_settings); }; + { return input_getter(input, sample, row_input_format_params, format_settings); }; ParallelParsingInputFormat::Params params{ diff --git a/src/Processors/Formats/Impl/NativeFormat.cpp b/src/Processors/Formats/Impl/NativeFormat.h similarity index 100% rename from src/Processors/Formats/Impl/NativeFormat.cpp rename to src/Processors/Formats/Impl/NativeFormat.h diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 1f66761be1a..2c4a567b16f 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -79,13 +79,6 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa */ ReadBuffer read_buffer(unit.segment.data(), unit.segment.size(), 0); - std::cout << unit.segment.size() << std::endl; - - for (size_t i = 0; i < unit.segment.size(); ++i) { - std::cout << *(unit.segment.data() + i); - } - std::cout << std::endl; - InputFormatPtr input_format = internal_parser_creator(read_buffer); InternalParser parser(input_format); diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 07d2bcb60c0..1f31cbdbbc7 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -322,7 +322,7 @@ def test_kafka_formats(kafka_cluster): # /src/IO/VarInt.h:149: DB::readVarUInt(unsigned long&, DB::ReadBuffer&) @ 0x15c68844 in /usr/bin/clickhouse # /src/DataStreams/NativeBlockInputStream.cpp:124: DB::NativeBlockInputStream::readImpl() @ 0x1d3e2778 in /usr/bin/clickhouse # /src/DataStreams/IBlockInputStream.cpp:60: DB::IBlockInputStream::read() @ 0x1c9c92fd in /usr/bin/clickhouse - # /src/Processors/Formats/Impl/NativeFormat.cpp:42: DB::NativeInputFormatFromNativeBlockInputStream::generate() @ 0x1df1ea79 in /usr/bin/clickhouse + # /src/Processors/Formats/Impl/NativeFormat.h:42: DB::NativeInputFormatFromNativeBlockInputStream::generate() @ 0x1df1ea79 in /usr/bin/clickhouse # /src/Processors/ISource.cpp:48: DB::ISource::work() @ 0x1dd79737 in /usr/bin/clickhouse ], }, From fa8823243dbf35711291529566b0652aabc4a1a2 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Thu, 17 Dec 2020 20:29:10 +0300 Subject: [PATCH 074/284] Update settings.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Добавил подраздел в русскую версию, чтобы исправить битую ссылку. --- docs/ru/operations/settings/settings.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 0a8094231c2..1b688a528d3 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2324,4 +2324,6 @@ SELECT number FROM numbers(3) FORMAT JSONEachRow; Значение по умолчанию: `0`. +## union_default_mode {#union-default-mode} + [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) From d4ff611d755c38c0d77d0a7ed50d3c35c190edf9 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Fri, 18 Dec 2020 02:31:01 +0300 Subject: [PATCH 075/284] build after merge --- src/Common/Arena.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/Arena.h b/src/Common/Arena.h index 904ac0ac1a2..0f4f0420c38 100644 --- a/src/Common/Arena.h +++ b/src/Common/Arena.h @@ -130,7 +130,7 @@ private: public: Arena(size_t initial_size_ = 4096, size_t growth_factor_ = 2, size_t linear_growth_threshold_ = 128 * 1024 * 1024) : growth_factor(growth_factor_), linear_growth_threshold(linear_growth_threshold_), - head(new MemoryChunk(initial_size_, nullptr)), size_in_bytes(head->size()) + head(new MemoryChunk(initial_size_, nullptr)), size_in_bytes(head->size()), page_size(static_cast(::getPageSize())) { } From 32cbfabfaf75b51ad60166c11f2b7554c80efad6 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Fri, 18 Dec 2020 23:04:51 +0300 Subject: [PATCH 076/284] Update settings.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Выполнил перевод на русский язык и внес поправки в английскую версию. --- .../mergetree-family/replication.md | 3 +-- docs/en/operations/settings/settings.md | 10 +++++++--- docs/ru/operations/settings/settings.md | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 4758ff321e8..03319fd7575 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -113,8 +113,6 @@ You can have any number of replicas of the same data. Yandex.Metrica uses double The system monitors data synchronicity on replicas and is able to recover after a failure. Failover is automatic (for small differences in data) or semi-automatic (when data differs too much, which may indicate a configuration error). -You can use a special logic for performing merges (for more information, see the documentation for [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) setting). - ## Creating Replicated Tables {#creating-replicated-tables} The `Replicated` prefix is added to the table engine name. For example:`ReplicatedMergeTree`. @@ -286,5 +284,6 @@ If the data in ZooKeeper was lost or damaged, you can save data by moving it to **See also** - [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) +- [execute_merges_on_single_replica_time_threshold setting](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) [Original article](https://clickhouse.tech/docs/en/operations/table_engines/replication/) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 6e34d76fb97..a9c26d4c06f 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2465,12 +2465,16 @@ Enables special logic to perform merges on replicas. Possible values: - Positive integer (in seconds). -- 0 — Special merges logic is not executed. +- 0 — Special merges logic is not used. Merges happen in the usual way on all the replicas. Default value: `0`. -**Special logic to perform merges** +**Usage** -Selects one replica to perform the merge on. Sets the time threshold from the start of the merge. Other replicas wait for the merge to finish, then download the result. If the time threshold passes, and the selected replica does not perform merge, then the merge is performed manually. +It can be useful when merges are CPU bounded not IO bounded (heavy compression is in use, calculation expensive aggregate functions or default expressions or just very high number of tiny merges). + +Selects one replica to perform the merge on. Sets the time threshold from the start of the merge. Other replicas wait for the merge to finish, then download the result. If the time threshold passes and the selected replica does not perform the merge, then the merge is performed on other replicas as usual. + +High values for that threshold may lead to replication delays. [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 0a8094231c2..2cf6c5e5795 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2324,4 +2324,23 @@ SELECT number FROM numbers(3) FORMAT JSONEachRow; Значение по умолчанию: `0`. +## execute_merges_on_single_replica_time_threshold {#execute-merges-on-single-replica-time-threshold} + +Включает особую логику выполнения слияний на репликах. + +Возможные значения: + +- Положительное целое число (в секундах). +- 0 — не используется особая логика выполнения слияний. Слияния происходят обычным образом на всех репликах. + +Значение по умолчанию: `0`. + +**Использование** + +Это может быть полезно, когда слияния ограничены процессором, а не вводом-выводом (используется тяжелое сжатие, вычисление дорогостоящих агрегатных функций или выражений по умолчанию или просто очень большое количество мелких слияний). + +Выбирается одна реплика для выполнения слияния. Устанавливается порог времени с момента начала слияния. Другие реплики ждут завершения слияния, а затем скачивают результат. Если время выполнения слияния превышает установленный порог и выбранная реплика не выполняет слияние, тогда слияние выполняется на других репликах как обычно. + +Большие значения этой настройки могут привести к задержкам репликации. + [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) From 5705fa5a34c3c172c6bca00f19040ad49e5fcec9 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sat, 19 Dec 2020 00:08:13 +0300 Subject: [PATCH 077/284] Update settings.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Внес поправки в английскую и русскую версии. --- .../en/engines/table-engines/mergetree-family/replication.md | 4 ++-- docs/en/operations/settings/settings.md | 4 ++-- .../ru/engines/table-engines/mergetree-family/replication.md | 5 +++++ docs/ru/operations/settings/settings.md | 4 ++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 03319fd7575..04a239e0b06 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -281,9 +281,9 @@ After this, you can launch the server, create a `MergeTree` table, move the data If the data in ZooKeeper was lost or damaged, you can save data by moving it to an unreplicated table as described above. -**See also** +**See Also** - [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) -- [execute_merges_on_single_replica_time_threshold setting](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) +- [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) [Original article](https://clickhouse.tech/docs/en/operations/table_engines/replication/) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index a9c26d4c06f..092c76af298 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2471,10 +2471,10 @@ Default value: `0`. **Usage** -It can be useful when merges are CPU bounded not IO bounded (heavy compression is in use, calculation expensive aggregate functions or default expressions or just very high number of tiny merges). - Selects one replica to perform the merge on. Sets the time threshold from the start of the merge. Other replicas wait for the merge to finish, then download the result. If the time threshold passes and the selected replica does not perform the merge, then the merge is performed on other replicas as usual. High values for that threshold may lead to replication delays. +Heavy compression is in use, calculation expensive aggregate functions or default expressions or just very high number of tiny merges when using the settings. It can be useful when merges are CPU bounded not IO bounded. + [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/ru/engines/table-engines/mergetree-family/replication.md b/docs/ru/engines/table-engines/mergetree-family/replication.md index 6d3930e33ce..f17e1b035d4 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replication.md +++ b/docs/ru/engines/table-engines/mergetree-family/replication.md @@ -246,4 +246,9 @@ $ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data Если данные в ZooKeeper оказались утеряны или повреждены, то вы можете сохранить данные, переместив их в нереплицируемую таблицу, как описано в пункте выше. +**Смотрите также** + +- [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) +- [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) + [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/replication/) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 2cf6c5e5795..8777dba6609 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2337,10 +2337,10 @@ SELECT number FROM numbers(3) FORMAT JSONEachRow; **Использование** -Это может быть полезно, когда слияния ограничены процессором, а не вводом-выводом (используется тяжелое сжатие, вычисление дорогостоящих агрегатных функций или выражений по умолчанию или просто очень большое количество мелких слияний). - Выбирается одна реплика для выполнения слияния. Устанавливается порог времени с момента начала слияния. Другие реплики ждут завершения слияния, а затем скачивают результат. Если время выполнения слияния превышает установленный порог и выбранная реплика не выполняет слияние, тогда слияние выполняется на других репликах как обычно. Большие значения этой настройки могут привести к задержкам репликации. +При использовании настройки происходит тяжелое сжатие, вычисление слишком затратных агрегатных функций или выражений по умолчанию или очень большое количество мелких слияний. Эта настройка полезна, когда слияния ограничены процессором, а не вводом-выводом. + [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) From 906fd18f4aa23814079c9958a85caa7a8561111b Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sun, 20 Dec 2020 14:21:04 +0300 Subject: [PATCH 078/284] Update settings.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Внес поправки в английскую и русскую версии. --- docs/en/operations/settings/settings.md | 2 +- docs/ru/operations/settings/settings.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 092c76af298..8b9790b4d5d 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2475,6 +2475,6 @@ Selects one replica to perform the merge on. Sets the time threshold from the st High values for that threshold may lead to replication delays. -Heavy compression is in use, calculation expensive aggregate functions or default expressions or just very high number of tiny merges when using the settings. It can be useful when merges are CPU bounded not IO bounded. +It can be useful when merges are CPU bounded not IO bounded (performing heavy data compression, calculating aggregate functions or default expressions that require a large amount of calculations, or just very high number of tiny merges). [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 8777dba6609..2f1b89fe645 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2341,6 +2341,6 @@ SELECT number FROM numbers(3) FORMAT JSONEachRow; Большие значения этой настройки могут привести к задержкам репликации. -При использовании настройки происходит тяжелое сжатие, вычисление слишком затратных агрегатных функций или выражений по умолчанию или очень большое количество мелких слияний. Эта настройка полезна, когда слияния ограничены процессором, а не вводом-выводом. +Эта настройка полезна, когда скорость слияния ограничивается мощностью процессора, а не скоростью операций ввода-вывода (при выполнении "тяжелого" сжатия данных, при расчете агрегатных функций или выражений по умолчанию, требующих большого объема вычислений, или просто при большом количестве мелких слияний). [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) From 442f9cf59046c2f494278047111d5fff53a726d3 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sun, 20 Dec 2020 15:25:14 +0300 Subject: [PATCH 079/284] Update settings.md and union-all.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Выполнил перевод на русский язык. --- docs/en/operations/settings/settings.md | 18 ------ docs/ru/operations/settings/settings.md | 59 +++++++++++++++++++ .../statements/select/union-all.md | 8 +-- 3 files changed, 63 insertions(+), 22 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 2caf4a6f370..98ccdab0c2e 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2519,22 +2519,4 @@ Result: └───┘ ``` -**Example of using an empty string as a setting value** - -Query: - -```sql -SET union_default_mode = ''; -SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; -``` - -Result: - -```text -Query id: 8f3755e8-ef76-4d1a-bdb2-7f6fc8a669ec - -Received exception from server (version 20.11.1): -Code: 2005. DB::Exception: Received from localhost:9000. DB::Exception: Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty. -``` - [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 1b688a528d3..d94f7ddfae8 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2326,4 +2326,63 @@ SELECT number FROM numbers(3) FORMAT JSONEachRow; ## union_default_mode {#union-default-mode} +Устанавливает особый режим объединения результатов `SELECT` запросов, используя выражение [UNION](../../sql-reference/statements/select/union-all.md). + +Возможные значения: + +- `'DISTINCT'` — ClickHouse выводит строки в результате объединения результатов запросов, удаляя повторяющиеся строки. +- `'ALL'` — ClickHouse выводит все строки в результате объединения результатов запросов, включая повторяющиеся строки. +- `''` — Clickhouse генерирует исключение при использовании с `UNION`. + +Значение по умолчанию: `'DISTINCT'`. + +**Пример использования 'DISTINCT' в качестве значения настройки** + +Запрос: + +```sql +SET union_default_mode = 'DISTINCT'; +SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; +``` + +Результат: + +```text +┌─1─┐ +│ 1 │ +└───┘ +┌─1─┐ +│ 2 │ +└───┘ +┌─1─┐ +│ 3 │ +└───┘ +``` + +**Пример использования 'ALL' в качестве значения настройки** + +Запрос: + +```sql +SET union_default_mode = 'ALL'; +SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; +``` + +Результат: + +```text +┌─1─┐ +│ 1 │ +└───┘ +┌─1─┐ +│ 2 │ +└───┘ +┌─1─┐ +│ 2 │ +└───┘ +┌─1─┐ +│ 3 │ +└───┘ +``` + [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) diff --git a/docs/ru/sql-reference/statements/select/union-all.md b/docs/ru/sql-reference/statements/select/union-all.md index 86bcf6a53cc..72fc7299b30 100644 --- a/docs/ru/sql-reference/statements/select/union-all.md +++ b/docs/ru/sql-reference/statements/select/union-all.md @@ -25,16 +25,16 @@ SELECT CounterID, 2 AS table, sum(Sign) AS c Запросы, которые являются частью `UNION ALL` не могут быть заключен в круглые скобки. [ORDER BY](order-by.md) и [LIMIT](limit.md) применяются к отдельным запросам, а не к конечному результату. Если вам нужно применить преобразование к конечному результату, вы можете разместить все объединенные с помощью `UNION ALL` запросы в подзапрос в секции [FROM](from.md). -## Ограничения {#limitations} +# Секция UNION DISTINCT {#union-distinct-clause} -Поддерживается только `UNION ALL`. Обычный `UNION` (`UNION DISTINCT`) не поддерживается. Если вам это нужно `UNION DISTINCT`, вы можете написать `SELECT DISTINCT` из подзапроса, содержащего `UNION ALL`. +Разница между `UNION ALL` и `UNION DISTINCT` в том, что `UNION DISTINCT` выполняет явное преобразование для результата объединения. Это равнозначно выражению `SELECT DISTINCT` из подзапроса, содержащего `UNION ALL`. # Секция UNION {#union-clause} -По умолчанию, `UNION` ведет себя так же, как и `UNION DISTINCT`. Но вы можете указать режим объединения с помощью настройки [union_default_mode](../../../operations/settings/settings.md#union-default-mode), значениями которой могут быть `ALL`, `DISTINCT` или пустая строка. Однако, если вы используете `UNION` с настройкой `union_default_mode`, значением которой является пустая строка, то сгенерируется исключение. +По умолчанию, `UNION` ведет себя так же, как и `UNION DISTINCT`. Но вы можете указать особый режим объединения с помощью настройки [union_default_mode](../../../operations/settings/settings.md#union-default-mode), значениями которой могут быть `ALL`, `DISTINCT` или пустая строка. Однако если вы используете `UNION` с настройкой `union_default_mode`, значением которой является пустая строка, то будет сгенерировано исключение. ## Детали реализации {#implementation-details} -Запросы, которые являются частью `UNION ALL` выполняются параллельно, и их результаты могут быть смешаны вместе. +Запросы, которые являются частью `UNION ALL`, выполняются параллельно, и их результаты могут быть смешаны вместе. [Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/select/union-all/) From 17774583de89fe8a781fd24ee829333bf0e9b43a Mon Sep 17 00:00:00 2001 From: ana-uvarova Date: Sun, 20 Dec 2020 15:57:14 +0300 Subject: [PATCH 080/284] Russian version --- .../server-configuration-parameters/settings.md | 2 +- .../server-configuration-parameters/settings.md | 15 +++++++++++++++ docs/ru/operations/system-tables/metric_log.md | 14 +------------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 1f9302f8be3..78402e8e7a8 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -569,7 +569,7 @@ For more information, see the MergeTreeSettings.h header file. ## metric_log {#metric_log} -To turn on metrics history collection on `system.metric_log`, create `/etc/clickhouse-server/config.d/metric_log.xml` with following content: +To turn on metrics history collection on [`system.metric_log`](../../operations/system-tables/metric_log.md), create `/etc/clickhouse-server/config.d/metric_log.xml` with following content: ``` xml diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 58aae05f188..25197b57c14 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -556,6 +556,21 @@ ClickHouse проверяет условия для `min_part_size` и `min_part ``` +## metric_log {#metric_log} + +Чтобы включить сбор истории метрик в таблице [`system.metric_log`](../../operations/system-tables/metric_log.md), создайте `/etc/clickhouse-server/config.d/metric_log.xml` следующего содержания: + +``` xml + + + system + metric_log
+ 7500 + 1000 + + +``` + ## replicated\_merge\_tree {#server_configuration_parameters-replicated_merge_tree} Тонкая настройка таблиц в [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/mergetree.md). diff --git a/docs/ru/operations/system-tables/metric_log.md b/docs/ru/operations/system-tables/metric_log.md index ad25e8d5cfd..2bd23dd2c61 100644 --- a/docs/ru/operations/system-tables/metric_log.md +++ b/docs/ru/operations/system-tables/metric_log.md @@ -2,19 +2,6 @@ Содержит историю значений метрик из таблиц `system.metrics` и `system.events`, периодически сбрасываемую на диск. -Для включения сбора истории метрик в таблице `system.metric_log` создайте `/etc/clickhouse-server/config.d/metric_log.xml` следующего содержания: - -``` xml - - - system - metric_log
- 7500 - 1000 - - -``` - Столбцы: - `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата события. - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время события. @@ -55,6 +42,7 @@ CurrentMetric_ReplicatedChecks: 0 **Смотрите также** +- [Настройка metric_log](../../operations/server-configuration-parameters/settings.md#metric_log) — то, что вы должны добавить для записи истории. - [system.asynchronous_metrics](#system_tables-asynchronous_metrics) — таблица с периодически вычисляемыми метриками. - [system.events](#system_tables-events) — таблица с количеством произошедших событий. - [system.metrics](#system_tables-metrics) — таблица с мгновенно вычисляемыми метриками. From 93485260788cb83af2cdcbce667d85cbb5b13344 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sun, 15 Nov 2020 18:05:52 +0800 Subject: [PATCH 081/284] Devirtualize -If and vectorize count --- .../AggregateFunctionArray.h | 2 +- .../AggregateFunctionCount.h | 34 ++++++ .../AggregateFunctionDistinct.h | 2 + .../AggregateFunctionForEach.h | 2 + src/AggregateFunctions/AggregateFunctionIf.h | 30 +++++ .../AggregateFunctionMerge.h | 2 + .../AggregateFunctionNull.h | 8 +- .../AggregateFunctionOrFill.h | 84 +++++++++++-- .../AggregateFunctionResample.h | 2 + .../AggregateFunctionState.h | 2 +- src/AggregateFunctions/AggregateFunctionSum.h | 36 +++++- src/AggregateFunctions/IAggregateFunction.h | 112 +++++++++++++++--- src/Columns/ColumnsCommon.cpp | 68 ++++++++--- src/Columns/ColumnsCommon.h | 2 + tests/performance/countIf.xml | 3 + tests/performance/sumIf.xml | 3 + 16 files changed, 336 insertions(+), 56 deletions(-) create mode 100644 tests/performance/countIf.xml create mode 100644 tests/performance/sumIf.xml diff --git a/src/AggregateFunctions/AggregateFunctionArray.h b/src/AggregateFunctions/AggregateFunctionArray.h index 24b07010707..e72fd3ab6ff 100644 --- a/src/AggregateFunctions/AggregateFunctionArray.h +++ b/src/AggregateFunctions/AggregateFunctionArray.h @@ -129,7 +129,7 @@ public: return nested_func->allocatesMemoryInArena(); } - AggregateFunctionPtr getNestedFunction() const { return nested_func; } + AggregateFunctionPtr getNestedFunction() const override { return nested_func; } }; } diff --git a/src/AggregateFunctions/AggregateFunctionCount.h b/src/AggregateFunctions/AggregateFunctionCount.h index eb1583df92a..a8cb9005388 100644 --- a/src/AggregateFunctions/AggregateFunctionCount.h +++ b/src/AggregateFunctions/AggregateFunctionCount.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -42,6 +43,39 @@ public: ++data(place).count; } + void addBatchSinglePlace( + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena *, size_t num_arguments) const override + { + if (num_arguments > 0) + { + const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + data(place).count += countBytesInFilter(flags); + } + else + { + data(place).count += batch_size; + } + } + + void addBatchSinglePlaceNotNull( + size_t batch_size, + AggregateDataPtr place, + const IColumn ** columns, + const UInt8 * null_map, + Arena *, + size_t num_arguments) const override + { + if (num_arguments > 0) + { + const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + data(place).count += countBytesInFilterWithNull(flags, null_map); + } + else + { + data(place).count += batch_size - countBytesInFilter(null_map, batch_size); + } + } + void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override { data(place).count += data(rhs).count; diff --git a/src/AggregateFunctions/AggregateFunctionDistinct.h b/src/AggregateFunctions/AggregateFunctionDistinct.h index 01a9c71d94f..f9c8f2651dc 100644 --- a/src/AggregateFunctions/AggregateFunctionDistinct.h +++ b/src/AggregateFunctions/AggregateFunctionDistinct.h @@ -235,6 +235,8 @@ public: { return true; } + + AggregateFunctionPtr getNestedFunction() const override { return nested_func; } }; } diff --git a/src/AggregateFunctions/AggregateFunctionForEach.h b/src/AggregateFunctions/AggregateFunctionForEach.h index ee4a168cceb..c3b1b09ab3c 100644 --- a/src/AggregateFunctions/AggregateFunctionForEach.h +++ b/src/AggregateFunctions/AggregateFunctionForEach.h @@ -252,6 +252,8 @@ public: { return nested_func->isState(); } + + AggregateFunctionPtr getNestedFunction() const override { return nested_func; } }; diff --git a/src/AggregateFunctions/AggregateFunctionIf.h b/src/AggregateFunctions/AggregateFunctionIf.h index d5d2b9be0dd..8d7dced837d 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.h +++ b/src/AggregateFunctions/AggregateFunctionIf.h @@ -80,6 +80,34 @@ public: nested_func->add(place, columns, row_num, arena); } + void addBatch( + size_t batch_size, + AggregateDataPtr * places, + size_t place_offset, + const IColumn ** columns, + Arena * arena, + size_t) const override + { + nested_func->addBatch(batch_size, places, place_offset, columns, arena, num_arguments); + } + + void addBatchSinglePlace( + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t) const override + { + nested_func->addBatchSinglePlace(batch_size, place, columns, arena, num_arguments); + } + + void addBatchSinglePlaceNotNull( + size_t batch_size, + AggregateDataPtr place, + const IColumn ** columns, + const UInt8 * null_map, + Arena * arena, + size_t) const override + { + nested_func->addBatchSinglePlaceNotNull(batch_size, place, columns, null_map, arena, num_arguments); + } + void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena * arena) const override { nested_func->merge(place, rhs, arena); @@ -113,6 +141,8 @@ public: AggregateFunctionPtr getOwnNullAdapter( const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array & params, const AggregateFunctionProperties & properties) const override; + + AggregateFunctionPtr getNestedFunction() const override { return nested_func; } }; } diff --git a/src/AggregateFunctions/AggregateFunctionMerge.h b/src/AggregateFunctions/AggregateFunctionMerge.h index 066f7a762f8..721a736fcb7 100644 --- a/src/AggregateFunctions/AggregateFunctionMerge.h +++ b/src/AggregateFunctions/AggregateFunctionMerge.h @@ -102,6 +102,8 @@ public: { return nested_func->allocatesMemoryInArena(); } + + AggregateFunctionPtr getNestedFunction() const override { return nested_func; } }; } diff --git a/src/AggregateFunctions/AggregateFunctionNull.h b/src/AggregateFunctions/AggregateFunctionNull.h index 2f34aef0932..79bebb9caa8 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.h +++ b/src/AggregateFunctions/AggregateFunctionNull.h @@ -180,6 +180,8 @@ public: { return nested_function->isState(); } + + AggregateFunctionPtr getNestedFunction() const override { return nested_function; } }; @@ -209,13 +211,15 @@ public: } } - void addBatchSinglePlace(size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena) const override + void addBatchSinglePlace( + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t num_arguments = 0) const override { const ColumnNullable * column = assert_cast(columns[0]); const IColumn * nested_column = &column->getNestedColumn(); const UInt8 * null_map = column->getNullMapData().data(); - this->nested_function->addBatchSinglePlaceNotNull(batch_size, this->nestedPlace(place), &nested_column, null_map, arena); + this->nested_function->addBatchSinglePlaceNotNull( + batch_size, this->nestedPlace(place), &nested_column, null_map, arena, num_arguments); if constexpr (result_is_nullable) if (!memoryIsByte(null_map, batch_size, 1)) diff --git a/src/AggregateFunctions/AggregateFunctionOrFill.h b/src/AggregateFunctions/AggregateFunctionOrFill.h index 456c0e22305..13a76e29d63 100644 --- a/src/AggregateFunctions/AggregateFunctionOrFill.h +++ b/src/AggregateFunctions/AggregateFunctionOrFill.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -96,37 +97,94 @@ public: place[size_of_data] = 1; } - void addBatch(size_t batch_size, AggregateDataPtr * places, size_t place_offset, const IColumn ** columns, Arena * arena) const override + void addBatch( + size_t batch_size, + AggregateDataPtr * places, + size_t place_offset, + const IColumn ** columns, + Arena * arena, + size_t num_arguments = 0) const override { - nested_function->addBatch(batch_size, places, place_offset, columns, arena); - for (size_t i = 0; i < batch_size; ++i) - (places[i] + place_offset)[size_of_data] = 1; + // TODO we can devirtualize this too + if (num_arguments > 0) + { + const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + for (size_t i = 0; i < batch_size; ++i) + { + if (flags[i]) + add(places[i] + place_offset, columns, i, arena); + } + } + else + { + nested_function->addBatch(batch_size, places, place_offset, columns, arena, num_arguments); + for (size_t i = 0; i < batch_size; ++i) + (places[i] + place_offset)[size_of_data] = 1; + } } - void addBatchSinglePlace(size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena) const override + void addBatchSinglePlace( + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t num_arguments = 0) const override { - if (batch_size) + if (num_arguments > 0) { - nested_function->addBatchSinglePlace(batch_size, place, columns, arena); - place[size_of_data] = 1; + const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + nested_function->addBatchSinglePlace(batch_size, place, columns, arena, num_arguments); + for (size_t i = 0; i < batch_size; ++i) + { + if (flags[i]) + { + place[size_of_data] = 1; + break; + } + } + } + else + { + if (batch_size) + { + nested_function->addBatchSinglePlace(batch_size, place, columns, arena, num_arguments); + place[size_of_data] = 1; + } } } void addBatchSinglePlaceNotNull( - size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena * arena) const override + size_t batch_size, + AggregateDataPtr place, + const IColumn ** columns, + const UInt8 * null_map, + Arena * arena, + size_t num_arguments = 0) const override { - if (batch_size) + if (num_arguments > 0) { - nested_function->addBatchSinglePlaceNotNull(batch_size, place, columns, null_map, arena); + const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + nested_function->addBatchSinglePlaceNotNull(batch_size, place, columns, null_map, arena, num_arguments); for (size_t i = 0; i < batch_size; ++i) { - if (!null_map[i]) + if (flags[i] && !null_map[i]) { place[size_of_data] = 1; break; } } } + else + { + if (batch_size) + { + nested_function->addBatchSinglePlaceNotNull(batch_size, place, columns, null_map, arena, num_arguments); + for (size_t i = 0; i < batch_size; ++i) + { + if (!null_map[i]) + { + place[size_of_data] = 1; + break; + } + } + } + } } void merge( @@ -207,6 +265,8 @@ public: else to.insertDefault(); } + + AggregateFunctionPtr getNestedFunction() const override { return nested_function; } }; } diff --git a/src/AggregateFunctions/AggregateFunctionResample.h b/src/AggregateFunctions/AggregateFunctionResample.h index c1528686785..51252fe0b89 100644 --- a/src/AggregateFunctions/AggregateFunctionResample.h +++ b/src/AggregateFunctions/AggregateFunctionResample.h @@ -198,6 +198,8 @@ public: col_offsets.getData().push_back(col.getData().size()); } + + AggregateFunctionPtr getNestedFunction() const override { return nested_function; } }; } diff --git a/src/AggregateFunctions/AggregateFunctionState.h b/src/AggregateFunctions/AggregateFunctionState.h index 2f70025c053..01614b7f0ab 100644 --- a/src/AggregateFunctions/AggregateFunctionState.h +++ b/src/AggregateFunctions/AggregateFunctionState.h @@ -92,7 +92,7 @@ public: return nested_func->allocatesMemoryInArena(); } - AggregateFunctionPtr getNestedFunction() const { return nested_func; } + AggregateFunctionPtr getNestedFunction() const override { return nested_func; } }; } diff --git a/src/AggregateFunctions/AggregateFunctionSum.h b/src/AggregateFunctions/AggregateFunctionSum.h index 9c453a63fc8..1a207cbff0b 100644 --- a/src/AggregateFunctions/AggregateFunctionSum.h +++ b/src/AggregateFunctions/AggregateFunctionSum.h @@ -282,17 +282,41 @@ public: } /// Vectorized version when there is no GROUP BY keys. - void addBatchSinglePlace(size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena *) const override + void addBatchSinglePlace( + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t num_arguments) const override { - const auto & column = static_cast(*columns[0]); - this->data(place).addMany(column.getData().data(), batch_size); + if (num_arguments > 0) + { + const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + for (size_t i = 0; i < batch_size; ++i) + { + if (flags[i]) + add(place, columns, i, arena); + } + } + else + { + const auto & column = static_cast(*columns[0]); + this->data(place).addMany(column.getData().data(), batch_size); + } } void addBatchSinglePlaceNotNull( - size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena *) const override + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena * arena, size_t num_arguments) + const override { - const auto & column = static_cast(*columns[0]); - this->data(place).addManyNotNull(column.getData().data(), null_map, batch_size); + if (num_arguments > 0) + { + const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + for (size_t i = 0; i < batch_size; ++i) + if (!null_map[i] && flags[i]) + add(place, columns, i, arena); + } + else + { + const auto & column = static_cast(*columns[0]); + this->data(place).addManyNotNull(column.getData().data(), null_map, batch_size); + } } void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index b591bd3acd7..8f8fba23707 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB @@ -140,19 +141,32 @@ public: /** Contains a loop with calls to "add" function. You can collect arguments into array "places" * and do a single call to "addBatch" for devirtualization and inlining. */ - virtual void addBatch(size_t batch_size, AggregateDataPtr * places, size_t place_offset, const IColumn ** columns, Arena * arena) const = 0; + virtual void addBatch( + size_t batch_size, + AggregateDataPtr * places, + size_t place_offset, + const IColumn ** columns, + Arena * arena, + size_t num_arguments = 0) const = 0; /** The same for single place. */ - virtual void addBatchSinglePlace(size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena) const = 0; + virtual void addBatchSinglePlace( + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t num_arguments = 0) const = 0; /** The same for single place when need to aggregate only filtered data. */ virtual void addBatchSinglePlaceNotNull( - size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena * arena) const = 0; + size_t batch_size, + AggregateDataPtr place, + const IColumn ** columns, + const UInt8 * null_map, + Arena * arena, + size_t num_arguments = 0) const = 0; virtual void addBatchSinglePlaceFromInterval( - size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena) const = 0; + size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t num_arguments = 0) + const = 0; /** In addition to addBatch, this method collects multiple rows of arguments into array "places" * as long as they are between offsets[i-1] and offsets[i]. This is used for arrayReduce and @@ -192,6 +206,11 @@ public: return nullptr; } + /** Return the nested function if this is an Aggregate Function Combinator. + * Otherwise return nullptr. + */ + virtual AggregateFunctionPtr getNestedFunction() const { return {}; } + const DataTypes & getArgumentTypes() const { return argument_types; } const Array & getParameters() const { return parameters; } @@ -217,31 +236,90 @@ public: AddFunc getAddressOfAddFunction() const override { return &addFree; } - void addBatch(size_t batch_size, AggregateDataPtr * places, size_t place_offset, const IColumn ** columns, Arena * arena) const override + void addBatch( + size_t batch_size, + AggregateDataPtr * places, + size_t place_offset, + const IColumn ** columns, + Arena * arena, + size_t num_arguments = 0) const override { - for (size_t i = 0; i < batch_size; ++i) - static_cast(this)->add(places[i] + place_offset, columns, i, arena); + if (num_arguments > 0) + { + const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + for (size_t i = 0; i < batch_size; ++i) + { + if (flags[i]) + static_cast(this)->add(places[i] + place_offset, columns, i, arena); + } + } + else + { + for (size_t i = 0; i < batch_size; ++i) + static_cast(this)->add(places[i] + place_offset, columns, i, arena); + } } - void addBatchSinglePlace(size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena) const override + void addBatchSinglePlace( + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t num_arguments = 0) const override { - for (size_t i = 0; i < batch_size; ++i) - static_cast(this)->add(place, columns, i, arena); + if (num_arguments > 0) + { + const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + for (size_t i = 0; i < batch_size; ++i) + { + if (flags[i]) + static_cast(this)->add(place, columns, i, arena); + } + } + else + { + for (size_t i = 0; i < batch_size; ++i) + static_cast(this)->add(place, columns, i, arena); + } } void addBatchSinglePlaceNotNull( - size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena * arena) const override + size_t batch_size, + AggregateDataPtr place, + const IColumn ** columns, + const UInt8 * null_map, + Arena * arena, + size_t num_arguments = 0) const override { - for (size_t i = 0; i < batch_size; ++i) - if (!null_map[i]) - static_cast(this)->add(place, columns, i, arena); + if (num_arguments > 0) + { + const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + for (size_t i = 0; i < batch_size; ++i) + if (!null_map[i] && flags[i]) + static_cast(this)->add(place, columns, i, arena); + } + else + { + for (size_t i = 0; i < batch_size; ++i) + if (!null_map[i]) + static_cast(this)->add(place, columns, i, arena); + } } void addBatchSinglePlaceFromInterval( - size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena) const override + size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t num_arguments = 0) + const override { - for (size_t i = batch_begin; i < batch_end; ++i) - static_cast(this)->add(place, columns, i, arena); + if (num_arguments > 0) + { + const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + for (size_t i = batch_begin; i < batch_end; ++i) + { + if (flags[i]) + static_cast(this)->add(place, columns, i, arena); + } + } + else + { + for (size_t i = batch_begin; i < batch_end; ++i) + static_cast(this)->add(place, columns, i, arena); + } } void addBatchArray( diff --git a/src/Columns/ColumnsCommon.cpp b/src/Columns/ColumnsCommon.cpp index 14ace8f8a4b..0c4d3967c6e 100644 --- a/src/Columns/ColumnsCommon.cpp +++ b/src/Columns/ColumnsCommon.cpp @@ -12,7 +12,53 @@ namespace DB { +#if defined(__SSE2__) && defined(__POPCNT__) +auto toBits64(const Int8 * bytes64) +{ + static const __m128i zero16 = _mm_setzero_si128(); + return static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8(_mm_loadu_si128(reinterpret_cast(bytes64)), zero16))) + | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8(_mm_loadu_si128(reinterpret_cast(bytes64 + 16)), zero16))) + << 16) + | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8(_mm_loadu_si128(reinterpret_cast(bytes64 + 32)), zero16))) + << 32) + | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8(_mm_loadu_si128(reinterpret_cast(bytes64 + 48)), zero16))) + << 48); +}; +#endif + +size_t countBytesInFilter(const UInt8 * filt, size_t sz) +{ + size_t count = 0; + + /** NOTE: In theory, `filt` should only contain zeros and ones. + * But, just in case, here the condition > 0 (to signed bytes) is used. + * It would be better to use != 0, then this does not allow SSE2. + */ + + const Int8 * pos = reinterpret_cast(filt); + const Int8 * end = pos + sz; + +#if defined(__SSE2__) && defined(__POPCNT__) + const Int8 * end64 = pos + sz / 64 * 64; + + for (; pos < end64; pos += 64) + count += __builtin_popcountll(toBits64(pos)); + + /// TODO Add duff device for tail? +#endif + + for (; pos < end; ++pos) + count += *pos > 0; + + return count; +} + size_t countBytesInFilter(const IColumn::Filter & filt) +{ + return countBytesInFilter(filt.data(), filt.size()); +} + +size_t countBytesInFilterWithNull(const IColumn::Filter & filt, const UInt8 * null_map) { size_t count = 0; @@ -22,32 +68,20 @@ size_t countBytesInFilter(const IColumn::Filter & filt) */ const Int8 * pos = reinterpret_cast(filt.data()); + const Int8 * pos2 = reinterpret_cast(null_map); const Int8 * end = pos + filt.size(); #if defined(__SSE2__) && defined(__POPCNT__) - const __m128i zero16 = _mm_setzero_si128(); const Int8 * end64 = pos + filt.size() / 64 * 64; - for (; pos < end64; pos += 64) - count += __builtin_popcountll( - static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( - _mm_loadu_si128(reinterpret_cast(pos)), - zero16))) - | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( - _mm_loadu_si128(reinterpret_cast(pos + 16)), - zero16))) << 16) - | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( - _mm_loadu_si128(reinterpret_cast(pos + 32)), - zero16))) << 32) - | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( - _mm_loadu_si128(reinterpret_cast(pos + 48)), - zero16))) << 48)); + for (; pos < end64; pos += 64, pos2 += 64) + count += __builtin_popcountll(toBits64(pos) & ~toBits64(pos2)); - /// TODO Add duff device for tail? + /// TODO Add duff device for tail? #endif for (; pos < end; ++pos) - count += *pos > 0; + count += (*pos & ~*pos2) > 0; return count; } diff --git a/src/Columns/ColumnsCommon.h b/src/Columns/ColumnsCommon.h index 6380aeb831d..7655edffa71 100644 --- a/src/Columns/ColumnsCommon.h +++ b/src/Columns/ColumnsCommon.h @@ -15,7 +15,9 @@ namespace ErrorCodes } /// Counts how many bytes of `filt` are greater than zero. +size_t countBytesInFilter(const UInt8 * filt, size_t sz); size_t countBytesInFilter(const IColumn::Filter & filt); +size_t countBytesInFilterWithNull(const IColumn::Filter & filt, const UInt8 * null_map); /// Returns vector with num_columns elements. vector[i] is the count of i values in selector. /// Selector must contain values from 0 to num_columns - 1. NOTE: this is not checked. diff --git a/tests/performance/countIf.xml b/tests/performance/countIf.xml new file mode 100644 index 00000000000..a2209d5ed22 --- /dev/null +++ b/tests/performance/countIf.xml @@ -0,0 +1,3 @@ + + SELECT countIf(number % 2) FROM numbers(100000000) + diff --git a/tests/performance/sumIf.xml b/tests/performance/sumIf.xml new file mode 100644 index 00000000000..26a3adafdea --- /dev/null +++ b/tests/performance/sumIf.xml @@ -0,0 +1,3 @@ + + SELECT sumIf(number, number % 2) FROM numbers(100000000) + From ae802574b2ff864703c8865ff49d6256f5d0b56d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 26 Nov 2020 07:29:06 +0300 Subject: [PATCH 082/284] Update ColumnsCommon.cpp --- src/Columns/ColumnsCommon.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Columns/ColumnsCommon.cpp b/src/Columns/ColumnsCommon.cpp index 0c4d3967c6e..f3f10a25df3 100644 --- a/src/Columns/ColumnsCommon.cpp +++ b/src/Columns/ColumnsCommon.cpp @@ -13,7 +13,8 @@ namespace DB { #if defined(__SSE2__) && defined(__POPCNT__) -auto toBits64(const Int8 * bytes64) +/// Transform 64-byte mask to 64-bit mask. +static UInt64 toBits64(const Int8 * bytes64) { static const __m128i zero16 = _mm_setzero_si128(); return static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8(_mm_loadu_si128(reinterpret_cast(bytes64)), zero16))) @@ -23,7 +24,7 @@ auto toBits64(const Int8 * bytes64) << 32) | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8(_mm_loadu_si128(reinterpret_cast(bytes64 + 48)), zero16))) << 48); -}; +} #endif size_t countBytesInFilter(const UInt8 * filt, size_t sz) From b8fa55582d4399075d4a23807ae8bc6e086da836 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sat, 12 Dec 2020 21:56:03 +0800 Subject: [PATCH 083/284] Trigger CI again From 282dd2358798eb21522b9d339b6c6f83c9018b86 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 21 Dec 2020 12:46:31 +0800 Subject: [PATCH 084/284] Split minmaxany --- ...MinMaxAny.cpp => AggregateFunctionAny.cpp} | 27 --------------- .../AggregateFunctionMax.cpp | 34 +++++++++++++++++++ .../AggregateFunctionMin.cpp | 34 +++++++++++++++++++ .../AggregateFunctionSimpleState.h | 2 +- 4 files changed, 69 insertions(+), 28 deletions(-) rename src/AggregateFunctions/{AggregateFunctionMinMaxAny.cpp => AggregateFunctionAny.cpp} (50%) create mode 100644 src/AggregateFunctions/AggregateFunctionMax.cpp create mode 100644 src/AggregateFunctions/AggregateFunctionMin.cpp diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.cpp b/src/AggregateFunctions/AggregateFunctionAny.cpp similarity index 50% rename from src/AggregateFunctions/AggregateFunctionMinMaxAny.cpp rename to src/AggregateFunctions/AggregateFunctionAny.cpp index a98eaccdabd..b237432c2e8 100644 --- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.cpp +++ b/src/AggregateFunctions/AggregateFunctionAny.cpp @@ -25,42 +25,15 @@ AggregateFunctionPtr createAggregateFunctionAnyHeavy(const std::string & name, c return AggregateFunctionPtr(createAggregateFunctionSingleValue(name, argument_types, parameters)); } -AggregateFunctionPtr createAggregateFunctionMin(const std::string & name, const DataTypes & argument_types, const Array & parameters) -{ - return AggregateFunctionPtr(createAggregateFunctionSingleValue(name, argument_types, parameters)); -} - -AggregateFunctionPtr createAggregateFunctionMax(const std::string & name, const DataTypes & argument_types, const Array & parameters) -{ - return AggregateFunctionPtr(createAggregateFunctionSingleValue(name, argument_types, parameters)); -} - -AggregateFunctionPtr createAggregateFunctionArgMin(const std::string & name, const DataTypes & argument_types, const Array & parameters) -{ - return AggregateFunctionPtr(createAggregateFunctionArgMinMax(name, argument_types, parameters)); -} - -AggregateFunctionPtr createAggregateFunctionArgMax(const std::string & name, const DataTypes & argument_types, const Array & parameters) -{ - return AggregateFunctionPtr(createAggregateFunctionArgMinMax(name, argument_types, parameters)); -} - } void registerAggregateFunctionsMinMaxAny(AggregateFunctionFactory & factory) { - factory.registerFunction("min", createAggregateFunctionMin, AggregateFunctionFactory::CaseInsensitive); - factory.registerFunction("max", createAggregateFunctionMax, AggregateFunctionFactory::CaseInsensitive); - - /// The functions below depend on the order of data. - AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true }; factory.registerFunction("any", { createAggregateFunctionAny, properties }); factory.registerFunction("anyLast", { createAggregateFunctionAnyLast, properties }); factory.registerFunction("anyHeavy", { createAggregateFunctionAnyHeavy, properties }); - factory.registerFunction("argMin", { createAggregateFunctionArgMin, properties }); - factory.registerFunction("argMax", { createAggregateFunctionArgMax, properties }); } } diff --git a/src/AggregateFunctions/AggregateFunctionMax.cpp b/src/AggregateFunctions/AggregateFunctionMax.cpp new file mode 100644 index 00000000000..5ba35cfea38 --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionMax.cpp @@ -0,0 +1,34 @@ +#include +#include +#include +#include "registerAggregateFunctions.h" + + +namespace DB +{ + +namespace +{ + +AggregateFunctionPtr createAggregateFunctionMax(const std::string & name, const DataTypes & argument_types, const Array & parameters) +{ + return AggregateFunctionPtr(createAggregateFunctionSingleValue(name, argument_types, parameters)); +} + +AggregateFunctionPtr createAggregateFunctionArgMax(const std::string & name, const DataTypes & argument_types, const Array & parameters) +{ + return AggregateFunctionPtr(createAggregateFunctionArgMinMax(name, argument_types, parameters)); +} + +} + +void registerAggregateFunctionsMinMaxAny(AggregateFunctionFactory & factory) +{ + factory.registerFunction("max", createAggregateFunctionMax, AggregateFunctionFactory::CaseInsensitive); + + /// The functions below depend on the order of data. + AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true }; + factory.registerFunction("argMax", { createAggregateFunctionArgMax, properties }); +} + +} diff --git a/src/AggregateFunctions/AggregateFunctionMin.cpp b/src/AggregateFunctions/AggregateFunctionMin.cpp new file mode 100644 index 00000000000..c5bffe4f74b --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionMin.cpp @@ -0,0 +1,34 @@ +#include +#include +#include +#include "registerAggregateFunctions.h" + + +namespace DB +{ + +namespace +{ + +AggregateFunctionPtr createAggregateFunctionMin(const std::string & name, const DataTypes & argument_types, const Array & parameters) +{ + return AggregateFunctionPtr(createAggregateFunctionSingleValue(name, argument_types, parameters)); +} + +AggregateFunctionPtr createAggregateFunctionArgMin(const std::string & name, const DataTypes & argument_types, const Array & parameters) +{ + return AggregateFunctionPtr(createAggregateFunctionArgMinMax(name, argument_types, parameters)); +} + +} + +void registerAggregateFunctionsMinMaxAny(AggregateFunctionFactory & factory) +{ + factory.registerFunction("min", createAggregateFunctionMin, AggregateFunctionFactory::CaseInsensitive); + + /// The functions below depend on the order of data. + AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true }; + factory.registerFunction("argMin", { createAggregateFunctionArgMin, properties }); +} + +} diff --git a/src/AggregateFunctions/AggregateFunctionSimpleState.h b/src/AggregateFunctions/AggregateFunctionSimpleState.h index eefe00e77c0..f60df1eddd3 100644 --- a/src/AggregateFunctions/AggregateFunctionSimpleState.h +++ b/src/AggregateFunctions/AggregateFunctionSimpleState.h @@ -71,7 +71,7 @@ public: bool allocatesMemoryInArena() const override { return nested_func->allocatesMemoryInArena(); } - AggregateFunctionPtr getNestedFunction() const { return nested_func; } + AggregateFunctionPtr getNestedFunction() const override { return nested_func; } }; } From 47b177d175fd0fc0d75508c0e721e33fd494b235 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 21 Dec 2020 13:51:11 +0800 Subject: [PATCH 085/284] Fix build --- src/AggregateFunctions/AggregateFunctionAny.cpp | 2 +- src/AggregateFunctions/AggregateFunctionMax.cpp | 2 +- src/AggregateFunctions/AggregateFunctionMin.cpp | 2 +- src/AggregateFunctions/registerAggregateFunctions.cpp | 8 ++++++-- src/AggregateFunctions/ya.make | 4 +++- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionAny.cpp b/src/AggregateFunctions/AggregateFunctionAny.cpp index b237432c2e8..0aeb2548af9 100644 --- a/src/AggregateFunctions/AggregateFunctionAny.cpp +++ b/src/AggregateFunctions/AggregateFunctionAny.cpp @@ -27,7 +27,7 @@ AggregateFunctionPtr createAggregateFunctionAnyHeavy(const std::string & name, c } -void registerAggregateFunctionsMinMaxAny(AggregateFunctionFactory & factory) +void registerAggregateFunctionsAny(AggregateFunctionFactory & factory) { AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true }; diff --git a/src/AggregateFunctions/AggregateFunctionMax.cpp b/src/AggregateFunctions/AggregateFunctionMax.cpp index 5ba35cfea38..921ea4179c9 100644 --- a/src/AggregateFunctions/AggregateFunctionMax.cpp +++ b/src/AggregateFunctions/AggregateFunctionMax.cpp @@ -22,7 +22,7 @@ AggregateFunctionPtr createAggregateFunctionArgMax(const std::string & name, con } -void registerAggregateFunctionsMinMaxAny(AggregateFunctionFactory & factory) +void registerAggregateFunctionsMax(AggregateFunctionFactory & factory) { factory.registerFunction("max", createAggregateFunctionMax, AggregateFunctionFactory::CaseInsensitive); diff --git a/src/AggregateFunctions/AggregateFunctionMin.cpp b/src/AggregateFunctions/AggregateFunctionMin.cpp index c5bffe4f74b..fec5be484e9 100644 --- a/src/AggregateFunctions/AggregateFunctionMin.cpp +++ b/src/AggregateFunctions/AggregateFunctionMin.cpp @@ -22,7 +22,7 @@ AggregateFunctionPtr createAggregateFunctionArgMin(const std::string & name, con } -void registerAggregateFunctionsMinMaxAny(AggregateFunctionFactory & factory) +void registerAggregateFunctionsMin(AggregateFunctionFactory & factory) { factory.registerFunction("min", createAggregateFunctionMin, AggregateFunctionFactory::CaseInsensitive); diff --git a/src/AggregateFunctions/registerAggregateFunctions.cpp b/src/AggregateFunctions/registerAggregateFunctions.cpp index 228ca2f30ab..d8e4eb7ba98 100644 --- a/src/AggregateFunctions/registerAggregateFunctions.cpp +++ b/src/AggregateFunctions/registerAggregateFunctions.cpp @@ -18,7 +18,9 @@ void registerAggregateFunctionsQuantile(AggregateFunctionFactory &); void registerAggregateFunctionsSequenceMatch(AggregateFunctionFactory &); void registerAggregateFunctionWindowFunnel(AggregateFunctionFactory &); void registerAggregateFunctionRate(AggregateFunctionFactory &); -void registerAggregateFunctionsMinMaxAny(AggregateFunctionFactory &); +void registerAggregateFunctionsMin(AggregateFunctionFactory &); +void registerAggregateFunctionsMax(AggregateFunctionFactory &); +void registerAggregateFunctionsAny(AggregateFunctionFactory &); void registerAggregateFunctionsStatisticsStable(AggregateFunctionFactory &); void registerAggregateFunctionsStatisticsSimple(AggregateFunctionFactory &); void registerAggregateFunctionSum(AggregateFunctionFactory &); @@ -71,7 +73,9 @@ void registerAggregateFunctions() registerAggregateFunctionsSequenceMatch(factory); registerAggregateFunctionWindowFunnel(factory); registerAggregateFunctionRate(factory); - registerAggregateFunctionsMinMaxAny(factory); + registerAggregateFunctionsMin(factory); + registerAggregateFunctionsMax(factory); + registerAggregateFunctionsAny(factory); registerAggregateFunctionsStatisticsStable(factory); registerAggregateFunctionsStatisticsSimple(factory); registerAggregateFunctionSum(factory); diff --git a/src/AggregateFunctions/ya.make b/src/AggregateFunctions/ya.make index 8faf738fd98..f2105688feb 100644 --- a/src/AggregateFunctions/ya.make +++ b/src/AggregateFunctions/ya.make @@ -10,6 +10,7 @@ PEERDIR( SRCS( AggregateFunctionAggThrow.cpp + AggregateFunctionAny.cpp AggregateFunctionArray.cpp AggregateFunctionAvg.cpp AggregateFunctionAvgWeighted.cpp @@ -30,9 +31,10 @@ SRCS( AggregateFunctionIf.cpp AggregateFunctionMLMethod.cpp AggregateFunctionMannWhitney.cpp + AggregateFunctionMax.cpp AggregateFunctionMaxIntersections.cpp AggregateFunctionMerge.cpp - AggregateFunctionMinMaxAny.cpp + AggregateFunctionMin.cpp AggregateFunctionNull.cpp AggregateFunctionOrFill.cpp AggregateFunctionQuantile.cpp From 24d01d2460cac741723424064c65088ee21ff5b9 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 21 Dec 2020 18:31:32 +0800 Subject: [PATCH 086/284] translate documentation for interfaces/formats --- docs/zh/interfaces/formats.md | 845 ++++++++++++++++-------- docs/zh/interfaces/third-party/index.md | 14 +- 2 files changed, 574 insertions(+), 285 deletions(-) diff --git a/docs/zh/interfaces/formats.md b/docs/zh/interfaces/formats.md index 58d06916ed8..808bff42335 100644 --- a/docs/zh/interfaces/formats.md +++ b/docs/zh/interfaces/formats.md @@ -1,179 +1,237 @@ -# 输入输出格式 {#formats} +--- +toc_priority: 21 +toc_title: 输入/输出格式 +--- -ClickHouse 可以接受多种数据格式，可以在 (`INSERT`) 以及 (`SELECT`) 请求中使用。 +# 输入/输出格式 {#formats} -下列表格列出了支持的数据格式以及在 (`INSERT`) 以及 (`SELECT`) 请求中使用它们的方式。 +ClickHouse可以接受和返回各种格式的数据。输入支持的格式可以用来解析提供给`INSERT`的数据，可以从文件备份表(如File, URL或HDFS)执行`SELECT`，或者读取外部字典。输出支持的格式可用于获取`SELECT`的结果，并支持执行`INSERT`文件的表中。 -| 格式 | INSERT | SELECT | -|-----------------------------------------------------------------|--------|--------| -| [TabSeparated](#tabseparated) | ✔ | ✔ | -| [TabSeparatedRaw](#tabseparatedraw) | ✗ | ✔ | -| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ | -| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ | -| [模板](#format-template) | ✔ | ✔ | -| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ | -| [CSV](#csv) | ✔ | ✔ | -| [CSVWithNames](#csvwithnames) | ✔ | ✔ | -| [自定义分离](#format-customseparated) | ✔ | ✔ | -| [值](#data-format-values) | ✔ | ✔ | -| [垂直](#vertical) | ✗ | ✔ | -| VerticalRaw | ✗ | ✔ | -| [JSON](#json) | ✗ | ✔ | -| [JSONCompact](#jsoncompact) | ✗ | ✔ | -| [JSONEachRow](#jsoneachrow) | ✔ | ✔ | -| [TSKV](#tskv) | ✔ | ✔ | -| [漂亮](#pretty) | ✗ | ✔ | -| [PrettyCompact](#prettycompact) | ✗ | ✔ | -| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ | -| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ | -| [PrettySpace](#prettyspace) | ✗ | ✔ | -| [Protobuf](#protobuf) | ✔ | ✔ | -| [Avro](#data-format-avro) | ✔ | ✔ | -| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ | -| [镶木地板](#data-format-parquet) | ✔ | ✔ | -| [ORC](#data-format-orc) | ✔ | ✗ | -| [RowBinary](#rowbinary) | ✔ | ✔ | -| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | -| [本地人](#native) | ✔ | ✔ | -| [Null](#null) | ✗ | ✔ | -| [XML](#xml) | ✗ | ✔ | -| [CapnProto](#capnproto) | ✔ | ✔ | +以下是支持的格式: + +| 格式 | 输入 | 输出 | +|-----------------------------------------------------------------------------------------|-------|--------| +| [TabSeparated](#tabseparated) | ✔ | ✔ | +| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ | +| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ | +| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ | +| [Template](#format-template) | ✔ | ✔ | +| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ | +| [CSV](#csv) | ✔ | ✔ | +| [CSVWithNames](#csvwithnames) | ✔ | ✔ | +| [CustomSeparated](#format-customseparated) | ✔ | ✔ | +| [Values](#data-format-values) | ✔ | ✔ | +| [Vertical](#vertical) | ✗ | ✔ | +| [VerticalRaw](#verticalraw) | ✗ | ✔ | +| [JSON](#json) | ✗ | ✔ | +| [JSONAsString](#jsonasstring) | ✔ | ✗ | +| [JSONString](#jsonstring) | ✗ | ✔ | +| [JSONCompact](#jsoncompact) | ✗ | ✔ | +| [JSONCompactString](#jsoncompactstring) | ✗ | ✔ | +| [JSONEachRow](#jsoneachrow) | ✔ | ✔ | +| [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ | +| [JSONStringEachRow](#jsonstringeachrow) | ✔ | ✔ | +| [JSONStringEachRowWithProgress](#jsonstringeachrowwithprogress) | ✗ | ✔ | +| [JSONCompactEachRow](#jsoncompacteachrow) | ✔ | ✔ | +| [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes) | ✔ | ✔ | +| [JSONCompactStringEachRow](#jsoncompactstringeachrow) | ✔ | ✔ | +| [JSONCompactStringEachRowWithNamesAndTypes](#jsoncompactstringeachrowwithnamesandtypes) | ✔ | ✔ | +| [TSKV](#tskv) | ✔ | ✔ | +| [Pretty](#pretty) | ✗ | ✔ | +| [PrettyCompact](#prettycompact) | ✗ | ✔ | +| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ | +| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ | +| [PrettySpace](#prettyspace) | ✗ | ✔ | +| [Protobuf](#protobuf) | ✔ | ✔ | +| [ProtobufSingle](#protobufsingle) | ✔ | ✔ | +| [Avro](#data-format-avro) | ✔ | ✔ | +| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ | +| [Parquet](#data-format-parquet) | ✔ | ✔ | +| [Arrow](#data-format-arrow) | ✔ | ✔ | +| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ | +| [ORC](#data-format-orc) | ✔ | ✗ | +| [RowBinary](#rowbinary) | ✔ | ✔ | +| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | +| [Native](#native) | ✔ | ✔ | +| [Null](#null) | ✗ | ✔ | +| [XML](#xml) | ✗ | ✔ | +| [CapnProto](#capnproto) | ✔ | ✗ | +| [LineAsString](#lineasstring) | ✔ | ✗ | + +您可以使用ClickHouse设置控制一些格式处理参数。更多详情设置请参考[设置](../operations/settings/settings.md) ## TabSeparated {#tabseparated} -在 TabSeparated 格式中，数据按行写入。每行包含由制表符分隔的值。除了行中的最后一个值（后面紧跟换行符）之外，每个值都跟随一个制表符。 在任何地方都可以使用严格的 Unix 命令行。最后一行还必须在最后包含换行符。值以文本格式编写，不包含引号，并且要转义特殊字符。 +在TabSeparated分隔格式中，数据按行写入。每行包含由制表符分隔的值。每个值后跟一个制表符，除了行中最后一个值后跟换行。在任何地方都采用严格的Unix换行。最后一行还必须在末尾包含换行。值以文本格式编写，不包含引号，并使用转义的特殊字符。 -这种格式也可以用 `TSV` 来表示。 +这种格式也可以用`TSV`来表示。 -TabSeparated 格式非常方便用于自定义程序或脚本处理数据。HTTP 客户端接口默认会用这种格式，命令行客户端批量模式下也会用这种格式。这种格式允许在不同数据库之间传输数据。例如，从 MYSQL 中导出数据然后导入到 ClickHouse 中，反之亦然。 +`TabSeparated`格式便于使用自定义程序和脚本处理数据。默认情况下，它在HTTP接口和命令行客户端的批处理模式中使用。这种格式还允许在不同dbms之间传输数据。例如，您可以从MySQL获取转储并将其上传到ClickHouse，反之亦然。 -TabSeparated 格式支持输出数据总值（当使用 WITH TOTALS） 以及极值（当 ‘extremes’ 设置是1）。这种情况下，总值和极值输出在主数据的后面。主要的数据，总值，极值会以一个空行隔开，例如： +`TabSeparated`格式支持输出total值(与TOTALS一起使用时)和extreme值(当`extreme`被设置为1时)。在这种情况下，total值和extreme值会在主数据后输出。主要结果、总值和极值之间用空行分隔。示例: ``` sql SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT TabSeparated`` ``` - 2014-03-17 1406958 - 2014-03-18 1383658 - 2014-03-19 1405797 - 2014-03-20 1353623 - 2014-03-21 1245779 - 2014-03-22 1031592 - 2014-03-23 1046491 +``` text +2014-03-17 1406958 +2014-03-18 1383658 +2014-03-19 1405797 +2014-03-20 1353623 +2014-03-21 1245779 +2014-03-22 1031592 +2014-03-23 1046491 - 1970-01-01 8873898 +1970-01-01 8873898 - 2014-03-17 1031592 - 2014-03-23 1406958 +2014-03-17 1031592 +2014-03-23 1406958 +``` -### 数据解析方式 {#shu-ju-jie-xi-fang-shi} +### 数据格式化 {#data-formatting} -整数以十进制形式写入。数字在开头可以包含额外的 `+` 字符（解析时忽略，格式化时不记录）。非负数不能包含负号。 读取时，允许将空字符串解析为零，或者（对于带符号的类型）将仅包含负号的字符串解析为零。 不符合相应数据类型的数字可能会被解析为不同的数字，而不会显示错误消息。 +整数是用十进制形式写的。数字可以在开头包含一个额外的`+`字符(解析时忽略，格式化时不记录)。非负数不能包含负号。在读取时，允许将空字符串解析为零，或者(对于有符号类型)将仅由一个负号组成的字符串解析为零。不符合相应数据类型的数字可以被解析为不同的数字，而不会出现错误消息。 -浮点数以十进制形式写入。点号用作小数点分隔符。支持指数等符号，如’inf’，‘+ inf’，‘-inf’和’nan’。 浮点数的输入可以以小数点开始或结束。 -格式化的时候，浮点数的精确度可能会丢失。 -解析的时候，没有严格需要去读取与机器可以表示的最接近的数值。 +浮点数以十进制形式书写。`.`号用作十进制分隔符。支持指数符号，如`inf`、`+inf`、`-inf`和`nan`。浮点数的条目可以以小数点开始或结束。 +在格式化期间，浮点数可能会丢失准确性。 +在解析期间，并不严格要求读取与机器可以表示的最接近的数值。 -日期会以 YYYY-MM-DD 格式写入和解析，但会以任何字符作为分隔符。 -带时间的日期会以 YYYY-MM-DD hh:mm:ss 格式写入和解析，但会以任何字符作为分隔符。 -这一切都发生在客户端或服务器启动时的系统时区（取决于哪一种格式的数据）。对于具有时间的日期，夏时制时间未指定。 因此，如果转储在夏令时中有时间，则转储不会明确地匹配数据，解析将选择两者之一。 -在读取操作期间，不正确的日期和具有时间的日期可以使用自然溢出或空日期和时间进行分析，而不会出现错误消息。 +日期以YYYY-MM-DD格式编写，并以相同的格式解析，但使用任何字符作为分隔符。 +日期和时间以`YYYY-MM-DD hh:mm:ss`的格式书写，并以相同的格式解析，但使用任何字符作为分隔符。 +这一切都发生在客户端或服务器启动时的系统时区(取决于它们对数据的格式)。对于带有时间的日期，夏时制时间未指定。因此，如果转储在夏令时有时间，则转储不会明确地与数据匹配，解析将选择这两次中的一次。 +在读取操作期间，不正确的日期和具有时间的日期可以使用自然溢出或null日期和时间进行分析，而不会出现错误消息。 -有个例外情况，Unix 时间戳格式（10个十进制数字）也支持使用时间解析日期。结果不是时区相关的。格式 YYYY-MM-DD hh:mm:ss和 NNNNNNNNNN 会自动区分。 +有个例外情况，Unix时间戳格式也支持用时间解析日期(如果它恰好由10个十进制数字组成)。其结果与时间区域无关。格式`YYYY-MM-DD hh:mm:ss`和`NNNNNNNNNN`是自动区分的。 -字符串以反斜线转义的特殊字符输出。 以下转义序列用于输出：`\b`，`\f`，`\r`，`\n`，`\t`，`\0`，`\'`，`\\`。 解析还支持`\a`，`\v`和`\xHH`（十六进制转义字符）和任何`\c`字符，其中`c`是任何字符（这些序列被转换为`c`）。 因此，读取数据支持可以将换行符写为`\n`或`\`的格式，或者换行。例如，字符串 `Hello world` 在单词之间换行而不是空格可以解析为以下任何形式： +字符串以反斜杠转义的特殊字符输出。下面的转义序列用于输出:`\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\'`, `\\`。解析还支持`\a`、`\v`和`\xHH`(十六进制转义字符)和任何`\c`字符，其中`c`是任何字符(这些序列被转换为`c`)。因此，读取数据支持这样一种格式，即可以将换行符写成`\n`或`\`，或者写成换行符。例如，字符串`Hello world`在单词之间有换行符，而不是空格，可以用以下语法进行解析: - Hello\nworld +``` text +Hello\nworld - Hello\ - world +Hello\ +world +``` -第二种形式是支持的，因为 MySQL 读取 tab-separated 格式数据集的时候也会使用它。 +第二种形式是支持的，因为MySQL读取tab-separated格式数据集的时候也会使用它。 -在 TabSeparated 格式中传递数据时需要转义的最小字符集为：Tab，换行符（LF）和反斜杠。 +在TabSeparated分隔格式传递数据时需要转义的最小字符集:`Tab`、换行符(LF)和反斜杠。 -只有一小组符号会被转义。你可以轻易地找到一个字符串值，但这不会正常在你的终端显示。 +只有一小部分符号被转义。您可以很容易地找到一个字符串值，而您的终端将在输出中不显示它。 -数组写在方括号内的逗号分隔值列表中。 通常情况下，数组中的数字项目会被拼凑，但日期，带时间的日期以及字符串将使用与上面相同的转义规则用单引号引起来。 +数组写在方括号内的逗号分隔值列表中。数组中的数字项按正常格式进行格式化。`Date`和`DateTime`类型用单引号表示。字符串使用与上面相同的转义规则在单引号中编写。 -[NULL](../sql-reference/syntax.md) 将输出为 `\N`。 +[NULL](../sql-reference/syntax.md)将输出为`\N`。 + +[Nested](../sql-reference/data-types/nested-data-structures/nested.md)结构的每个元素都表示为数组。 + +示例： + +``` sql +CREATE TABLE nestedt +( + `id` UInt8, + `aux` Nested( + a UInt8, + b String + ) +) +ENGINE = TinyLog +``` + +``` sql +INSERT INTO nestedt Values ( 1, [1], ['a']) +``` + +``` sql +SELECT * FROM nestedt FORMAT TSV +``` + +``` text +1 [1] ['a'] +``` ## TabSeparatedRaw {#tabseparatedraw} -与 `TabSeparated` 格式不一样的是，行数据是不会被转义的。 -该格式仅适用于输出查询结果，但不适用于解析输入（将数据插入到表中）。 +与`TabSeparated`格式的不同之处在于，写入的行没有转义。 +使用这种格式解析时，每个字段中不允许使用制表符或换行符。 -这种格式也可以使用名称 `TSVRaw` 来表示。 +这种格式也可以使用名称`TSVRaw`来表示。 ## TabSeparatedWithNames {#tabseparatedwithnames} -与 `TabSeparated` 格式不一样的是，第一行会显示列的名称。 -在解析过程中，第一行完全被忽略。您不能使用列名来确定其位置或检查其正确性。 -（未来可能会加入解析头行的功能） +与`TabSeparated`格式不同的是列名写在第一行。 +在解析过程中，第一行被完全忽略。不能使用列名来确定它们的位置或检查它们的正确性。 +(将来可能会添加对头行解析的支持。) -这种格式也可以使用名称 `TSVWithNames` 来表示。 +这种格式也可以使用名称`TSVWithNames`来表示。 ## TabSeparatedWithNamesAndTypes {#tabseparatedwithnamesandtypes} -与 `TabSeparated` 格式不一样的是，第一行会显示列的名称，第二行会显示列的类型。 -在解析过程中，第一行和第二行完全被忽略。 +与`TabSeparated`格式不同的是列名写在第一行，而列类型写在第二行。 +在解析过程中，将完全忽略第一行和第二行。 -这种格式也可以使用名称 `TSVWithNamesAndTypes` 来表示。 +这种格式也可以使用名称`TSVWithNamesAndTypes`来表示。 -## 模板 {#format-template} +## Template {#format-template} -此格式允许为具有指定转义规则的值指定带有占位符的自定义格式字符串。 +此格式允许指定带有占位符的自定义格式字符串，这些占位符用于指定转义规则。 -它使用设置 `format_schema`, `format_schema_rows`, `format_schema_rows_between_delimiter` and some settings of other formats (e.g. `output_format_json_quote_64bit_integers` 使用时 `JSON` 逃跑，进一步查看) +它使用设置`format_schema`, `format_schema_rows`, `format_schema_rows_between_delimiter`以及其他格式的一些设置(例如转义`JSON`时使用`output_format_json_quote_64bit_integers`) -格式字符串 `format_schema_rows` 使用以下语法指定行格式: +设置`format_template_row`指定文件的路径，该文件包含以下语法的行格式字符串: `delimiter_1${column_1:serializeAs_1}delimiter_2${column_2:serializeAs_2} ... delimiter_N`, - where `delimiter_i` is a delimiter between values (`$` symbol can be escaped as `$$`), - `column_i` is a name of a column whose values are to be selected or inserted (if empty, then column will be skipped), - `serializeAs_i` is an escaping rule for the column values. The following escaping rules are supported: +其中，`delimiter_i`是值之间的分隔符(`$`符号可以转义为`$$`)， +`column_i`是要选择或插入其值的列的名称或索引(如果为空，则跳过该列)， +`serializeAs_i`是列值的转义规则。支持以下转义规则: - - `CSV`, `JSON`, `XML` (similarly to the formats of the same names) - - `Escaped` (similarly to `TSV`) - - `Quoted` (similarly to `Values`) - - `Raw` (without escaping, similarly to `TSVRaw`) - - `None` (no escaping rule, see further) +- `CSV`, `JSON`, `XML` (类似于相同名称的格式) +- `Escaped` (类似于`TSV`) +- `Quoted` (类似于`Values`) +- `Raw` (类似于`TSVRaw`) +- `None` - If escaping rule is omitted, then`None` will be used. `XML` and `Raw` are suitable only for output. +如果省略了转义规则，那么将使用`None`。`XML`和`Raw`只适用于输出。 - So, for the following format string: +对于下面的格式字符串: `Search phrase: ${SearchPhrase:Quoted}, count: ${c:Escaped}, ad price: $$${price:JSON};` - the values of `SearchPhrase`, `c` and `price` columns, which are escaped as `Quoted`, `Escaped` and `JSON` will be printed (for select) or will be expected (for insert) between `Search phrase: `, `, count: `, `, ad price: $` and `;` delimiters respectively. For example: +`SearchPhrase`、`c`和`price`列的值被转义为`quotation`、`Escaped`和`JSON`将分别在`Search phrase:`， `， count: `， `， ad price: $`和`;`分隔符之间打印(用于选择)或expected(用于插入)。例如: - `Search phrase: 'bathroom interior design', count: 2166, ad price: $3;` +`Search phrase: 'bathroom interior design', count: 2166, ad price: $3;` -该 `format_schema_rows_between_delimiter` setting指定行之间的分隔符，该分隔符在除最后一行之外的每一行之后打印（或预期） (`\n` 默认情况下) +`format_template_rows_between_delimiter`设置指定行之间的分隔符，它将打印(或expected)在每一行之后，最后一行除外(默认为`\n`)。 -格式字符串 `format_schema` 具有相同的语法 `format_schema_rows` 并允许指定前缀，后缀和打印一些附加信息的方式。 它包含以下占位符而不是列名: +设置`format_template_resultset`指定文件路径，该文件包含resultset的格式字符串。resultset的格式字符串与row的格式字符串具有相同的语法，允许指定前缀、后缀和打印一些附加信息的方法。它包含以下占位符而不是列名: -- `data` 包含数据的行 `format_schema_rows` 格式，由分隔 `format_schema_rows_between_delimiter`. 此占位符必须是格式字符串中的第一个占位符。 -- `totals` 是包含总值的行 `format_schema_rows` 格式（与总计一起使用时) -- `min` 是具有最小值的行 `format_schema_rows` 格式（当极值设置为1时) -- `max` 是具有最大值的行 `format_schema_rows` 格式（当极值设置为1时) -- `rows` 输出行总数 -- `rows_before_limit` 是没有限制的最小行数。 仅当查询包含LIMIT时输出。 如果查询包含GROUP BY，则rows_before_limit_at_least是没有限制的确切行数。 -- `time` 请求执行时间以秒为单位 +- `data` `format_template_row`格式的数据行，由`format_template_rows_between_delimiter`分隔。此占位符必须是格式字符串中的第一个占位符。 +- `totals` `format_template_row`格式的总值(和WITH TOTALS一起使用) +- `min` `format_template_row`格式的最小值(当极值设置为1时) +- `max` `format_template_row`格式的最大值(当极值设置为1时) +- `rows` 输出行的总数 +- `rows_before_limit` 没有LIMIT的最小行数。仅当查询包含LIMIT时输出。如果查询包含GROUP BY，那么rows_before_limit_at_least就是没有LIMIT的确切行数。 +- `time` 请求执行时间（秒） - `rows_read` 已读取的行数 -- `bytes_read` 被读取的字节数（未压缩） +- `bytes_read` 已读取（未压缩）的字节数 -占位符 `data`, `totals`, `min` 和 `max` 必须没有指定转义规则（或 `None` 必须明确指定）。 其余的占位符可能具有指定的任何转义规则。 -如果 `format_schema` 设置为空字符串, `${data}` 用作默认值。 -对于插入查询格式允许跳过一些列或一些字段，如果前缀或后缀（见示例）。 +占位符`data`、`totals`、`min`和`max`必须没有指定转义规则(或者必须显式指定`None`)。其余占位符可以指定任何转义规则。 +如果`format_template_resultset`设置为空字符串，则使用`${data}`作为默认值。 +对于insert查询，格式允许跳过某些列或某些字段的前缀或后缀(参见示例)。 -`Select` 示例: +Select示例: ``` sql -SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase ORDER BY c DESC LIMIT 5 -FORMAT Template -SETTINGS format_schema = ' +SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase ORDER BY c DESC LIMIT 5 FORMAT Template SETTINGS +format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format', format_template_rows_between_delimiter = '\n ' +``` + +`/some/path/resultset.format`: + +``` text + Search phrases @@ -185,11 +243,17 @@ SETTINGS format_schema = '
Search phrases
Processed ${rows_read:XML} rows in ${time:XML} sec -', -format_schema_rows = ' ${SearchPhrase:XML} ${с:XML} ', -format_schema_rows_between_delimiter = '\n ' + ``` +`/some/path/row.format`: + +``` text + ${0:XML} ${1:XML} +``` + +结果： + ``` html Search phrases @@ -210,92 +274,122 @@ format_schema_rows_between_delimiter = '\n ' ``` -`Insert` 示例: +Insert示例： - Some header - Page views: 5, User id: 4324182021466249494, Useless field: hello, Duration: 146, Sign: -1 - Page views: 6, User id: 4324182021466249494, Useless field: world, Duration: 185, Sign: 1 - Total rows: 2 +``` text +Some header +Page views: 5, User id: 4324182021466249494, Useless field: hello, Duration: 146, Sign: -1 +Page views: 6, User id: 4324182021466249494, Useless field: world, Duration: 185, Sign: 1 +Total rows: 2 +``` ``` sql INSERT INTO UserActivity FORMAT Template SETTINGS -format_schema = 'Some header\n${data}\nTotal rows: ${:CSV}\n', -format_schema_rows = 'Page views: ${PageViews:CSV}, User id: ${UserID:CSV}, Useless field: ${:CSV}, Duration: ${Duration:CSV}, Sign: ${Sign:CSV}' +format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format' ``` -`PageViews`, `UserID`, `Duration` 和 `Sign` 占位符内部是表中列的名称。 值后 `Useless field` 在行和之后 `\nTotal rows:` in后缀将被忽略。 +`/some/path/resultset.format`: + +``` text +Some header\n${data}\nTotal rows: ${:CSV}\n +``` + +`/some/path/row.format`: + +``` text +Page views: ${PageViews:CSV}, User id: ${UserID:CSV}, Useless field: ${:CSV}, Duration: ${Duration:CSV}, Sign: ${Sign:CSV} +``` + +`PageViews`, `UserID`, `Duration`和`Sign` 内部占位符是表中列的名称。将忽略行中`Useless field`后面和后缀中`\nTotal rows:`之后的值。 输入数据中的所有分隔符必须严格等于指定格式字符串中的分隔符。 ## TemplateIgnoreSpaces {#templateignorespaces} -此格式仅适用于输入。 -类似于 `Template`，但跳过输入流中的分隔符和值之间的空格字符。 但是，如果格式字符串包含空格字符，则在输入流中将需要这些字符。 还允许指定空占位符 (`${}` 或 `${:None}`）将一些分隔符分成单独的部分，以忽略它们之间的空格。 此类占位符仅用于跳过空格字符。 -可以阅读 `JSON` 如果列的值在所有行中具有相同的顺序，则使用此格式。 例如，以下请求可用于从格式的输出示例中插入数据 [JSON](#json): +这种格式只适用于输入。 +类似于`Template`，但跳过输入流中分隔符和值之间的空白字符。但是，如果格式字符串包含空格字符，这些字符将会出现在输入流中。还允许指定空占位符(`${}`或`${:None}`)来将一些分隔符分割为单独的部分，以忽略它们之间的空格。这种占位符仅用于跳过空白字符。 +如果列的值在所有行的顺序相同，那么可以使用这种格式读取`JSON`。可以使用以下请求从格式为[JSON](#json)的输出示例中插入数据： ``` sql INSERT INTO table_name FORMAT TemplateIgnoreSpaces SETTINGS -format_schema = '{${}"meta"${}:${:JSON},${}"data"${}:${}[${data}]${},${}"totals"${}:${:JSON},${}"extremes"${}:${:JSON},${}"rows"${}:${:JSON},${}"rows_before_limit_at_least"${}:${:JSON}${}}', -format_schema_rows = '{${}"SearchPhrase"${}:${}${phrase:JSON}${},${}"c"${}:${}${cnt:JSON}${}}', -format_schema_rows_between_delimiter = ',' +format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format', format_template_rows_between_delimiter = ',' +``` + +`/some/path/resultset.format`: + +``` text +{${}"meta"${}:${:JSON},${}"data"${}:${}[${data}]${},${}"totals"${}:${:JSON},${}"extremes"${}:${:JSON},${}"rows"${}:${:JSON},${}"rows_before_limit_at_least"${}:${:JSON}${}} +``` + +`/some/path/row.format`: + +``` text +{${}"SearchPhrase"${}:${}${phrase:JSON}${},${}"c"${}:${}${cnt:JSON}${}} ``` ## TSKV {#tskv} -与 `TabSeparated` 格式类似，但它输出的是 `name=value` 的格式。名称会和 `TabSeparated` 格式一样被转义，`=` 字符也会被转义。 +类似于TabSeparated，但是输出的值是name=value格式。名称的转义方式与TabSeparated格式相同，=符号也是转义的。 - SearchPhrase= count()=8267016 - SearchPhrase=bathroom interior design count()=2166 - SearchPhrase=yandex count()=1655 - SearchPhrase=2014 spring fashion count()=1549 - SearchPhrase=freeform photos count()=1480 - SearchPhrase=angelina jolie count()=1245 - SearchPhrase=omsk count()=1112 - SearchPhrase=photos of dog breeds count()=1091 - SearchPhrase=curtain designs count()=1064 - SearchPhrase=baku count()=1000 +``` text +SearchPhrase= count()=8267016 +SearchPhrase=bathroom interior design count()=2166 +SearchPhrase=yandex count()=1655 +SearchPhrase=2014 spring fashion count()=1549 +SearchPhrase=freeform photos count()=1480 +SearchPhrase=angelina jolie count()=1245 +SearchPhrase=omsk count()=1112 +SearchPhrase=photos of dog breeds count()=1091 +SearchPhrase=curtain designs count()=1064 +SearchPhrase=baku count()=1000 +``` -[NULL](../sql-reference/syntax.md) 输出为 `\N`。 +[NULL](../sql-reference/syntax.md)格式为`\N`。 ``` sql SELECT * FROM t_null FORMAT TSKV ``` - x=1 y=\N +``` text +x=1 y=\N +``` -当有大量的小列时，这种格式是低效的，通常没有理由使用它。它被用于 Yandex 公司的一些部门。 +当有大量的小列时，这种格式是无效的，并且通常没有理由使用它。不过，就效率而言，它并不比JSONEachRow差。 +这种格式支持数据输出和解析。对于解析，不同列的值支持任何顺序。省略某些值是可以接受的——它们被视为与其默认值相等。在这种情况下，0和空白行被用作默认值。不支持在表中指定的复杂值作为缺省值。 -数据的输出和解析都支持这种格式。对于解析，任何顺序都支持不同列的值。可以省略某些值，用 `-` 表示， 它们被视为等于它们的默认值。在这种情况下，零和空行被用作默认值。作为默认值，不支持表中指定的复杂值。 - -对于不带等号或值，可以用附加字段 `tskv` 来表示，这种在解析上是被允许的。这样的话该字段被忽略。 +解析允许存在不带等号或值的附加字段`tskv`。此字段被忽略。 ## CSV {#csv} -按逗号分隔的数据格式([RFC](https://tools.ietf.org/html/rfc4180))。 +按`,`分隔的数据格式([RFC](https://tools.ietf.org/html/rfc4180))。 -格式化的时候，行是用双引号括起来的。字符串中的双引号会以两个双引号输出，除此之外没有其他规则来做字符转义了。日期和时间也会以双引号包括。数字的输出不带引号。值由一个单独的字符隔开，这个字符默认是 `,`。行使用 Unix 换行符（LF）分隔。 数组序列化成 CSV 规则如下：首先将数组序列化为 TabSeparated 格式的字符串，然后将结果字符串用双引号包括输出到 CSV。CSV 格式的元组被序列化为单独的列（即它们在元组中的嵌套关系会丢失）。 +格式化时，行是用双引号括起来的。字符串中的双引号会以两个双引号输出，除此之外没有其他规则来做字符转义了。日期和时间也会以双引号包括。数字的输出不带引号。值由一个单独的字符隔开，这个字符默认是`,`。行使用Unix换行符（LF）分隔。数组序列化成CSV规则如下：首先将数组序列化为`TabSeparated`格式的字符串，然后将结果字符串用双引号包括输出到`CSV`。`CSV`格式的元组被序列化为单独的列（即它们在元组中的嵌套关系会丢失）。 - clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FORMAT CSV" < data.csv +``` bash +$ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FORMAT CSV" < data.csv +``` -\*默认情况下间隔符是 `,` ，在 [format_csv_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter) 中可以了解更多间隔符配置。 +\* 默认情况下间隔符是`,` ，在[format_csv_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter)中可以了解更多分隔符配置。 -解析的时候，可以使用或不使用引号来解析所有值。支持双引号和单引号。行也可以不用引号排列。 在这种情况下，它们被解析为逗号或换行符（CR 或 LF）。在解析不带引号的行时，若违反 RFC 规则，会忽略前导和尾随的空格和制表符。 对于换行，全部支持 Unix（LF），Windows（CR LF）和 Mac OS Classic（CR LF）。 +解析的时候，可以使用或不使用引号来解析所有值。支持双引号和单引号。行也可以不用引号排列。在这种情况下，它们被解析为逗号或换行符（`CR或`LF`）。在解析不带引号的行时，若违反`RFC`规则，会忽略前缀和结尾的空格和制表符。对于换行，全部支持Unix（LF），Windows（CR LF）和Mac OS Classic（CR LF）。 -`NULL` 将输出为 `\N`。 +如果启用[input_format_defaults_for_omitted_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields)，空的末尾加引号的输入值将替换为相应列的默认值。 -CSV 格式是和 TabSeparated 一样的方式输出总数和极值。 +`NULL`被格式化为`\N`或`NULL`或一个空的非引号字符串(详见配置[input_format_csv_unquoted_null_literal_as_null](../operations/settings/settings.md#settings-input_format_csv_unquoted_null_literal_as_null)或[input_format_defaults_for_omitted_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields))。 + +`CSV`格式支持输出总数和极值的方式与`TabSeparated`相同。 ## CSVWithNames {#csvwithnames} -会输出带头部行，和 `TabSeparatedWithNames` 一样。 +会输出带头部的信息(字段列表)，和`TabSeparatedWithNames`一样。 -## 自定义分离 {#format-customseparated} +## CustomSeparated {#format-customseparated} -类似于 [模板](#format-template)，但它打印或读取所有列，并使用从设置转义规则 `format_custom_escaping_rule` 从设置和分隔符 `format_custom_field_delimiter`, `format_custom_row_before_delimiter`, `format_custom_row_after_delimiter`, `format_custom_row_between_delimiter`, `format_custom_result_before_delimiter` 和 `format_custom_result_after_delimiter`，而不是从格式字符串。 -也有 `CustomSeparatedIgnoreSpaces` 格式，这是类似于 `TemplateIgnoreSpaces`. +类似于[Template](#format-template)， 但它打印或读取所有列和使用转义规则在设置`format_custom_escaping_rule`和分隔符设置`format_custom_field_delimiter`,`format_custom_row_before_delimiter`,`format_custom_row_after_delimiter`,`format_custom_row_between_delimiter`,`format_custom_result_before_delimiter`,`format_custom_result_after_delimiter`中,而不是从格式字符串。 +也有`CustomSeparatedIgnoreSpaces`格式，这是类似于`TemplateIgnoreSpaces`。 ## JSON {#json} -以 JSON 格式输出数据。除了数据表之外，它还输出列名称和类型以及一些附加信息：输出行的总数以及在没有 LIMIT 时可以输出的行数。 例： +以JSON格式输出数据。除了数据表之外，它还输出列名和类型，以及一些附加信息: 输出行的总数，以及如果没有LIMIT的话可输出的行数。示例: ``` sql SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTALS ORDER BY c DESC LIMIT 5 FORMAT JSON @@ -306,166 +400,326 @@ SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTA "meta": [ { - "name": "SearchPhrase", + "name": "'hello'", "type": "String" }, { - "name": "c", + "name": "multiply(42, number)", "type": "UInt64" + }, + { + "name": "range(5)", + "type": "Array(UInt8)" } ], "data": [ { - "SearchPhrase": "", - "c": "8267016" + "'hello'": "hello", + "multiply(42, number)": "0", + "range(5)": [0,1,2,3,4] }, { - "SearchPhrase": "bathroom interior design", - "c": "2166" + "'hello'": "hello", + "multiply(42, number)": "42", + "range(5)": [0,1,2,3,4] }, { - "SearchPhrase": "yandex", - "c": "1655" - }, - { - "SearchPhrase": "spring 2014 fashion", - "c": "1549" - }, - { - "SearchPhrase": "freeform photos", - "c": "1480" + "'hello'": "hello", + "multiply(42, number)": "84", + "range(5)": [0,1,2,3,4] } ], - "totals": - { - "SearchPhrase": "", - "c": "8873898" - }, + "rows": 3, - "extremes": - { - "min": - { - "SearchPhrase": "", - "c": "1480" - }, - "max": - { - "SearchPhrase": "", - "c": "8267016" - } - }, - - "rows": 5, - - "rows_before_limit_at_least": 141137 + "rows_before_limit_at_least": 3 } ``` -JSON 与 JavaScript 兼容。为了确保这一点，一些字符被另外转义：斜线`/`被转义为`\/`; 替代的换行符 `U+2028` 和 `U+2029` 会打断一些浏览器解析，它们会被转义为 `\uXXXX`。 ASCII 控制字符被转义：退格，换页，换行，回车和水平制表符被替换为`\b`，`\f`，`\n`，`\r`，`\t` 作为使用`\uXXXX`序列的00-1F范围内的剩余字节。 无效的 UTF-8 序列更改为替换字符 ，因此输出文本将包含有效的 UTF-8 序列。 为了与 JavaScript 兼容，默认情况下，Int64 和 UInt64 整数用双引号引起来。要除去引号，可以将配置参数 output_format_json_quote_64bit_integers 设置为0。 +JSON与JavaScript兼容。为了确保这一点，一些字符被另外转义：斜线`/`被转义为`\/`; 替代的换行符`U+2028`和`U+2029`会打断一些浏览器解析，它们会被转义为`\uXXXX`。 ASCII控制字符被转义：退格，换页，换行，回车和水平制表符被替换为`\b`，`\f`，`\n`，`\r`，`\t` 作为使用`\uXXXX`序列的00-1F范围内的剩余字节。 无效的UTF-8序列更改为替换字符，因此输出文本将包含有效的UTF-8序列。 为了与JavaScript兼容，默认情况下，Int64和UInt64整数用双引号引起来。要除去引号，可以将配置参数`output_format_json_quote_64bit_integers`设置为0。 `rows` – 结果输出的行数。 -`rows_before_limit_at_least` 去掉 LIMIT 过滤后的最小行总数。 只会在查询包含 LIMIT 条件时输出。 -若查询包含 GROUP BY，rows_before_limit_at_least 就是去掉 LIMIT 后过滤后的准确行数。 +`rows_before_limit_at_least`去掉 LIMIT过滤后的最小行总数。 只会在查询包含LIMIT条件时输出。 +若查询包含 GROUP BY，`rows_before_limit_at_least`就是去掉LIMIT后过滤后的准确行数。 -`totals` – 总值 （当使用 TOTALS 条件时）。 +`totals` – 总值 （当使用TOTALS条件时）。 -`extremes` – 极值 （当 extremes 设置为 1时）。 +`extremes` – 极值（当extremes设置为1时）。 该格式仅适用于输出查询结果，但不适用于解析输入（将数据插入到表中）。 -ClickHouse 支持 [NULL](../sql-reference/syntax.md), 在 JSON 格式中以 `null` 输出来表示. +ClickHouse支持[NULL](../sql-reference/syntax.md), 在JSON输出中显示为`null`。若要在输出中启用`+nan`、`-nan`、`+inf`、`-inf`值，请设置[output_format_json_quote_denormals](../operations/settings/settings.md#settings-output_format_json_quote_denormals)为1。 -参考 JSONEachRow 格式。 +**参考** -## JSONCompact {#jsoncompact} +- [JSONEachRow](#jsoneachrow)格式 +- [output_format_json_array_of_rows](../operations/settings/settings.md#output-format-json-array-of-rows)设置 -与 JSON 格式不同的是它以数组的方式输出结果，而不是以结构体。 +## JSONString {#jsonstring} + +与JSON的不同之处在于数据字段以字符串输出，而不是以类型化JSON值输出。 示例： -``` json +```json { "meta": [ { - "name": "SearchPhrase", + "name": "'hello'", "type": "String" }, { - "name": "c", + "name": "multiply(42, number)", "type": "UInt64" + }, + { + "name": "range(5)", + "type": "Array(UInt8)" } ], "data": [ - ["", "8267016"], - ["bathroom interior design", "2166"], - ["yandex", "1655"], - ["fashion trends spring 2014", "1549"], - ["freeform photo", "1480"] + { + "'hello'": "hello", + "multiply(42, number)": "0", + "range(5)": "[0,1,2,3,4]" + }, + { + "'hello'": "hello", + "multiply(42, number)": "42", + "range(5)": "[0,1,2,3,4]" + }, + { + "'hello'": "hello", + "multiply(42, number)": "84", + "range(5)": "[0,1,2,3,4]" + } ], - "totals": ["","8873898"], + "rows": 3, - "extremes": - { - "min": ["","1480"], - "max": ["","8267016"] - }, - - "rows": 5, - - "rows_before_limit_at_least": 141137 + "rows_before_limit_at_least": 3 } ``` -这种格式仅仅适用于输出结果集，而不适用于解析（将数据插入到表中）。 -参考 `JSONEachRow` 格式。 +## JSONAsString {#jsonasstring} -## JSONEachRow {#jsoneachrow} +在这种格式中，一个JSON对象被解释为一个值。如果输入有几个JSON对象(逗号分隔)，它们将被解释为独立的行。 -将数据结果每一行以 JSON 结构体输出（换行分割 JSON 结构体）。 +这种格式只能对具有单个字段类型的表进行解析[String](../sql-reference/data-types/string.md)。其余的列必须设置为[DEFAULT](../sql-reference/statements/create/table.md#default)或[MATERIALIZED](../sql-reference/statements/create/table.md#materialized)，或者忽略。一旦将整个JSON对象收集为字符串，就可以使用[JSON函数](../sql-reference/functions/json-functions.md)运行它。 -``` json -{"SearchPhrase":"","count()":"8267016"} -{"SearchPhrase": "bathroom interior design","count()": "2166"} -{"SearchPhrase":"yandex","count()":"1655"} -{"SearchPhrase":"2014 spring fashion","count()":"1549"} -{"SearchPhrase":"freeform photo","count()":"1480"} -{"SearchPhrase":"angelina jolie","count()":"1245"} -{"SearchPhrase":"omsk","count()":"1112"} -{"SearchPhrase":"photos of dog breeds","count()":"1091"} -{"SearchPhrase":"curtain designs","count()":"1064"} -{"SearchPhrase":"baku","count()":"1000"} +**示例** + +查询： + +``` sql +DROP TABLE IF EXISTS json_as_string; +CREATE TABLE json_as_string (json String) ENGINE = Memory; +INSERT INTO json_as_string FORMAT JSONAsString {"foo":{"bar":{"x":"y"},"baz":1}},{},{"any json stucture":1} +SELECT * FROM json_as_string; ``` -与 JSON 格式不同的是，没有替换无效的UTF-8序列。任何一组字节都可以在行中输出。这是必要的，因为这样数据可以被格式化而不会丢失任何信息。值的转义方式与JSON相同。 +结果： -对于解析，任何顺序都支持不同列的值。可以省略某些值 - 它们被视为等于它们的默认值。在这种情况下，零和空行被用作默认值。 作为默认值，不支持表中指定的复杂值。元素之间的空白字符被忽略。如果在对象之后放置逗号，它将被忽略。对象不一定必须用新行分隔。 +``` text +┌─json──────────────────────────────┐ +│ {"foo":{"bar":{"x":"y"},"baz":1}} │ +│ {} │ +│ {"any json stucture":1} │ +└───────────────────────────────────┘ +``` -### 嵌套结构的使用 {#jsoneachrow-nested} +## JSONCompact {#jsoncompact} +## JSONCompactString {#jsoncompactstring} -如果你有一张桌子 [嵌套式](../sql-reference/data-types/nested-data-structures/nested.md) 数据类型列，可以插入具有相同结构的JSON数据。 启用此功能与 [input_format_import_nested_json](../operations/settings/settings.md#settings-input_format_import_nested_json) 设置。 +与JSON格式不同的是它以数组的方式输出结果，而不是以结构体。 -例如，请考虑下表: +示例： + +``` json +// JSONCompact +{ + "meta": + [ + { + "name": "'hello'", + "type": "String" + }, + { + "name": "multiply(42, number)", + "type": "UInt64" + }, + { + "name": "range(5)", + "type": "Array(UInt8)" + } + ], + + "data": + [ + ["hello", "0", [0,1,2,3,4]], + ["hello", "42", [0,1,2,3,4]], + ["hello", "84", [0,1,2,3,4]] + ], + + "rows": 3, + + "rows_before_limit_at_least": 3 +} +``` + +```json +// JSONCompactString +{ + "meta": + [ + { + "name": "'hello'", + "type": "String" + }, + { + "name": "multiply(42, number)", + "type": "UInt64" + }, + { + "name": "range(5)", + "type": "Array(UInt8)" + } + ], + + "data": + [ + ["hello", "0", "[0,1,2,3,4]"], + ["hello", "42", "[0,1,2,3,4]"], + ["hello", "84", "[0,1,2,3,4]"] + ], + + "rows": 3, + + "rows_before_limit_at_least": 3 +} +``` + +## JSONEachRow {#jsoneachrow} +## JSONStringEachRow {#jsonstringeachrow} +## JSONCompactEachRow {#jsoncompacteachrow} +## JSONCompactStringEachRow {#jsoncompactstringeachrow} + +使用这些格式时，ClickHouse会将行输出为分隔的、换行分隔的JSON值，但数据作为一个整体不是有效的JSON。 + +``` json +{"some_int":42,"some_str":"hello","some_tuple":[1,"a"]} // JSONEachRow +[42,"hello",[1,"a"]] // JSONCompactEachRow +["42","hello","(2,'a')"] // JSONCompactStringsEachRow +``` + +在插入数据时，应该为每一行提供一个单独的JSON值。 + +## JSONEachRowWithProgress {#jsoneachrowwithprogress} +## JSONStringEachRowWithProgress {#jsonstringeachrowwithprogress} + +与`JSONEachRow`/`JSONStringEachRow`不同的是，ClickHouse还将生成作为JSON值的进度信息。 + +```json +{"row":{"'hello'":"hello","multiply(42, number)":"0","range(5)":[0,1,2,3,4]}} +{"row":{"'hello'":"hello","multiply(42, number)":"42","range(5)":[0,1,2,3,4]}} +{"row":{"'hello'":"hello","multiply(42, number)":"84","range(5)":[0,1,2,3,4]}} +{"progress":{"read_rows":"3","read_bytes":"24","written_rows":"0","written_bytes":"0","total_rows_to_read":"3"}} +``` + +## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes} +## JSONCompactStringEachRowWithNamesAndTypes {#jsoncompactstringeachrowwithnamesandtypes} + +与`JSONCompactEachRow`/`JSONCompactStringEachRow`不同的是，其中列名和类型被写入前两行。 + +```json +["'hello'", "multiply(42, number)", "range(5)"] +["String", "UInt64", "Array(UInt8)"] +["hello", "0", [0,1,2,3,4]] +["hello", "42", [0,1,2,3,4]] +["hello", "84", [0,1,2,3,4]] +``` + +### Inserting Data {#inserting-data} + +``` sql +INSERT INTO UserActivity FORMAT JSONEachRow {"PageViews":5, "UserID":"4324182021466249494", "Duration":146,"Sign":-1} {"UserID":"4324182021466249494","PageViews":6,"Duration":185,"Sign":1} +``` + +ClickHouse允许: + +- 对象中key-value的任何顺序。 +- 省略一些值。 + +ClickHouse忽略元素之间的空格和对象后面的逗号。您可以在一行中传递所有对象。你不需要用换行符把它们分开。 + +**省略值处理** + +ClickHouse将省略的值替换为对应的[data types](../sql-reference/data-types/index.md)默认值。 + +如果指定了`DEFAULT expr`，则ClickHouse根据属性使用不同的替换规则，详看[input_format_defaults_for_omitted_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields)设置。 + +参考下表： + +``` sql +CREATE TABLE IF NOT EXISTS example_table +( + x UInt32, + a DEFAULT x * 2 +) ENGINE = Memory; +``` + +- 如果`input_format_defaults_for_omitted_fields = 0`, 那么`x`和`a`的默认值等于`0`(作为`UInt32`数据类型的默认值)。 +- 如果`input_format_defaults_for_omitted_fields = 1`, 那么`x`的默认值为`0`，但`a`的默认值为`x * 2`。 + +!!! note "注意" +当使用`insert_sample_with_metadata = 1`插入数据时，与使用`insert_sample_with_metadata = 0`插入数据相比，ClickHouse消耗更多的计算资源。 + +### Selecting Data {#selecting-data} + +以`UserActivity`表为例: + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 5 │ 146 │ -1 │ +│ 4324182021466249494 │ 6 │ 185 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` + +当查询`SELECT * FROM UserActivity FORMAT JSONEachRow`返回: + +``` text +{"UserID":"4324182021466249494","PageViews":5,"Duration":146,"Sign":-1} +{"UserID":"4324182021466249494","PageViews":6,"Duration":185,"Sign":1} +``` + +与[JSON](#json)格式不同，没有替换无效的UTF-8序列。值以与`JSON`相同的方式转义。 + +!!! note "提示" +字符串中可以输出任意一组字节。如果您确信表中的数据可以被格式化为JSON而不会丢失任何信息，那么就使用`JSONEachRow`格式。 + +### Nested Structures {#jsoneachrow-nested} + +如果您有一个包含[Nested](../sql-reference/data-types/nested-data-structures/nested.md)数据类型列的表，您可以插入具有相同结构的JSON数据。使用[input_format_import_nested_json](../operations/settings/settings.md#settings-input_format_import_nested_json)设置启用该特性。 + +例如，请参考下表: ``` sql CREATE TABLE json_each_row_nested (n Nested (s String, i Int32) ) ENGINE = Memory ``` -正如你可以在找到 `Nested` 数据类型说明，ClickHouse将嵌套结构的每个组件视为单独的列, `n.s` 和 `n.i` 为了我们的桌子 所以你可以通过以下方式插入数据: +正如您在`Nested`数据类型描述中看到的，ClickHouse将嵌套结构的每个组件作为一个单独的列(`n.s`和`n.i`是我们的表)。您可以通过以下方式插入数据: ``` sql INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n.s": ["abc", "def"], "n.i": [1, 23]} ``` -将数据作为分层JSON对象集插入 [input_format_import_nested_json=1](../operations/settings/settings.md#settings-input_format_import_nested_json). +将数据作为分层JSON对象集插入[input_format_import_nested_json=1](../operations/settings/settings.md#settings-input_format_import_nested_json)。 ``` json { @@ -508,74 +762,93 @@ SELECT * FROM json_each_row_nested └───────────────┴────────┘ ``` -## 本地人 {#native} +## Native {#native} -最高性能的格式。 据通过二进制格式的块进行写入和读取。对于每个块，该块中的行数，列数，列名称和类型以及列的部分将被相继记录。 换句话说，这种格式是 «列式»的 - 它不会将列转换为行。 这是用于在服务器之间进行交互的本地界面中使用的格式，用于使用命令行客户端和 C++ 客户端。 +最高性能的格式。通过二进制格式的块进行写入和读取。对于每个块，该中的行数，列数，列名称和类型以及列的部分将被相继记录。 换句话说，这种格式是`columnar`的 - 它不会将列转换为行。这是用于在服务器之间进行交互的本地界面中使用的格式，用于使用命令行客户端和C++客户端。 -您可以使用此格式快速生成只能由 ClickHouse DBMS 读取的格式。但自己处理这种格式是没有意义的。 +您可以使用此格式快速生成只能由ClickHouse DBMS读取的格式。但自己处理这种格式是没有意义的。 ## Null {#null} -没有输出。但是，查询已处理完毕，并且在使用命令行客户端时，数据将传输到客户端。这仅用于测试，包括生产力测试。 +没有输出。但是，查询已处理完毕，并且在使用命令行客户端时，数据将传输到客户端。这仅用于测试，包括性能测试。 显然，这种格式只适用于输出，不适用于解析。 -## 漂亮 {#pretty} +## Pretty {#pretty} -将数据以表格形式输出，也可以使用 ANSI 转义字符在终端中设置颜色。 +将数据以表格形式输出，也可以使用ANSI转义字符在终端中设置颜色。 它会绘制一个完整的表格，每行数据在终端中占用两行。 -每一个结果块都会以单独的表格输出。这是很有必要的，以便结果块不用缓冲结果输出（缓冲在可以预见结果集宽度的时候是很有必要的）。 +每个结果块作为一个单独的表输出。这是必要的，以便在输出块时不需要缓冲结果(为了预先计算所有值的可见宽度，缓冲是必要的)。 -[NULL](../sql-reference/syntax.md) 输出为 `ᴺᵁᴸᴸ`。 +[NULL](../sql-reference/syntax.md)输出为`ᴺᵁᴸᴸ`。 + +示例(显示[PrettyCompact](#prettycompact)格式) ``` sql SELECT * FROM t_null ``` - ┌─x─┬────y─┐ - │ 1 │ ᴺᵁᴸᴸ │ - └───┴──────┘ +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +└───┴──────┘ +``` -为避免将太多数据传输到终端，只打印前10,000行。 如果行数大于或等于10,000，则会显示消息«Showed first 10 000»。 +行没有转义为Pretty\* 格式。示例显示了[PrettyCompact](#prettycompact)格式: + +``` sql +SELECT 'String with \'quotes\' and \t character' AS Escaping_test +``` + +``` text +┌─Escaping_test────────────────────────┐ +│ String with 'quotes' and character │ +└──────────────────────────────────────┘ +``` + +为避免将太多数据传输到终端，只打印前10,000行。 如果行数大于或等于10,000，则会显示消息`Showed first 10 000`。 该格式仅适用于输出查询结果，但不适用于解析输入（将数据插入到表中）。 -Pretty格式支持输出总值（当使用 WITH TOTALS 时）和极值（当 `extremes` 设置为1时）。 在这些情况下，总数值和极值在主数据之后以单独的表格形式输出。 示例（以 PrettyCompact 格式显示）： +Pretty格式支持输出合计值(当使用WITH TOTALS时)和极值(当`extremes`设置为1时)。在这些情况下，合计值和极值将输出在主要数据之后，在单独的表中。示例(显示为[PrettyCompact](#prettycompact)格式): ``` sql SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT PrettyCompact ``` - ┌──EventDate─┬───────c─┐ - │ 2014-03-17 │ 1406958 │ - │ 2014-03-18 │ 1383658 │ - │ 2014-03-19 │ 1405797 │ - │ 2014-03-20 │ 1353623 │ - │ 2014-03-21 │ 1245779 │ - │ 2014-03-22 │ 1031592 │ - │ 2014-03-23 │ 1046491 │ - └────────────┴─────────┘ +``` text +┌──EventDate─┬───────c─┐ +│ 2014-03-17 │ 1406958 │ +│ 2014-03-18 │ 1383658 │ +│ 2014-03-19 │ 1405797 │ +│ 2014-03-20 │ 1353623 │ +│ 2014-03-21 │ 1245779 │ +│ 2014-03-22 │ 1031592 │ +│ 2014-03-23 │ 1046491 │ +└────────────┴─────────┘ - Totals: - ┌──EventDate─┬───────c─┐ - │ 1970-01-01 │ 8873898 │ - └────────────┴─────────┘ +Totals: +┌──EventDate─┬───────c─┐ +│ 1970-01-01 │ 8873898 │ +└────────────┴─────────┘ - Extremes: - ┌──EventDate─┬───────c─┐ - │ 2014-03-17 │ 1031592 │ - │ 2014-03-23 │ 1406958 │ - └────────────┴─────────┘ +Extremes: +┌──EventDate─┬───────c─┐ +│ 2014-03-17 │ 1031592 │ +│ 2014-03-23 │ 1406958 │ +└────────────┴─────────┘ +``` ## PrettyCompact {#prettycompact} -与 `Pretty` 格式不一样的是，`PrettyCompact` 去掉了行之间的表格分割线，这样使得结果更加紧凑。这种格式会在交互命令行客户端下默认使用。 +与[Pretty](#pretty)格式不一样的是`PrettyCompact`去掉了行之间的表格分割线，这样使得结果更加紧凑。 +这种格式会在交互命令行客户端下默认使用。 ## PrettyCompactMonoBlock {#prettycompactmonoblock} -与 `PrettyCompact` 格式不一样的是，它支持 10,000 行数据缓冲，然后输出在一个表格中，不会按照块来区分 +与[PrettyCompact](#prettycompact)格式不一样的是，它支持10,000行数据缓冲，然后输出在一个表格中，不会按照块来区分。 ## PrettyNoEscapes {#prettynoescapes} -与 `Pretty` 格式不一样的是，它不使用 ANSI 字符转义， 这在浏览器显示数据以及在使用 `watch` 命令行工具是有必要的。 +与`Pretty`格式不一样的是，它不使用ANSI字符转义，这在浏览器显示数据以及在使用`watch`命令行工具是有必要的。 示例： @@ -583,7 +856,7 @@ SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORD watch -n1 "clickhouse-client --query='SELECT event, value FROM system.events FORMAT PrettyCompactNoEscapes'" ``` -您可以使用 HTTP 接口来获取数据，显示在浏览器中。 +您可以使用HTTP接口来获取数据，显示在浏览器中。 ### PrettyCompactNoEscapes {#prettycompactnoescapes} @@ -593,9 +866,17 @@ watch -n1 "clickhouse-client --query='SELECT event, value FROM system.events FOR 用法类似上述。 +### PrettyCompactNoEscapes {#prettycompactnoescapes} + +与前面的设置相同。 + +### PrettySpaceNoEscapes {#prettyspacenoescapes} + +与前面的设置相同。 + ## PrettySpace {#prettyspace} -与 `PrettyCompact`(#prettycompact) 格式不一样的是，它使用空格来代替网格来显示数据。 +与[PrettyCompact](#prettycompact)格式不一样的是，它使用空格来代替网格来显示数据。 ## RowBinary {#rowbinary} diff --git a/docs/zh/interfaces/third-party/index.md b/docs/zh/interfaces/third-party/index.md index b55897073da..40206f6b57b 100644 --- a/docs/zh/interfaces/third-party/index.md +++ b/docs/zh/interfaces/third-party/index.md @@ -1,8 +1,16 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u7B2C\u4E09\u65B9" +toc_folder_title: 第三方工具 toc_priority: 24 --- +# 第三方工具 {#third-party-interfaces} +这是第三方工具的链接集合，它们提供了一些ClickHouse的接口。它可以是可视化界面、命令行界面或API: + +- [Client libraries](../../interfaces/third-party/client-libraries.md) +- [Integrations](../../interfaces/third-party/integrations.md) +- [GUI](../../interfaces/third-party/gui.md) +- [Proxies](../../interfaces/third-party/proxy.md) + +!!! note "注意" +支持通用API的通用工具[ODBC](../../interfaces/odbc.md)或[JDBC](../../interfaces/jdbc.md)，通常也适用于ClickHouse，但这里没有列出，因为它们实在太多了。 From 1e3951ebc83ac1d066b57e4fbad8c910ea347391 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 21 Dec 2020 18:39:54 +0800 Subject: [PATCH 087/284] translate documentation for interfaces/thied-party --- .../third-party/client-libraries.md | 65 ++++++++----- .../zh/interfaces/third-party/integrations.md | 94 ++++++++++--------- docs/zh/interfaces/third-party/proxy.md | 49 +++++----- 3 files changed, 119 insertions(+), 89 deletions(-) diff --git a/docs/zh/interfaces/third-party/client-libraries.md b/docs/zh/interfaces/third-party/client-libraries.md index 77c929b9730..e94eb8bcfc0 100644 --- a/docs/zh/interfaces/third-party/client-libraries.md +++ b/docs/zh/interfaces/third-party/client-libraries.md @@ -1,51 +1,66 @@ -# 第三方开发的库 {#di-san-fang-kai-fa-de-ku} +--- +toc_priority: 26 +toc_title: 客户端开发库 +--- -!!! warning "放弃" - Yandex不维护下面列出的库，也没有进行任何广泛的测试以确保其质量。 +# 第三方开发库 {#client-libraries-from-third-party-developers} + +!!! warning "声明" +Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试来确保它们的质量。 - Python - [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) - - [ﾂ环板driverｮﾂ嘉ｯﾂ偲](https://github.com/mymarilyn/clickhouse-driver) - - [ﾂ环板clientｮﾂ嘉ｯﾂ偲](https://github.com/yurial/clickhouse-client) + - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) + - [clickhouse-client](https://github.com/yurial/clickhouse-client) + - [aiochclient](https://github.com/maximdanilchenko/aiochclient) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - - [8bitov/clickhouse-php客户端](https://packagist.org/packages/8bitov/clickhouse-php-client) - - [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://packagist.org/packages/bozerkins/clickhouse-client) - - [ﾂ环板clientｮﾂ嘉ｯﾂ偲](https://packagist.org/packages/simpod/clickhouse-client) + - [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) + - [bozerkins/clickhouse-client](https://packagist.org/packages/bozerkins/clickhouse-client) + - [simpod/clickhouse-client](https://packagist.org/packages/simpod/clickhouse-client) - [seva-code/php-click-house-client](https://packagist.org/packages/seva-code/php-click-house-client) - - [ﾂ环板clientｮﾂ嘉ｯﾂ偲](https://github.com/SeasX/SeasClick) -- 走吧 + - [SeasClick C++ client](https://github.com/SeasX/SeasClick) + - [one-ck](https://github.com/lizhichao/one-ck) + - [glushkovds/phpclickhouse-laravel](https://packagist.org/packages/glushkovds/phpclickhouse-laravel) +- Go - [clickhouse](https://github.com/kshvakov/clickhouse/) - - [ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/roistat/go-clickhouse) - - [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/mailru/go-clickhouse) + - [go-clickhouse](https://github.com/roistat/go-clickhouse) + - [mailrugo-clickhouse](https://github.com/mailru/go-clickhouse) - [golang-clickhouse](https://github.com/leprosus/golang-clickhouse) +- Swift + - [ClickHouseNIO](https://github.com/patrick-zippenfenig/ClickHouseNIO) + - [ClickHouseVapor ORM](https://github.com/patrick-zippenfenig/ClickHouseVapor) - NodeJs - - [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人)](https://github.com/TimonKK/clickhouse) - - [ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/apla/node-clickhouse) + - [clickhouse (NodeJs)](https://github.com/TimonKK/clickhouse) + - [node-clickhouse](https://github.com/apla/node-clickhouse) - Perl - [perl-DBD-ClickHouse](https://github.com/elcamlost/perl-DBD-ClickHouse) - [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse) - - [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://metacpan.org/release/AnyEvent-ClickHouse) + - [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) - Ruby - - [ﾂ暗ｪﾂ氾环催ﾂ団)](https://github.com/shlima/click_house) - - [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/PNixx/clickhouse-activerecord) + - [ClickHouse (Ruby)](https://github.com/shlima/click_house) + - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) - R - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - - [RClickhouse](https://github.com/IMSMWU/RClickhouse) + - [RClickHouse](https://github.com/IMSMWU/RClickHouse) - Java - [clickhouse-client-java](https://github.com/VirtusAI/clickhouse-client-java) -- 斯卡拉 - - [掳胫client-禄脢鹿脷露胫鲁隆鹿-client酶](https://github.com/crobox/clickhouse-scala-client) + - [clickhouse-client](https://github.com/Ecwid/clickhouse-client) +- Scala + - [clickhouse-scala-client](https://github.com/crobox/clickhouse-scala-client) - Kotlin - [AORM](https://github.com/TanVD/AORM) - C# - [Octonica.ClickHouseClient](https://github.com/Octonica/ClickHouseClient) - - [克莱克豪斯Ado](https://github.com/killwort/ClickHouse-Net) + - [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net) + - [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) - - [克莱克豪斯客户](https://github.com/DarkWanderer/ClickHouse.Client) -- 仙丹 +- Elixir - [clickhousex](https://github.com/appodeal/clickhousex/) -- 尼姆 + - [pillar](https://github.com/sofakingworld/pillar) +- Nim - [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse) +- Haskell + - [hdbc-clickhouse](https://github.com/zaneli/hdbc-clickhouse) -[来源文章](https://clickhouse.tech/docs/zh/interfaces/third-party/client_libraries/) +[来源文章](https://clickhouse.tech/docs/en/interfaces/third-party/client_libraries/) diff --git a/docs/zh/interfaces/third-party/integrations.md b/docs/zh/interfaces/third-party/integrations.md index 2c57e23b724..403ef994bb9 100644 --- a/docs/zh/interfaces/third-party/integrations.md +++ b/docs/zh/interfaces/third-party/integrations.md @@ -1,100 +1,108 @@ -# 第三方集成库 {#di-san-fang-ji-cheng-ku} +--- +toc_priority: 27 +toc_title: 第三方集成库 +--- + +# 第三方集成库 {#integration-libraries-from-third-party-developers} !!! warning "声明" - Yandex不维护下面列出的库，也没有进行任何广泛的测试以确保其质量。 +Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试来确保它们的质量。 -## 基建产品 {#ji-jian-chan-pin} +## 基础设施 {#infrastructure-products} -- 关系数据库管理系统 +- 关系数据库 - [MySQL](https://www.mysql.com) - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) - - [horgh-复制器](https://github.com/larsnovikov/horgh-replicator) + - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator) - [PostgreSQL](https://www.postgresql.org) - [clickhousedb_fdw](https://github.com/Percona-Lab/clickhousedb_fdw) - - [infi.clickhouse_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (使用 [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm)) + - [infi.clickhouse_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (uses [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm)) - [pg2ch](https://github.com/mkabilov/pg2ch) + - [clickhouse_fdw](https://github.com/adjust/clickhouse_fdw) - [MSSQL](https://en.wikipedia.org/wiki/Microsoft_SQL_Server) - - [ClickHouseMightrator](https://github.com/zlzforever/ClickHouseMigrator) + - [ClickHouseMigrator](https://github.com/zlzforever/ClickHouseMigrator) - 消息队列 - - [卡夫卡](https://kafka.apache.org) - - [clickhouse_sinker](https://github.com/housepower/clickhouse_sinker) (使用 [去客户](https://github.com/ClickHouse/clickhouse-go/)) + - [Kafka](https://kafka.apache.org) + - [clickhouse_sinker](https://github.com/housepower/clickhouse_sinker) (uses [Go client](https://github.com/ClickHouse/clickhouse-go/)) - [stream-loader-clickhouse](https://github.com/adform/stream-loader) - 流处理 - [Flink](https://flink.apache.org) - [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink) - 对象存储 - [S3](https://en.wikipedia.org/wiki/Amazon_S3) - - [ﾂ环板backupｮﾂ嘉ｯﾂ偲](https://github.com/AlexAkulov/clickhouse-backup) + - [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup) - 容器编排 - [Kubernetes](https://kubernetes.io) - - [clickhouse-操](https://github.com/Altinity/clickhouse-operator) + - [clickhouse-operator](https://github.com/Altinity/clickhouse-operator) - 配置管理 - - [木偶](https://puppet.com) - - [ﾂ环板/ｮﾂ嘉ｯﾂ偲](https://forge.puppet.com/innogames/clickhouse) - - [mfedotov/clickhouse](https://forge.puppet.com/mfedotov/clickhouse) -- 监控 - - [石墨](https://graphiteapp.org) + - [puppet](https://puppet.com) + - [innogames/clickhouse](https://forge.puppet.com/innogames/clickhouse) + - [mfedotov/clickhouse](https://forge.puppet.com/mfedotov/clickhouse) +- Monitoring + - [Graphite](https://graphiteapp.org) - [graphouse](https://github.com/yandex/graphouse) - - [ﾂ暗ｪﾂ氾环催ﾂ団](https://github.com/lomik/carbon-clickhouse) + - - [ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/lomik/graphite-clickhouse) - - [石墨-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) -优化静态分区 [\*GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) 如果从规则 [汇总配置](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) 可以应用 + - [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse) + + - [graphite-clickhouse](https://github.com/lomik/graphite-clickhouse) + - [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) - optimizes staled partitions in [\*GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) if rules from [rollup configuration](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) could be applied - [Grafana](https://grafana.com/) - [clickhouse-grafana](https://github.com/Vertamedia/clickhouse-grafana) - - [普罗米修斯号](https://prometheus.io/) + - [Prometheus](https://prometheus.io/) - [clickhouse_exporter](https://github.com/f1yegor/clickhouse_exporter) - [PromHouse](https://github.com/Percona-Lab/PromHouse) - - [clickhouse_exporter](https://github.com/hot-wifi/clickhouse_exporter) （用途 [去客户](https://github.com/kshvakov/clickhouse/)) + - [clickhouse_exporter](https://github.com/hot-wifi/clickhouse_exporter) (uses [Go client](https://github.com/kshvakov/clickhouse/)) - [Nagios](https://www.nagios.org/) - [check_clickhouse](https://github.com/exogroup/check_clickhouse/) - [check_clickhouse.py](https://github.com/innogames/igmonplugins/blob/master/src/check_clickhouse.py) - [Zabbix](https://www.zabbix.com) - - [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/Altinity/clickhouse-zabbix-template) + - [clickhouse-zabbix-template](https://github.com/Altinity/clickhouse-zabbix-template) - [Sematext](https://sematext.com/) - - [clickhouse积分](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse) -- 记录 + - [clickhouse integration](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse) +- Logging - [rsyslog](https://www.rsyslog.com/) - - [鹿茫house omhousee酶](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) + - [omclickhouse](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) - [fluentd](https://www.fluentd.org) - - [loghouse](https://github.com/flant/loghouse) (对于 [Kubernetes](https://kubernetes.io)) - - [Sematext](https://www.sematext.com/logagent) - - [logagent输出-插件-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/) -- 地理 + - [loghouse](https://github.com/flant/loghouse) (for [Kubernetes](https://kubernetes.io)) + - [logagent](https://www.sematext.com/logagent) + - [logagent output-plugin-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/) +- Geo - [MaxMind](https://dev.maxmind.com/geoip/) - - [ﾂ环板-ｮﾂ嘉ｯﾂ偲青clickｼﾂ氾ｶﾂ鉄ﾂ工ﾂ渉](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip) + - [clickhouse-maxmind-geoip](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip) -## 编程语言生态系统 {#bian-cheng-yu-yan-sheng-tai-xi-tong} +## 编程语言 {#programming-language-ecosystems} - Python - [SQLAlchemy](https://www.sqlalchemy.org) - - [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/cloudflare/sqlalchemy-clickhouse) (使用 [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm)) - - [熊猫](https://pandas.pydata.org) + - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (uses [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm)) + - [pandas](https://pandas.pydata.org) - [pandahouse](https://github.com/kszucs/pandahouse) - PHP - [Doctrine](https://www.doctrine-project.org/) - [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) - R - [dplyr](https://db.rstudio.com/dplyr/) - - [RClickhouse](https://github.com/IMSMWU/RClickhouse) (使用 [ﾂ暗ｪﾂ氾环催ﾂ団](https://github.com/artpaul/clickhouse-cpp)) + - [RClickHouse](https://github.com/IMSMWU/RClickHouse) (uses [clickhouse-cpp](https://github.com/artpaul/clickhouse-cpp)) - Java - [Hadoop](http://hadoop.apache.org) - - [clickhouse-hdfs-装载机](https://github.com/jaykelin/clickhouse-hdfs-loader) (使用 [JDBC](../../sql-reference/table-functions/jdbc.md)) -- 斯卡拉 + - [clickhouse-hdfs-loader](https://github.com/jaykelin/clickhouse-hdfs-loader) (uses [JDBC](../../sql-reference/table-functions/jdbc.md)) +- Scala - [Akka](https://akka.io) - - [掳胫client-禄脢鹿脷露胫鲁隆鹿-client酶](https://github.com/crobox/clickhouse-scala-client) + - [clickhouse-scala-client](https://github.com/crobox/clickhouse-scala-client) - C# - [ADO.NET](https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/ado-net-overview) - - [克莱克豪斯Ado](https://github.com/killwort/ClickHouse-Net) - - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) - - [ClickHouse.Net.Migrations](https://github.com/ilyabreev/ClickHouse.Net.Migrations) -- 仙丹 + - [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net) + - [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client) + - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) + - [ClickHouse.Net.Migrations](https://github.com/ilyabreev/ClickHouse.Net.Migrations) +- Elixir - [Ecto](https://github.com/elixir-ecto/ecto) - - [clickhouse_ecto](https://github.com/appodeal/clickhouse_ecto) + - [clickhouse_ecto](https://github.com/appodeal/clickhouse_ecto) - Ruby - [Ruby on Rails](https://rubyonrails.org/) - [activecube](https://github.com/bitquery/activecube) + - [ActiveRecord](https://github.com/PNixx/clickhouse-activerecord) - [GraphQL](https://github.com/graphql) - [activecube-graphql](https://github.com/bitquery/activecube-graphql) -[来源文章](https://clickhouse.tech/docs/zh/interfaces/third-party/integrations/) +[源文章](https://clickhouse.tech/docs/en/interfaces/third-party/integrations/) diff --git a/docs/zh/interfaces/third-party/proxy.md b/docs/zh/interfaces/third-party/proxy.md index 798717c0602..a2050863f30 100644 --- a/docs/zh/interfaces/third-party/proxy.md +++ b/docs/zh/interfaces/third-party/proxy.md @@ -1,37 +1,44 @@ -# 来自第三方开发人员的代理服务器 {#lai-zi-di-san-fang-kai-fa-ren-yuan-de-dai-li-fu-wu-qi} +--- +toc_priority: 29 +toc_title: 第三方代理 +--- -[chproxy](https://github.com/Vertamedia/chproxy) 是ClickHouse数据库的http代理和负载均衡器。 +# 第三方代理 {#proxy-servers-from-third-party-developers} -特征 +## chproxy {#chproxy} -*每用户路由和响应缓存。 -*灵活的限制。 -\*自动SSL证书续订。 +[chproxy](https://github.com/Vertamedia/chproxy), 是一个用于ClickHouse数据库的HTTP代理和负载均衡器。 -在Go中实现。 +特性: + +- 用户路由和响应缓存。 +- 灵活的限制。 +- 自动SSL证书续订。 + +使用go语言实现。 ## KittenHouse {#kittenhouse} -[KittenHouse](https://github.com/VKCOM/kittenhouse) 设计为ClickHouse和应用程序服务器之间的本地代理，以防在应用程序端缓冲INSERT数据是不可能或不方便的。 +[KittenHouse](https://github.com/VKCOM/kittenhouse)被设计为ClickHouse和应用服务器之间的本地代理，以防不可能或不方便在应用程序端缓冲插入数据。 -特征： +特性: -*内存和磁盘数据缓冲。 -*每表路由。 -\*负载平衡和健康检查。 +- 内存和磁盘上的数据缓冲。 +- 表路由。 +- 负载平衡和运行状况检查。 -在Go中实现。 +使用go语言实现。 -## ﾂ环板-ｮﾂ嘉ｯﾂ偲 {#clickhouse-bulk} +## ClickHouse-Bulk {#clickhouse-bulk} -[ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/nikepan/clickhouse-bulk) 是一个简单的ClickHouse插入收集器。 +[ClickHouse-Bulk](https://github.com/nikepan/clickhouse-bulk)是一个简单的ClickHouse收集器。 -特征： +特性: -*分组请求并按阈值或间隔发送。 -*多个远程服务器。 -\*基本身份验证。 +- 按阈值或间隔对请求进行分组并发送。 +- 多个远程服务器。 +- 基本身份验证。 -在Go中实现。 +使用go语言实现。 -[来源文章](https://clickhouse.tech/docs/zh/interfaces/third-party/proxy/) +[Original article](https://clickhouse.tech/docs/en/interfaces/third-party/proxy/) From 321ca2654eb5b41500a7fbacaf16894a3c90591a Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 21 Dec 2020 19:40:55 +0800 Subject: [PATCH 088/284] translate documentation for whats-new --- docs/zh/whats-new/index.md | 8 +-- docs/zh/whats-new/roadmap.md | 15 ++---- docs/zh/whats-new/security-changelog.md | 65 +++++++++++++++++++------ 3 files changed, 57 insertions(+), 31 deletions(-) diff --git a/docs/zh/whats-new/index.md b/docs/zh/whats-new/index.md index 0f248773402..1c28a538eed 100644 --- a/docs/zh/whats-new/index.md +++ b/docs/zh/whats-new/index.md @@ -1,8 +1,8 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u65B0\u589E\u5185\u5BB9" -toc_priority: 72 +toc_folder_title: ClickHouse事迹 +toc_priority: 82 --- +# ClickHouse变更及改动? {#whats-new-in-clickhouse} +对于已经发布的版本，有一个[roadmap](../whats-new/roadmap.md)和一个详细的[changelog](../whats-new/changelog/index.md)。 diff --git a/docs/zh/whats-new/roadmap.md b/docs/zh/whats-new/roadmap.md index 377746efcb7..df4b027f7da 100644 --- a/docs/zh/whats-new/roadmap.md +++ b/docs/zh/whats-new/roadmap.md @@ -1,17 +1,10 @@ --- toc_priority: 74 -toc_title: 路线图 +toc_title: Roadmap --- -# 路线图 {#roadmap} +# Roadmap {#roadmap} -## Q2 2020 {#q2-2020} - -- 和外部认证服务集成 - -## Q3 2020 {#q3-2020} - -- 资源池，为用户提供更精准的集群资源分配 - -{## [原始文档](https://clickhouse.tech/docs/en/roadmap/) ##} +`2021年Roadmap`已公布供公开讨论查看[这里](https://github.com/ClickHouse/ClickHouse/issues/17623). +{## [源文章](https://clickhouse.tech/docs/en/roadmap/) ##} diff --git a/docs/zh/whats-new/security-changelog.md b/docs/zh/whats-new/security-changelog.md index f21158c7aed..9cf0ca0b08c 100644 --- a/docs/zh/whats-new/security-changelog.md +++ b/docs/zh/whats-new/security-changelog.md @@ -1,41 +1,74 @@ -## 修复于 ClickHouse Release 18.12.13, 2018-09-10 {#xiu-fu-yu-clickhouse-release-18-12-13-2018-09-10} +--- +toc_priority: 76 +toc_title: 安全更新日志 +--- + +## 修复于ClickHouse Release 19.14.3.3, 2019-09-10 {#fixed-in-clickhouse-release-19-14-3-3-2019-09-10} + +### CVE-2019-15024 {#cve-2019-15024} + +对ZooKeeper具有写访问权限并且可以运行ClickHouse所在网络上可用的自定义服务器的攻击者可以创建一个自定义的恶意服务器，该服务器将充当ClickHouse副本并在ZooKeeper中注册。当另一个副本将从恶意副本获取数据部分时，它可以强制clickhouse服务器写入文件系统上的任意路径。 + +作者：Yandex信息安全团队Eldar Zaitov + +### CVE-2019-16535 {#cve-2019-16535} + +解压算法中的OOB-read、OOB-write和整数下溢可以通过本机协议实现RCE或DoS。 + +作者: Yandex信息安全团队Eldar Zaitov + +### CVE-2019-16536 {#cve-2019-16536} + +恶意的经过身份验证的客户端可能会触发导致DoS的堆栈溢出。 + +作者: Yandex信息安全团队Eldar Zaitov + +## 修复于ClickHouse Release 19.13.6.1, 2019-09-20 {#fixed-in-clickhouse-release-19-13-6-1-2019-09-20} + +### CVE-2019-18657 {#cve-2019-18657} + +表函数`url`存在允许攻击者在请求中插入任意HTTP标头的漏洞。 + +作者: [Nikita Tikhomirov](https://github.com/NSTikhomirov) + +## 修复于ClickHouse Release 18.12.13, 2018-09-10 {#fixed-in-clickhouse-release-18-12-13-2018-09-10} ### CVE-2018-14672 {#cve-2018-14672} -加载CatBoost模型的功能，允许遍历路径并通过错误消息读取任意文件。 +加载CatBoost模型的函数允许路径遍历和通过错误消息读取任意文件。 -来源: Yandex信息安全团队的Andrey Krasichkov +作者：Yandex信息安全团队Andrey Krasichkov -## 修复于 ClickHouse Release 18.10.3, 2018-08-13 {#xiu-fu-yu-clickhouse-release-18-10-3-2018-08-13} +## 修复于Release 18.10.3, 2018-08-13 {#fixed-in-clickhouse-release-18-10-3-2018-08-13} ### CVE-2018-14671 {#cve-2018-14671} -unixODBC允许从文件系统加载任意共享对象，从而导致«远程执行代码»漏洞。 +unixODBC允许从文件系统加载任意共享对象，从而导致远程代码执行漏洞。 -来源：Yandex信息安全团队的Andrey Krasichkov和Evgeny Sidorov +作者：Yandex信息安全团队Andrey Krasichkov和Evgeny Sidorov -## 修复于 ClickHouse Release 1.1.54388, 2018-06-28 {#xiu-fu-yu-clickhouse-release-1-1-54388-2018-06-28} +## 修复于ClickHouse Release 1.1.54388, 2018-06-28 {#fixed-in-clickhouse-release-1-1-54388-2018-06-28} ### CVE-2018-14668 {#cve-2018-14668} -远程表函数功能允许在 «user», «password» 及 «default_database» 字段中使用任意符号，从而导致跨协议请求伪造攻击。 +`remote`表函数允许在`user`，`password`和`default_database`字段中使用任意符号，从而导致跨协议请求伪造攻击。 -来源：Yandex信息安全团队的Andrey Krasichkov +者：Yandex信息安全团队Andrey Krasichkov -## 修复于 ClickHouse Release 1.1.54390, 2018-07-06 {#xiu-fu-yu-clickhouse-release-1-1-54390-2018-07-06} +## 修复于ClickHouse Release 1.1.54390, 2018-07-06 {#fixed-in-clickhouse-release-1-1-54390-2018-07-06} ### CVE-2018-14669 {#cve-2018-14669} -ClickHouse MySQL客户端启用了 «LOAD DATA LOCAL INFILE» 功能，该功能允许恶意MySQL数据库从连接的ClickHouse服务器读取任意文件。 +ClickHouse MySQL客户端启用了`LOAD DATA LOCAL INFILE`功能，允许恶意MySQL数据库从连接的ClickHouse服务器读取任意文件。 -来源：Yandex信息安全团队的Andrey Krasichkov和Evgeny Sidorov +作者：Yandex信息安全团队Andrey Krasichkov和Evgeny Sidorov -## 修复于 ClickHouse Release 1.1.54131, 2017-01-10 {#xiu-fu-yu-clickhouse-release-1-1-54131-2017-01-10} +## 修复于ClickHouse Release 1.1.54131, 2017-01-10 {#fixed-in-clickhouse-release-1-1-54131-2017-01-10} ### CVE-2018-14670 {#cve-2018-14670} -deb软件包中的错误配置可能导致使用未经授权的数据库。 +deb包中的错误配置可能导致未经授权使用数据库。 -来源：英国国家网络安全中心（NCSC） +作者：英国国家网络安全中心（NCSC） -[来源文章](https://clickhouse.tech/docs/en/security_changelog/) +{## [Original article](https://clickhouse.tech/docs/en/security_changelog/) ##} From e0400e335b28d1337a8dfd7eb0709747ad976f11 Mon Sep 17 00:00:00 2001 From: "Islam Israfilov (Islam93)" Date: Mon, 21 Dec 2020 16:21:44 +0300 Subject: [PATCH 089/284] see also block fixes for datetime rus doc --- docs/ru/sql-reference/data-types/datetime.md | 12 +++++++----- docs/ru/sql-reference/data-types/datetime64.md | 11 ++++++----- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/ru/sql-reference/data-types/datetime.md b/docs/ru/sql-reference/data-types/datetime.md index 87c5da68f35..f9b7c8e142c 100644 --- a/docs/ru/sql-reference/data-types/datetime.md +++ b/docs/ru/sql-reference/data-types/datetime.md @@ -116,12 +116,14 @@ FROM dt ## See Also {#see-also} -- [Функции преобразования типов](../../sql-reference/data-types/datetime.md) -- [Функции для работы с датой и временем](../../sql-reference/data-types/datetime.md) -- [Функции для работы с массивами](../../sql-reference/data-types/datetime.md) +- [Функции преобразования типов](../../sql-reference/functions/type-conversion-functions.md) +- [Функции для работы с датой и временем](../../sql-reference/functions/date-time-functions.md) +- [Функции для работы с массивами](../../sql-reference/functions/array-functions.md) - [Настройка `date_time_input_format`](../../operations/settings/settings.md#settings-date_time_input_format) -- [Конфигурационный параметр сервера `timezone`](../../sql-reference/data-types/datetime.md#server_configuration_parameters-timezone) -- [Операторы для работы с датой и временем](../../sql-reference/data-types/datetime.md#operators-datetime) +- [Настройка `date_time_output_format`](../../operations/settings/settings.md#settings-date_time_output_format) +- [Конфигурационный параметр сервера `timezone`](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) +- [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime) - [Тип данных `Date`](date.md) +- [Тип данных `DateTime64`](datetime64.md) [Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/datetime/) diff --git a/docs/ru/sql-reference/data-types/datetime64.md b/docs/ru/sql-reference/data-types/datetime64.md index 0a602e44636..b5e185c9132 100644 --- a/docs/ru/sql-reference/data-types/datetime64.md +++ b/docs/ru/sql-reference/data-types/datetime64.md @@ -92,11 +92,12 @@ FROM dt ## See Also {#see-also} -- [Функции преобразования типов](../../sql-reference/data-types/datetime64.md) -- [Функции для работы с датой и временем](../../sql-reference/data-types/datetime64.md) -- [Функции для работы с массивами](../../sql-reference/data-types/datetime64.md) +- [Функции преобразования типов](../../sql-reference/functions/type-conversion-functions.md) +- [Функции для работы с датой и временем](../../sql-reference/functions/date-time-functions.md) +- [Функции для работы с массивами](../../sql-reference/functions/array-functions.md) - [Настройка `date_time_input_format`](../../operations/settings/settings.md#settings-date_time_input_format) -- [Конфигурационный параметр сервера `timezone`](../../sql-reference/data-types/datetime64.md#server_configuration_parameters-timezone) -- [Операторы для работы с датой и временем](../../sql-reference/data-types/datetime64.md#operators-datetime) +- [Настройка `date_time_output_format`](../../operations/settings/settings.md#settings-date_time_output_format) +- [Конфигурационный параметр сервера `timezone`](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) +- [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime) - [Тип данных `Date`](date.md) - [Тип данных `DateTime`](datetime.md) From 6a93e487c6e08de2180b76e6a68b07cf8b1db20e Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Mon, 21 Dec 2020 22:30:37 +0300 Subject: [PATCH 090/284] Document the countMatches function. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Задокументировал функцию countMatches. --- .../functions/string-search-functions.md | 57 +++++++++++++++++-- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 3cde7dd71d4..957866faea3 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -400,7 +400,8 @@ Result: └──────────────────────────────────────────────────────────────────────────────────────────┘ ``` -**See also** +**See Also** + - [extractAllGroupsVertical](#extractallgroups-vertical) ## extractAllGroupsVertical {#extractallgroups-vertical} @@ -440,7 +441,8 @@ Result: └────────────────────────────────────────────────────────────────────────────────────────┘ ``` -**See also** +**See Also** + - [extractAllGroupsHorizontal](#extractallgroups-horizontal) ## like(haystack, pattern), haystack LIKE pattern operator {#function-like} @@ -590,8 +592,55 @@ Result: └───────────────────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/string_search_functions/) - ## countMatches(haystack, pattern) {#countmatcheshaystack-pattern} Returns the number of regular expression matches for a `pattern` in a `haystack`. + +**Syntax** + +``` sql +countMatches(haystack, pattern) +``` + +**Parameters** + +- `haystack` — The string to search in. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `pattern` — The regular expression with [re2 syntax](https://github.com/google/re2/wiki/Syntax). [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- The number of matches. + +Type: [UInt64](../../sql-reference/data-types/int-uint.md). + +**Examples** + +Query: + +``` sql +SELECT countMatches('foobar.com', 'o') +``` + +Result: + +``` text +┌─countMatches('foobar.com', 'o')─┐ +│ 3 │ +└─────────────────────────────────┘ +``` + +Query: + +``` sql +SELECT countMatches('aaaa', 'aa') +``` + +Result: + +``` text +┌─countMatches('aaaa', 'aa')─┐ +│ 2 │ +└───────────────────────────────┘ +``` + +[Original article](https://clickhouse.tech/docs/en/query_language/functions/string_search_functions/) From 302e8f697d4039b9887fecd8ff09e530de7cb638 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Mon, 21 Dec 2020 23:13:26 +0300 Subject: [PATCH 091/284] Document tcpPort function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Задокументировал функцию tcp_port. --- .../functions/other-functions.md | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 51a1f6b4cd7..4d69edb3ec1 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -1677,4 +1677,44 @@ Result: UNSUPPORTED_METHOD ``` +## tcpPort {#tcpPort} + +Returns TCP port number listened by this server. + +**Syntax** + +``` sql +tcpPort() +``` + +**Parameters** + +- None. + +**Returned value** + +- The TCP port number. + +Type: [UInt16](../../sql-reference/data-types/int-uint.md). + +**Example** + +Query: + +``` sql +SELECT tcpPort(); +``` + +Result: + +``` text +┌─tcpPort()─┐ +│ 9000 │ +└───────────┘ +``` + +**See Also** + +- [tcp_port](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port) + [Original article](https://clickhouse.tech/docs/en/query_language/functions/other_functions/) From 356e9ad19567357a1062638c62aefeadd0d4924d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 21 Dec 2020 23:51:44 +0300 Subject: [PATCH 092/284] Update datetime.md --- docs/ru/sql-reference/data-types/datetime.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/sql-reference/data-types/datetime.md b/docs/ru/sql-reference/data-types/datetime.md index f9b7c8e142c..9894fa2802b 100644 --- a/docs/ru/sql-reference/data-types/datetime.md +++ b/docs/ru/sql-reference/data-types/datetime.md @@ -119,8 +119,8 @@ FROM dt - [Функции преобразования типов](../../sql-reference/functions/type-conversion-functions.md) - [Функции для работы с датой и временем](../../sql-reference/functions/date-time-functions.md) - [Функции для работы с массивами](../../sql-reference/functions/array-functions.md) -- [Настройка `date_time_input_format`](../../operations/settings/settings.md#settings-date_time_input_format) -- [Настройка `date_time_output_format`](../../operations/settings/settings.md#settings-date_time_output_format) +- [Настройка `date_time_input_format`](../../operations/settings/settings/#settings-date_time_input_format) +- [Настройка `date_time_output_format`](../../operations/settings/settings/) - [Конфигурационный параметр сервера `timezone`](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) - [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime) - [Тип данных `Date`](date.md) From 35413635e7e36cd09fc2ba8667e5a23c92bcbb09 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 22 Dec 2020 13:41:12 +0300 Subject: [PATCH 093/284] Add ability to change some types for primary key --- src/Storages/MergeTree/MergeTreeData.cpp | 23 +++++-- ...ing_to_low_cardinality_key_alter.reference | 8 +++ ...11_string_to_low_cardinality_key_alter.sql | 67 +++++++++++++++++++ 3 files changed, 91 insertions(+), 7 deletions(-) create mode 100644 tests/queries/0_stateless/01611_string_to_low_cardinality_key_alter.reference create mode 100644 tests/queries/0_stateless/01611_string_to_low_cardinality_key_alter.sql diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 9fa19859c7f..888fbcc7bde 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -1316,10 +1317,10 @@ void MergeTreeData::dropIfEmpty() namespace { -/// Conversion that is allowed for partition key. -/// Partition key should be serialized in the same way after conversion. +/// Conversion that is allowed for serializable key (primary key, sorting key). +/// Key should be serialized in the same way after conversion. /// NOTE: The list is not complete. -bool isSafeForPartitionKeyConversion(const IDataType * from, const IDataType * to) +bool isSafeForKeyConversion(const IDataType * from, const IDataType * to) { if (from->getName() == to->getName()) return true; @@ -1346,6 +1347,12 @@ bool isSafeForPartitionKeyConversion(const IDataType * from, const IDataType * t return false; } + if (const auto * from_lc = typeid_cast(from)) + return from_lc->getDictionaryType()->equals(*to); + + if (const auto * to_lc = typeid_cast(to)) + return to_lc->getDictionaryType()->equals(*from); + return false; } @@ -1540,7 +1547,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S auto it = old_types.find(command.column_name); assert(it != old_types.end()); - if (!isSafeForPartitionKeyConversion(it->second, command.data_type.get())) + if (!isSafeForKeyConversion(it->second, command.data_type.get())) throw Exception("ALTER of partition key column " + backQuoteIfNeed(command.column_name) + " from type " + it->second->getName() + " to type " + command.data_type->getName() + " is not safe because it can change the representation of partition key", @@ -1554,9 +1561,11 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S { auto it = old_types.find(command.column_name); assert(it != old_types.end()); - throw Exception("ALTER of key column " + backQuoteIfNeed(command.column_name) + " from type " - + it->second->getName() + " to type " + command.data_type->getName() + " must be metadata-only", - ErrorCodes::ALTER_OF_COLUMN_IS_FORBIDDEN); + if (!isSafeForKeyConversion(it->second, command.data_type.get())) + throw Exception("ALTER of key column " + backQuoteIfNeed(command.column_name) + " from type " + + it->second->getName() + " to type " + command.data_type->getName() + + " is not safe because it can change the representation of primary key", + ErrorCodes::ALTER_OF_COLUMN_IS_FORBIDDEN); } } } diff --git a/tests/queries/0_stateless/01611_string_to_low_cardinality_key_alter.reference b/tests/queries/0_stateless/01611_string_to_low_cardinality_key_alter.reference new file mode 100644 index 00000000000..07a160752d4 --- /dev/null +++ b/tests/queries/0_stateless/01611_string_to_low_cardinality_key_alter.reference @@ -0,0 +1,8 @@ +CREATE TABLE default.table_with_lc_key\n(\n `enum_key` Enum8(\'y\' = 1, \'x\' = 2),\n `lc_key` String,\n `value` String\n)\nENGINE = MergeTree\nORDER BY (enum_key, lc_key)\nSETTINGS index_granularity = 8192 +y hello world +CREATE TABLE default.table_with_lc_key\n(\n `enum_key` Enum8(\'y\' = 1, \'x\' = 2, \'z\' = 3),\n `lc_key` String,\n `value` String\n)\nENGINE = MergeTree\nORDER BY (enum_key, lc_key)\nSETTINGS index_granularity = 8192 +y hello world +CREATE TABLE default.table_with_lc_key\n(\n `enum_key` Int8,\n `lc_key` String,\n `value` String\n)\nENGINE = MergeTree\nORDER BY (enum_key, lc_key)\nSETTINGS index_granularity = 8192 +1 hello world +CREATE TABLE default.table_with_string_key\n(\n `int_key` Int8,\n `str_key` LowCardinality(String),\n `value` String\n)\nENGINE = MergeTree\nORDER BY (int_key, str_key)\nSETTINGS index_granularity = 8192 +1 hello world diff --git a/tests/queries/0_stateless/01611_string_to_low_cardinality_key_alter.sql b/tests/queries/0_stateless/01611_string_to_low_cardinality_key_alter.sql new file mode 100644 index 00000000000..6478d33dfcc --- /dev/null +++ b/tests/queries/0_stateless/01611_string_to_low_cardinality_key_alter.sql @@ -0,0 +1,67 @@ +DROP TABLE IF EXISTS table_with_lc_key; + +CREATE TABLE table_with_lc_key +( + enum_key Enum8('x' = 2, 'y' = 1), + lc_key LowCardinality(String), + value String +) +ENGINE MergeTree() +ORDER BY (enum_key, lc_key); + +INSERT INTO table_with_lc_key VALUES(1, 'hello', 'world'); + +ALTER TABLE table_with_lc_key MODIFY COLUMN lc_key String; + +SHOW CREATE TABLE table_with_lc_key; + +DETACH TABLE table_with_lc_key; +ATTACH TABLE table_with_lc_key; + +SELECT * FROM table_with_lc_key WHERE enum_key > 0 and lc_key like 'h%'; + +ALTER TABLE table_with_lc_key MODIFY COLUMN enum_key Enum('x' = 2, 'y' = 1, 'z' = 3); +ALTER TABLE table_with_lc_key MODIFY COLUMN enum_key Enum16('x' = 2, 'y' = 1, 'z' = 3); --{serverError 524} +SHOW CREATE TABLE table_with_lc_key; + +DETACH TABLE table_with_lc_key; +ATTACH TABLE table_with_lc_key; + +SELECT * FROM table_with_lc_key WHERE enum_key > 0 and lc_key like 'h%'; + +ALTER TABLE table_with_lc_key MODIFY COLUMN enum_key Int8; + +SHOW CREATE TABLE table_with_lc_key; + +DETACH TABLE table_with_lc_key; +ATTACH TABLE table_with_lc_key; + +SELECT * FROM table_with_lc_key WHERE enum_key > 0 and lc_key like 'h%'; + +DROP TABLE IF EXISTS table_with_lc_key; + + +DROP TABLE IF EXISTS table_with_string_key; +CREATE TABLE table_with_string_key +( + int_key Int8, + str_key String, + value String +) +ENGINE MergeTree() +ORDER BY (int_key, str_key); + +INSERT INTO table_with_string_key VALUES(1, 'hello', 'world'); + +ALTER TABLE table_with_string_key MODIFY COLUMN str_key LowCardinality(String); + +SHOW CREATE TABLE table_with_string_key; + +DETACH TABLE table_with_string_key; +ATTACH TABLE table_with_string_key; + +SELECT * FROM table_with_string_key WHERE int_key > 0 and str_key like 'h%'; + +ALTER TABLE table_with_string_key MODIFY COLUMN int_key Enum8('y' = 1, 'x' = 2); --{serverError 524} + +DROP TABLE IF EXISTS table_with_string_key; From effb404c7efd8c4887bb76819b9dd702523bb100 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 22 Dec 2020 23:35:07 +0800 Subject: [PATCH 094/284] Better --- .../AggregateFunctionCount.h | 12 +++---- src/AggregateFunctions/AggregateFunctionIf.h | 19 +++++++---- .../AggregateFunctionNull.h | 4 +-- .../AggregateFunctionOrFill.h | 29 ++++++++--------- src/AggregateFunctions/AggregateFunctionSum.h | 12 +++---- src/AggregateFunctions/IAggregateFunction.h | 32 +++++++++---------- 6 files changed, 57 insertions(+), 51 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionCount.h b/src/AggregateFunctions/AggregateFunctionCount.h index a8cb9005388..63d3d34a0fd 100644 --- a/src/AggregateFunctions/AggregateFunctionCount.h +++ b/src/AggregateFunctions/AggregateFunctionCount.h @@ -44,11 +44,11 @@ public: } void addBatchSinglePlace( - size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena *, size_t num_arguments) const override + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena *, ssize_t if_argument_pos) const override { - if (num_arguments > 0) + if (if_argument_pos >= 0) { - const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); data(place).count += countBytesInFilter(flags); } else @@ -63,11 +63,11 @@ public: const IColumn ** columns, const UInt8 * null_map, Arena *, - size_t num_arguments) const override + ssize_t if_argument_pos) const override { - if (num_arguments > 0) + if (if_argument_pos >= 0) { - const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); data(place).count += countBytesInFilterWithNull(flags, null_map); } else diff --git a/src/AggregateFunctions/AggregateFunctionIf.h b/src/AggregateFunctions/AggregateFunctionIf.h index 8d7dced837d..4de72de883e 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.h +++ b/src/AggregateFunctions/AggregateFunctionIf.h @@ -35,6 +35,13 @@ public: if (num_arguments == 0) throw Exception("Aggregate function " + getName() + " require at least one argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + IAggregateFunction * sub = this; + while ((sub = sub->getNestedFunction().get())) + { + if (dynamic_cast(sub)) + throw Exception("Cannot nest -If combinator", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + if (!isUInt8(types.back())) throw Exception("Last argument for aggregate function " + getName() + " must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -86,15 +93,15 @@ public: size_t place_offset, const IColumn ** columns, Arena * arena, - size_t) const override + ssize_t) const override { - nested_func->addBatch(batch_size, places, place_offset, columns, arena, num_arguments); + nested_func->addBatch(batch_size, places, place_offset, columns, arena, num_arguments - 1); } void addBatchSinglePlace( - size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t) const override + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t) const override { - nested_func->addBatchSinglePlace(batch_size, place, columns, arena, num_arguments); + nested_func->addBatchSinglePlace(batch_size, place, columns, arena, num_arguments - 1); } void addBatchSinglePlaceNotNull( @@ -103,9 +110,9 @@ public: const IColumn ** columns, const UInt8 * null_map, Arena * arena, - size_t) const override + ssize_t) const override { - nested_func->addBatchSinglePlaceNotNull(batch_size, place, columns, null_map, arena, num_arguments); + nested_func->addBatchSinglePlaceNotNull(batch_size, place, columns, null_map, arena, num_arguments - 1); } void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena * arena) const override diff --git a/src/AggregateFunctions/AggregateFunctionNull.h b/src/AggregateFunctions/AggregateFunctionNull.h index 79bebb9caa8..5c94e68cb26 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.h +++ b/src/AggregateFunctions/AggregateFunctionNull.h @@ -212,14 +212,14 @@ public: } void addBatchSinglePlace( - size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t num_arguments = 0) const override + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos = -1) const override { const ColumnNullable * column = assert_cast(columns[0]); const IColumn * nested_column = &column->getNestedColumn(); const UInt8 * null_map = column->getNullMapData().data(); this->nested_function->addBatchSinglePlaceNotNull( - batch_size, this->nestedPlace(place), &nested_column, null_map, arena, num_arguments); + batch_size, this->nestedPlace(place), &nested_column, null_map, arena, if_argument_pos); if constexpr (result_is_nullable) if (!memoryIsByte(null_map, batch_size, 1)) diff --git a/src/AggregateFunctions/AggregateFunctionOrFill.h b/src/AggregateFunctions/AggregateFunctionOrFill.h index 13a76e29d63..0b0b5e717a2 100644 --- a/src/AggregateFunctions/AggregateFunctionOrFill.h +++ b/src/AggregateFunctions/AggregateFunctionOrFill.h @@ -103,12 +103,11 @@ public: size_t place_offset, const IColumn ** columns, Arena * arena, - size_t num_arguments = 0) const override + ssize_t if_argument_pos = -1) const override { - // TODO we can devirtualize this too - if (num_arguments > 0) + if (if_argument_pos >= 0) { - const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); for (size_t i = 0; i < batch_size; ++i) { if (flags[i]) @@ -117,19 +116,19 @@ public: } else { - nested_function->addBatch(batch_size, places, place_offset, columns, arena, num_arguments); + nested_function->addBatch(batch_size, places, place_offset, columns, arena, if_argument_pos); for (size_t i = 0; i < batch_size; ++i) (places[i] + place_offset)[size_of_data] = 1; } } void addBatchSinglePlace( - size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t num_arguments = 0) const override + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos = -1) const override { - if (num_arguments > 0) + if (if_argument_pos >= 0) { - const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); - nested_function->addBatchSinglePlace(batch_size, place, columns, arena, num_arguments); + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); + nested_function->addBatchSinglePlace(batch_size, place, columns, arena, if_argument_pos); for (size_t i = 0; i < batch_size; ++i) { if (flags[i]) @@ -143,7 +142,7 @@ public: { if (batch_size) { - nested_function->addBatchSinglePlace(batch_size, place, columns, arena, num_arguments); + nested_function->addBatchSinglePlace(batch_size, place, columns, arena, if_argument_pos); place[size_of_data] = 1; } } @@ -155,12 +154,12 @@ public: const IColumn ** columns, const UInt8 * null_map, Arena * arena, - size_t num_arguments = 0) const override + ssize_t if_argument_pos = -1) const override { - if (num_arguments > 0) + if (if_argument_pos >= 0) { - const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); - nested_function->addBatchSinglePlaceNotNull(batch_size, place, columns, null_map, arena, num_arguments); + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); + nested_function->addBatchSinglePlaceNotNull(batch_size, place, columns, null_map, arena, if_argument_pos); for (size_t i = 0; i < batch_size; ++i) { if (flags[i] && !null_map[i]) @@ -174,7 +173,7 @@ public: { if (batch_size) { - nested_function->addBatchSinglePlaceNotNull(batch_size, place, columns, null_map, arena, num_arguments); + nested_function->addBatchSinglePlaceNotNull(batch_size, place, columns, null_map, arena, if_argument_pos); for (size_t i = 0; i < batch_size; ++i) { if (!null_map[i]) diff --git a/src/AggregateFunctions/AggregateFunctionSum.h b/src/AggregateFunctions/AggregateFunctionSum.h index 1a207cbff0b..ecc47eecab4 100644 --- a/src/AggregateFunctions/AggregateFunctionSum.h +++ b/src/AggregateFunctions/AggregateFunctionSum.h @@ -283,11 +283,11 @@ public: /// Vectorized version when there is no GROUP BY keys. void addBatchSinglePlace( - size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t num_arguments) const override + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos) const override { - if (num_arguments > 0) + if (if_argument_pos >= 0) { - const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); for (size_t i = 0; i < batch_size; ++i) { if (flags[i]) @@ -302,12 +302,12 @@ public: } void addBatchSinglePlaceNotNull( - size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena * arena, size_t num_arguments) + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena * arena, ssize_t if_argument_pos) const override { - if (num_arguments > 0) + if (if_argument_pos >= 0) { - const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); for (size_t i = 0; i < batch_size; ++i) if (!null_map[i] && flags[i]) add(place, columns, i, arena); diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 8f8fba23707..7b702c7c335 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -147,12 +147,12 @@ public: size_t place_offset, const IColumn ** columns, Arena * arena, - size_t num_arguments = 0) const = 0; + ssize_t if_argument_pos = -1) const = 0; /** The same for single place. */ virtual void addBatchSinglePlace( - size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t num_arguments = 0) const = 0; + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos = -1) const = 0; /** The same for single place when need to aggregate only filtered data. */ @@ -162,10 +162,10 @@ public: const IColumn ** columns, const UInt8 * null_map, Arena * arena, - size_t num_arguments = 0) const = 0; + ssize_t if_argument_pos = -1) const = 0; virtual void addBatchSinglePlaceFromInterval( - size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t num_arguments = 0) + size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos = -1) const = 0; /** In addition to addBatch, this method collects multiple rows of arguments into array "places" @@ -242,11 +242,11 @@ public: size_t place_offset, const IColumn ** columns, Arena * arena, - size_t num_arguments = 0) const override + ssize_t if_argument_pos = -1) const override { - if (num_arguments > 0) + if (if_argument_pos >= 0) { - const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); for (size_t i = 0; i < batch_size; ++i) { if (flags[i]) @@ -261,11 +261,11 @@ public: } void addBatchSinglePlace( - size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t num_arguments = 0) const override + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos = -1) const override { - if (num_arguments > 0) + if (if_argument_pos >= 0) { - const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); for (size_t i = 0; i < batch_size; ++i) { if (flags[i]) @@ -285,11 +285,11 @@ public: const IColumn ** columns, const UInt8 * null_map, Arena * arena, - size_t num_arguments = 0) const override + ssize_t if_argument_pos = -1) const override { - if (num_arguments > 0) + if (if_argument_pos >= 0) { - const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); for (size_t i = 0; i < batch_size; ++i) if (!null_map[i] && flags[i]) static_cast(this)->add(place, columns, i, arena); @@ -303,12 +303,12 @@ public: } void addBatchSinglePlaceFromInterval( - size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena, size_t num_arguments = 0) + size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos = -1) const override { - if (num_arguments > 0) + if (if_argument_pos >= 0) { - const auto & flags = assert_cast(*columns[num_arguments - 1]).getData(); + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); for (size_t i = batch_begin; i < batch_end; ++i) { if (flags[i]) From a25e00a3d92cd45266316de809189be638487238 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Tue, 22 Dec 2020 18:37:34 +0300 Subject: [PATCH 095/284] Update docs/en/sql-reference/functions/other-functions.md --- docs/en/sql-reference/functions/other-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 4d69edb3ec1..43f66913780 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -1679,7 +1679,7 @@ UNSUPPORTED_METHOD ## tcpPort {#tcpPort} -Returns TCP port number listened by this server. +Returns [native interface](../../interfaces/tcp.md) TCP port number listened by this server. **Syntax** From a6d420d809e4adecb1fc4c6a3b9bb5f8fdd0822e Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Tue, 22 Dec 2020 21:32:31 +0300 Subject: [PATCH 096/284] Update other-functions.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Перевел на русский язык. --- .../functions/other-functions.md | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 91bf0f5b3a0..4df16739f60 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -1589,4 +1589,44 @@ SELECT countDigits(toDecimal32(1, 9)), countDigits(toDecimal32(-1, 9)), 10 10 19 19 39 39 ``` +## tcpPort {#tcpPort} + +Вовращает номер TCP порта, который использует сервер для [нативного протокола](../../interfaces/tcp.md). + +**Синтаксис** + +``` sql +tcpPort() +``` + +**Параметры** + +- Нет. + +**Возвращаемое значение** + +- Номер TCP порта. + +Тип: [UInt16](../../sql-reference/data-types/int-uint.md). + +**Example** + +Запрос: + +``` sql +SELECT tcpPort(); +``` + +Результат: + +``` text +┌─tcpPort()─┐ +│ 9000 │ +└───────────┘ +``` + +**Смотрите также** + +- [tcp_port](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port) + [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/other_functions/) From 14c11ac46f1616aef766061ca84c741d3d8bae18 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Tue, 22 Dec 2020 21:37:42 +0300 Subject: [PATCH 097/284] Update other-functions.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Перевел example. --- docs/ru/sql-reference/functions/other-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 4df16739f60..14e09ce7295 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -1609,7 +1609,7 @@ tcpPort() Тип: [UInt16](../../sql-reference/data-types/int-uint.md). -**Example** +**Пример** Запрос: From 46a8027947493f84e908fef1c5f4a1da6214a2f3 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Tue, 22 Dec 2020 22:10:03 +0300 Subject: [PATCH 098/284] Translation into Russian language MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Выполнил перевод на русский язык. --- .../functions/string-search-functions.md | 8 +-- .../functions/string-search-functions.md | 51 +++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 957866faea3..ef7e378ad11 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -618,15 +618,15 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Query: ``` sql -SELECT countMatches('foobar.com', 'o') +SELECT countMatches('foobar.com', 'o+') ``` Result: ``` text -┌─countMatches('foobar.com', 'o')─┐ -│ 3 │ -└─────────────────────────────────┘ +┌─countMatches('foobar.com', 'o+')─┐ +│ 2 │ +└──────────────────────────────────┘ ``` Query: diff --git a/docs/ru/sql-reference/functions/string-search-functions.md b/docs/ru/sql-reference/functions/string-search-functions.md index d2f1119783b..8bc2bb12382 100644 --- a/docs/ru/sql-reference/functions/string-search-functions.md +++ b/docs/ru/sql-reference/functions/string-search-functions.md @@ -521,5 +521,56 @@ SELECT * FROM Months WHERE ilike(name, '%j%') !!! note "Примечание" Для случая UTF-8 мы используем триграммное расстояние. Вычисление n-граммного расстояния не совсем честное. Мы используем 2-х байтные хэши для хэширования n-грамм, а затем вычисляем (не)симметрическую разность между хэш таблицами – могут возникнуть коллизии. В формате UTF-8 без учета регистра мы не используем честную функцию `tolower` – мы обнуляем 5-й бит (нумерация с нуля) каждого байта кодовой точки, а также первый бит нулевого байта, если байтов больше 1 – это работает для латиницы и почти для всех кириллических букв. + +## countMatches(haystack, pattern) {#countmatcheshaystack-pattern} + +Возвращает количество совпадений, найденных в строке `haystack`, для регулярного выражения `pattern`. + +**Синтаксис** + +``` sql +countMatches(haystack, pattern) +``` + +**Параметры** + +- `haystack` — строка, по которой выполняется поиск. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `pattern` — регулярное выражение, построенное по синтаксическим правилам [re2](https://github.com/google/re2/wiki/Syntax). [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +- Количество совпадений. + +Тип: [UInt64](../../sql-reference/data-types/int-uint.md). + +**Примеры** + +Запрос: + +``` sql +SELECT countMatches('foobar.com', 'o+') +``` + +Результат: + +``` text +┌─countMatches('foobar.com', 'o+')─┐ +│ 2 │ +└──────────────────────────────────┘ +``` + +Запрос: + +``` sql +SELECT countMatches('aaaa', 'aa') +``` + +Результат: + +``` text +┌─countMatches('aaaa', 'aa')─┐ +│ 2 │ +└───────────────────────────────┘ +``` [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/string_search_functions/) From f91adc1207a702b104f15e5504484a4b9df728ac Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Wed, 23 Dec 2020 01:58:02 +0300 Subject: [PATCH 099/284] DOCSUP-5043: Add RU translation for PR #16378 --- docs/ru/commercial/cloud.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/ru/commercial/cloud.md b/docs/ru/commercial/cloud.md index e6b0309c456..8d02a6332a7 100644 --- a/docs/ru/commercial/cloud.md +++ b/docs/ru/commercial/cloud.md @@ -18,4 +18,16 @@ toc_title: "\u041f\u043e\u0441\u0442\u0430\u0432\u0449\u0438\u043a\u0438\u0020\u - Шифрование и изоляция - Автоматизированное техническое обслуживание +## Altinity.Cloud {#altinity.cloud} + +[Altinity.Cloud](https://altinity.com/cloud-database/) — это полностью управляемый ClickHouse-as-a-Service для публичного облака Amazon. +- Быстрое развертывание кластеров ClickHouse на ресурсах Amazon +- Easy scale-out/scale-in as well as vertical scaling of nodes +- Легкое горизонтальное масштабирование также, как и вертикальное масштабирование узлов +- Изолированные виртуальные сети для каждого клиента с общедоступным эндпоинтом или пирингом VPC +- Настраиваемые типы и объемы хранилищ +- Cross-AZ scaling for performance and high availability +- Cross-AZ масштабирование для повышения производительности и высокой доступности +- Встроенный мониторинг и редактор SQL-запросов + {## [Оригинальная статья](https://clickhouse.tech/docs/ru/commercial/cloud/) ##} From 41397b7455763a8f78a7391a5cda420313dabf65 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Wed, 23 Dec 2020 02:02:04 +0300 Subject: [PATCH 100/284] DOCSUP-5043: Add RU translation for PR # 15423 --- docs/en/commercial/cloud.md | 2 +- docs/ru/operations/server-configuration-parameters/settings.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/commercial/cloud.md b/docs/en/commercial/cloud.md index 06216517db8..03c8b7645a3 100644 --- a/docs/en/commercial/cloud.md +++ b/docs/en/commercial/cloud.md @@ -20,7 +20,7 @@ toc_title: Cloud ## Altinity.Cloud {#altinity.cloud} -[Altinity.Cloud](https://altinity.com/cloud-database/) is a fully managed ClickHouse-as-a-Service for the Amazon public cloud. +[Altinity.Cloud](https://altinity.com/cloud-database/) is a fully managed ClickHouse-as-a-Service for the Amazon public cloud. - Fast deployment of ClickHouse clusters on Amazon resources - Easy scale-out/scale-in as well as vertical scaling of nodes - Isolated per-tenant VPCs with public endpoint or VPC peering diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 58aae05f188..e07ae968177 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -420,7 +420,7 @@ ClickHouse проверяет условия для `min_part_size` и `min_part Возможные значения: - Положительное целое число. -- 0 — объём используемой памяти не ограничен. +- 0 — автоматически. Значение по умолчанию: `0`. From 6af30a5dca8db4fbe7d25bc283663cb755ee5488 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Wed, 23 Dec 2020 02:23:10 +0300 Subject: [PATCH 101/284] DOCSUP-5043: Add translations and updates for PR #15416 (EN & RU) --- .../settings.md | 17 +++++++++++---- docs/en/operations/settings/settings.md | 2 +- .../settings.md | 21 +++++++++++++++++++ docs/ru/operations/settings/settings.md | 2 +- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 533fcea5500..1877e474555 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -82,16 +82,25 @@ List of prefixes for [custom settings](../../operations/settings/index.md#custom ## core_dump -Configures soft limit for core dump file size, one gigabyte by default. +Configures soft limit for core dump file size. + +Possible values: + +- Positive integer. + +Default value: `1073741824`. + +!!! info "Note" + Hard limit is configured via system tools + +**Example** + ```xml 1073741824 ``` -(Hard limit is configured via system tools) - - ## default_database {#default-database} The default database. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 86e16fe4819..93d5bb9c67f 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1312,7 +1312,7 @@ See also: Write to a quorum timeout in milliseconds. If the timeout has passed and no write has taken place yet, ClickHouse will generate an exception and the client must repeat the query to write the same block to the same or any other replica. -Default value: 600000 milliseconds (ten minutes). +Default value: 600 000 milliseconds (ten minutes). See also: diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index e07ae968177..affe09c7703 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -80,6 +80,27 @@ ClickHouse проверяет условия для `min_part_size` и `min_part - [Пользовательские настройки](../../operations/settings/index.md#custom_settings) +## core_dump + +Задает мягкое ограничение для размера файла дампа ядра. + +Возможные значения: + +- Положительное целое число. + +Значение по умолчанию: `1073741824`. + +!!! info "Примечание" + Жесткое ограничение настраивается с помощью системных инструментов. + +**Пример** + +```xml + + 1073741824 + +``` + ## default\_database {#default-database} База данных по умолчанию. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 0a8094231c2..8e41fc421c6 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1258,7 +1258,7 @@ ClickHouse генерирует исключение Время ожидания кворумной записи в миллисекундах. Если время прошло, а запись так не состоялась, то ClickHouse сгенерирует исключение и клиент должен повторить запрос на запись того же блока на эту же или любую другую реплику. -Значение по умолчанию: 600000 миллисекунд (10 минут). +Значение по умолчанию: 600 000 миллисекунд (10 минут). См. также: From 7932f1ababbe722cfbb88f6df6496e4c9907c7e7 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Wed, 23 Dec 2020 02:27:53 +0300 Subject: [PATCH 102/284] DOCSUP-5043: Minor fix --- docs/ru/sql-reference/functions/tuple-map-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/tuple-map-functions.md b/docs/ru/sql-reference/functions/tuple-map-functions.md index 5d71443da7a..a2b25e68fe5 100644 --- a/docs/ru/sql-reference/functions/tuple-map-functions.md +++ b/docs/ru/sql-reference/functions/tuple-map-functions.md @@ -23,7 +23,7 @@ mapAdd(Tuple(Array, Array), Tuple(Array, Array) [, ...]) **Возвращаемое значение** -- Возвращает один [кортеж](../../sql-reference/data-types/tuple.md#tuplet1-t2), в котором первый массив содержит отсортированные ключи, а второй - значения. +- Возвращает один [кортеж](../../sql-reference/data-types/tuple.md#tuplet1-t2), в котором первый массив содержит отсортированные ключи, а второй — значения. **Пример** From f98cbd7696a24a722ad49185f255db1fa7394ba1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 23 Dec 2020 05:10:37 +0300 Subject: [PATCH 103/284] Fix TSan report in StorageSet and StorageJoin --- src/Interpreters/HashJoin.cpp | 2 ++ src/Interpreters/Set.cpp | 19 +++++++++++++++++++ src/Interpreters/Set.h | 9 +++------ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index cf73581c6d8..5d7fb213134 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -359,6 +359,7 @@ void HashJoin::init(Type type_) size_t HashJoin::getTotalRowCount() const { + std::shared_lock lock(data->rwlock); size_t res = 0; if (data->type == Type::CROSS) @@ -376,6 +377,7 @@ size_t HashJoin::getTotalRowCount() const size_t HashJoin::getTotalByteCount() const { + std::shared_lock lock(data->rwlock); size_t res = 0; if (data->type == Type::CROSS) diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index dcf7b3b3c24..ae2937ed796 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -286,6 +286,25 @@ ColumnPtr Set::execute(const Block & block, bool negative) const } +bool Set::empty() const +{ + std::shared_lock lock(rwlock); + return data.empty(); +} + +size_t Set::getTotalRowCount() const +{ + std::shared_lock lock(rwlock); + return data.getTotalRowCount(); +} + +size_t Set::getTotalByteCount() const +{ + std::shared_lock lock(rwlock); + return data.getTotalByteCount(); +} + + template void NO_INLINE Set::executeImpl( Method & method, diff --git a/src/Interpreters/Set.h b/src/Interpreters/Set.h index d3c9b8f684b..c9bfbf0625c 100644 --- a/src/Interpreters/Set.h +++ b/src/Interpreters/Set.h @@ -36,8 +36,6 @@ public: { } - bool empty() const { return data.empty(); } - /** Set can be created either from AST or from a stream of data (subquery result). */ @@ -58,8 +56,9 @@ public: */ ColumnPtr execute(const Block & block, bool negative) const; - size_t getTotalRowCount() const { return data.getTotalRowCount(); } - size_t getTotalByteCount() const { return data.getTotalByteCount(); } + bool empty() const; + size_t getTotalRowCount() const; + size_t getTotalByteCount() const; const DataTypes & getDataTypes() const { return data_types; } const DataTypes & getElementsTypes() const { return set_elements_types; } @@ -127,8 +126,6 @@ private: /** Protects work with the set in the functions `insertFromBlock` and `execute`. * These functions can be called simultaneously from different threads only when using StorageSet, - * and StorageSet calls only these two functions. - * Therefore, the rest of the functions for working with set are not protected. */ mutable std::shared_mutex rwlock; From 12a36c0e9f2ec8df35271779410ea84234a43b98 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 23 Dec 2020 12:34:15 +0800 Subject: [PATCH 104/284] allow nested if (useless but also harmless) --- src/AggregateFunctions/AggregateFunctionIf.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionIf.h b/src/AggregateFunctions/AggregateFunctionIf.h index 4de72de883e..e3a23a432c7 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.h +++ b/src/AggregateFunctions/AggregateFunctionIf.h @@ -35,13 +35,6 @@ public: if (num_arguments == 0) throw Exception("Aggregate function " + getName() + " require at least one argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - IAggregateFunction * sub = this; - while ((sub = sub->getNestedFunction().get())) - { - if (dynamic_cast(sub)) - throw Exception("Cannot nest -If combinator", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - if (!isUInt8(types.back())) throw Exception("Last argument for aggregate function " + getName() + " must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } From 7ab38d5007193aacefac3cffec83780016c22d43 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 23 Dec 2020 15:54:20 +0300 Subject: [PATCH 105/284] Use Port::Data instead of Chunk in LazyOutputFormat. --- src/Processors/Formats/IOutputFormat.cpp | 40 +++++++++++++++------ src/Processors/Formats/IOutputFormat.h | 13 ++++--- src/Processors/Formats/LazyOutputFormat.cpp | 14 ++++---- src/Processors/Formats/LazyOutputFormat.h | 16 ++++----- src/Processors/Port.h | 15 ++++---- 5 files changed, 63 insertions(+), 35 deletions(-) diff --git a/src/Processors/Formats/IOutputFormat.cpp b/src/Processors/Formats/IOutputFormat.cpp index 88649d9ca25..2f0ef603022 100644 --- a/src/Processors/Formats/IOutputFormat.cpp +++ b/src/Processors/Formats/IOutputFormat.cpp @@ -5,6 +5,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + IOutputFormat::IOutputFormat(const Block & header_, WriteBuffer & out_) : IProcessor({header_, header_, header_}, {}), out(out_) { @@ -30,7 +35,7 @@ IOutputFormat::Status IOutputFormat::prepare() if (!input.hasData()) return Status::NeedData; - current_chunk = input.pull(true); + current_chunk = input.pullData(true); current_block_kind = kind; has_input = true; return Status::Ready; @@ -44,23 +49,31 @@ IOutputFormat::Status IOutputFormat::prepare() return Status::Finished; } -static Chunk prepareTotals(Chunk chunk) +static Port::Data prepareTotals(Port::Data data) { - if (!chunk.hasRows()) + if (data.exception) + return data; + + if (!data.chunk.hasRows()) return {}; - if (chunk.getNumRows() > 1) + if (data.chunk.getNumRows() > 1) { /// This may happen if something like ARRAY JOIN was executed on totals. /// Skip rows except the first one. - auto columns = chunk.detachColumns(); + auto columns = data.chunk.detachColumns(); for (auto & column : columns) column = column->cut(0, 1); - chunk.setColumns(std::move(columns), 1); + data.chunk.setColumns(std::move(columns), 1); } - return chunk; + return data; +} + +void IOutputFormat::consume(Chunk) +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method consume s not implemented for {}", getName()); } void IOutputFormat::work() @@ -84,17 +97,24 @@ void IOutputFormat::work() switch (current_block_kind) { case Main: - result_rows += current_chunk.getNumRows(); - result_bytes += current_chunk.allocatedBytes(); + { + result_rows += current_chunk.chunk.getNumRows(); + result_bytes += current_chunk.chunk.allocatedBytes(); consume(std::move(current_chunk)); break; + } case Totals: - if (auto totals = prepareTotals(std::move(current_chunk))) + { + auto totals = prepareTotals(std::move(current_chunk)); + if (totals.exception || totals.chunk) consumeTotals(std::move(totals)); break; + } case Extremes: + { consumeExtremes(std::move(current_chunk)); break; + } } if (auto_flush) diff --git a/src/Processors/Formats/IOutputFormat.h b/src/Processors/Formats/IOutputFormat.h index 67c307df2aa..e3552b734e8 100644 --- a/src/Processors/Formats/IOutputFormat.h +++ b/src/Processors/Formats/IOutputFormat.h @@ -28,7 +28,7 @@ public: protected: WriteBuffer & out; - Chunk current_chunk; + Port::Data current_chunk; PortKind current_block_kind = PortKind::Main; bool has_input = false; bool finished = false; @@ -39,9 +39,14 @@ protected: RowsBeforeLimitCounterPtr rows_before_limit_counter; - virtual void consume(Chunk) = 0; + virtual void consume(Chunk); virtual void consumeTotals(Chunk) {} virtual void consumeExtremes(Chunk) {} + + virtual void consume(Port::Data data) { consume(data.getChunkOrTrow()); } + virtual void consumeTotals(Port::Data data) { consumeTotals(data.getChunkOrTrow()); } + virtual void consumeExtremes(Port::Data data) { consumeExtremes(data.getChunkOrTrow()); } + virtual void finalize() {} public: @@ -77,8 +82,8 @@ public: virtual void doWritePrefix() {} virtual void doWriteSuffix() { finalize(); } - void setTotals(const Block & totals) { consumeTotals(Chunk(totals.getColumns(), totals.rows())); } - void setExtremes(const Block & extremes) { consumeExtremes(Chunk(extremes.getColumns(), extremes.rows())); } + void setTotals(const Block & totals) { consumeTotals(Port::Data{.chunk = Chunk(totals.getColumns(), totals.rows())}); } + void setExtremes(const Block & extremes) { consumeExtremes(Port::Data{.chunk = Chunk(extremes.getColumns(), extremes.rows())}); } size_t getResultRows() const { return result_rows; } size_t getResultBytes() const { return result_bytes; } diff --git a/src/Processors/Formats/LazyOutputFormat.cpp b/src/Processors/Formats/LazyOutputFormat.cpp index 46287d1cce9..72996de9593 100644 --- a/src/Processors/Formats/LazyOutputFormat.cpp +++ b/src/Processors/Formats/LazyOutputFormat.cpp @@ -15,24 +15,24 @@ Chunk LazyOutputFormat::getChunk(UInt64 milliseconds) return {}; } - Chunk chunk; - if (!queue.tryPop(chunk, milliseconds)) + Port::Data data; + if (!queue.tryPop(data, milliseconds)) return {}; - if (chunk) - info.update(chunk.getNumRows(), chunk.allocatedBytes()); + if (!data.exception) + info.update(data.chunk.getNumRows(), data.chunk.allocatedBytes()); - return chunk; + return data.getChunkOrTrow(); } Chunk LazyOutputFormat::getTotals() { - return std::move(totals); + return totals.getChunkOrTrow(); } Chunk LazyOutputFormat::getExtremes() { - return std::move(extremes); + return extremes.getChunkOrTrow(); } void LazyOutputFormat::setRowsBeforeLimit(size_t rows_before_limit) diff --git a/src/Processors/Formats/LazyOutputFormat.h b/src/Processors/Formats/LazyOutputFormat.h index 06ec116f3dd..9f24e54735c 100644 --- a/src/Processors/Formats/LazyOutputFormat.h +++ b/src/Processors/Formats/LazyOutputFormat.h @@ -37,28 +37,28 @@ public: } protected: - void consume(Chunk chunk) override + void consume(Port::Data data) override { if (!finished_processing) - queue.emplace(std::move(chunk)); + queue.emplace(std::move(data)); } - void consumeTotals(Chunk chunk) override { totals = std::move(chunk); } - void consumeExtremes(Chunk chunk) override { extremes = std::move(chunk); } + void consumeTotals(Port::Data data) override { totals = std::move(data); } + void consumeExtremes(Port::Data data) override { extremes = std::move(data); } void finalize() override { finished_processing = true; /// In case we are waiting for result. - queue.emplace(Chunk()); + queue.emplace(Port::Data{}); } private: - ConcurrentBoundedQueue queue; - Chunk totals; - Chunk extremes; + ConcurrentBoundedQueue queue; + Port::Data totals; + Port::Data extremes; /// Is not used. static WriteBuffer out; diff --git a/src/Processors/Port.h b/src/Processors/Port.h index ac71c394518..c7401a18afe 100644 --- a/src/Processors/Port.h +++ b/src/Processors/Port.h @@ -60,6 +60,14 @@ protected: /// Note: std::variant can be used. But move constructor for it can't be inlined. Chunk chunk; std::exception_ptr exception; + + Chunk getChunkOrTrow() + { + if (exception) + std::rethrow_exception(std::move(exception)); + + return std::move(chunk); + } }; private: @@ -303,12 +311,7 @@ public: Chunk ALWAYS_INLINE pull(bool set_not_needed = false) { - auto data_ = pullData(set_not_needed); - - if (data_.exception) - std::rethrow_exception(data_.exception); - - return std::move(data_.chunk); + return pullData(set_not_needed).getChunkOrTrow(); } bool ALWAYS_INLINE isFinished() const From 12a659b9c03a108d2b98cef91133de181aafd1f0 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 23 Dec 2020 19:51:49 +0300 Subject: [PATCH 106/284] Fix build. --- src/Processors/Formats/IOutputFormat.h | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/Processors/Formats/IOutputFormat.h b/src/Processors/Formats/IOutputFormat.h index e3552b734e8..1080e5f6b06 100644 --- a/src/Processors/Formats/IOutputFormat.h +++ b/src/Processors/Formats/IOutputFormat.h @@ -82,8 +82,19 @@ public: virtual void doWritePrefix() {} virtual void doWriteSuffix() { finalize(); } - void setTotals(const Block & totals) { consumeTotals(Port::Data{.chunk = Chunk(totals.getColumns(), totals.rows())}); } - void setExtremes(const Block & extremes) { consumeExtremes(Port::Data{.chunk = Chunk(extremes.getColumns(), extremes.rows())}); } + void setTotals(const Block & totals) + { + Port::Data data; + data.chunk = Chunk(totals.getColumns(), totals.rows()); + consumeTotals(std::move(data)); + } + + void setExtremes(const Block & extremes) + { + Port::Data data; + data.chunk = Chunk(extremes.getColumns(), extremes.rows()); + consumeExtremes(std::move(data)); + } size_t getResultRows() const { return result_rows; } size_t getResultBytes() const { return result_bytes; } From 13e633c1ad009b474ac6a74b2748c16ec0d70fca Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 23 Dec 2020 13:17:18 -0500 Subject: [PATCH 107/284] Regenerating ldap/authentication/requirements/requirements.py --- .../requirements/requirements.py | 305 +++++++++++++++--- 1 file changed, 255 insertions(+), 50 deletions(-) diff --git a/tests/testflows/ldap/authentication/requirements/requirements.py b/tests/testflows/ldap/authentication/requirements/requirements.py index 4e955bf801b..25b943d18c2 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.py +++ b/tests/testflows/ldap/authentication/requirements/requirements.py @@ -790,9 +790,74 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite = Requirement( level=3, num='4.2.27') +RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed\n' + 'to be successfully authenticated for all consecutive requests without contacting the [LDAP] server.\n' + 'After period of time since the last successful attempt expires then on the authentication attempt\n' + 'SHALL result in contacting the [LDAP] server to verify the username and password. \n' + '\n' + ), + link=None, + level=3, + num='4.2.28') + +RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Default = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'SHALL have a default value of `0` that disables caching and forces contacting\n' + 'the [LDAP] server for each authentication request.\n' + '\n' + ), + link=None, + level=3, + num='4.2.29') + +RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Invalid = Requirement( + name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer.\n' + '\n' + 'For example:\n' + '\n' + '* negative integer\n' + '* string\n' + '* empty value\n' + '* extremely large positive value (overflow)\n' + '* extremely large negative value (overflow)\n' + '\n' + 'The error SHALL appear in the log and SHALL be similar to the following:\n' + '\n' + '```bash\n' + ' Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value*\n' + '```\n' + '\n' + ), + link=None, + level=3, + num='4.2.30') + RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Syntax', - version='1.0', + version='2.0', priority=None, group=None, type=None, @@ -808,6 +873,7 @@ RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( ' 636\n' ' cn=\n' ' , ou=users, dc=example, dc=com\n' + ' 0\n' ' yes\n' ' tls1.2\n' ' demand\n' @@ -823,7 +889,7 @@ RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( ), link=None, level=3, - num='4.2.28') + num='4.2.31') RQ_SRS_007_LDAP_Configuration_User_RBAC = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.RBAC', @@ -843,7 +909,7 @@ RQ_SRS_007_LDAP_Configuration_User_RBAC = Requirement( ), link=None, level=3, - num='4.2.29') + num='4.2.32') RQ_SRS_007_LDAP_Configuration_User_Syntax = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Syntax', @@ -871,7 +937,7 @@ RQ_SRS_007_LDAP_Configuration_User_Syntax = Requirement( ), link=None, level=3, - num='4.2.30') + num='4.2.33') RQ_SRS_007_LDAP_Configuration_User_Name_Empty = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.Empty', @@ -886,7 +952,7 @@ RQ_SRS_007_LDAP_Configuration_User_Name_Empty = Requirement( ), link=None, level=3, - num='4.2.31') + num='4.2.34') RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP', @@ -902,7 +968,7 @@ RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP = Requirement( ), link=None, level=3, - num='4.2.32') + num='4.2.35') RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined', @@ -919,7 +985,7 @@ RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined = Requireme ), link=None, level=3, - num='4.2.33') + num='4.2.36') RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty', @@ -936,7 +1002,7 @@ RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty = Requirement( ), link=None, level=3, - num='4.2.34') + num='4.2.37') RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer', @@ -951,7 +1017,7 @@ RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer = Requirement( ), link=None, level=3, - num='4.2.35') + num='4.2.38') RQ_SRS_007_LDAP_Configuration_User_Name_Long = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.Long', @@ -967,7 +1033,7 @@ RQ_SRS_007_LDAP_Configuration_User_Name_Long = Requirement( ), link=None, level=3, - num='4.2.36') + num='4.2.39') RQ_SRS_007_LDAP_Configuration_User_Name_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.UTF8', @@ -982,7 +1048,7 @@ RQ_SRS_007_LDAP_Configuration_User_Name_UTF8 = Requirement( ), link=None, level=3, - num='4.2.37') + num='4.2.40') RQ_SRS_007_LDAP_Authentication_Username_Empty = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.Empty', @@ -997,7 +1063,7 @@ RQ_SRS_007_LDAP_Authentication_Username_Empty = Requirement( ), link=None, level=3, - num='4.2.38') + num='4.2.41') RQ_SRS_007_LDAP_Authentication_Username_Long = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.Long', @@ -1012,7 +1078,7 @@ RQ_SRS_007_LDAP_Authentication_Username_Long = Requirement( ), link=None, level=3, - num='4.2.39') + num='4.2.42') RQ_SRS_007_LDAP_Authentication_Username_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.UTF8', @@ -1027,7 +1093,7 @@ RQ_SRS_007_LDAP_Authentication_Username_UTF8 = Requirement( ), link=None, level=3, - num='4.2.40') + num='4.2.43') RQ_SRS_007_LDAP_Authentication_Password_Empty = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.Empty', @@ -1044,7 +1110,7 @@ RQ_SRS_007_LDAP_Authentication_Password_Empty = Requirement( ), link=None, level=3, - num='4.2.41') + num='4.2.44') RQ_SRS_007_LDAP_Authentication_Password_Long = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.Long', @@ -1060,7 +1126,7 @@ RQ_SRS_007_LDAP_Authentication_Password_Long = Requirement( ), link=None, level=3, - num='4.2.42') + num='4.2.45') RQ_SRS_007_LDAP_Authentication_Password_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.UTF8', @@ -1076,7 +1142,64 @@ RQ_SRS_007_LDAP_Authentication_Password_UTF8 = Requirement( ), link=None, level=3, - num='4.2.43') + num='4.2.46') + +RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Performance = Requirement( + name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL provide better login performance of [LDAP] authenticated users\n' + 'when `verification_cooldown` parameter is set to a positive value when comparing\n' + 'to the the case when `verification_cooldown` is turned off either for a single user or multiple users\n' + 'making a large number of repeated requests.\n' + '\n' + ), + link=None, + level=3, + num='4.2.47') + +RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters = Requirement( + name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the\n' + '`verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values\n' + 'change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user\n' + "to result in contacting the [LDAP] server to verify user's username and password.\n" + '\n' + ), + link=None, + level=3, + num='4.2.48') + +RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_InvalidPassword = Requirement( + name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the\n' + '`verification_cooldown` parameter in the [LDAP] server configuration section\n' + 'for the user if the password provided in the current authentication attempt does not match\n' + 'the valid password provided during the first successful authentication request that was cached\n' + 'for this exact user. The reset SHALL cause the next authentication attempt for this user\n' + "to result in contacting the [LDAP] server to verify user's username and password.\n" + '\n' + ), + link=None, + level=3, + num='4.2.49') SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( name='SRS-007 ClickHouse Authentication of Users via LDAP', @@ -1150,22 +1273,28 @@ SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir', level=3, num='4.2.25'), Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile', level=3, num='4.2.26'), Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite', level=3, num='4.2.27'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.Syntax', level=3, num='4.2.28'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.RBAC', level=3, num='4.2.29'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.Syntax', level=3, num='4.2.30'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.Name.Empty', level=3, num='4.2.31'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP', level=3, num='4.2.32'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined', level=3, num='4.2.33'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty', level=3, num='4.2.34'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer', level=3, num='4.2.35'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.Name.Long', level=3, num='4.2.36'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.Name.UTF8', level=3, num='4.2.37'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Username.Empty', level=3, num='4.2.38'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Username.Long', level=3, num='4.2.39'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Username.UTF8', level=3, num='4.2.40'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Password.Empty', level=3, num='4.2.41'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Password.Long', level=3, num='4.2.42'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Password.UTF8', level=3, num='4.2.43'), + Heading(name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown', level=3, num='4.2.28'), + Heading(name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default', level=3, num='4.2.29'), + Heading(name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid', level=3, num='4.2.30'), + Heading(name='RQ.SRS-007.LDAP.Configuration.Server.Syntax', level=3, num='4.2.31'), + Heading(name='RQ.SRS-007.LDAP.Configuration.User.RBAC', level=3, num='4.2.32'), + Heading(name='RQ.SRS-007.LDAP.Configuration.User.Syntax', level=3, num='4.2.33'), + Heading(name='RQ.SRS-007.LDAP.Configuration.User.Name.Empty', level=3, num='4.2.34'), + Heading(name='RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP', level=3, num='4.2.35'), + Heading(name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined', level=3, num='4.2.36'), + Heading(name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty', level=3, num='4.2.37'), + Heading(name='RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer', level=3, num='4.2.38'), + Heading(name='RQ.SRS-007.LDAP.Configuration.User.Name.Long', level=3, num='4.2.39'), + Heading(name='RQ.SRS-007.LDAP.Configuration.User.Name.UTF8', level=3, num='4.2.40'), + Heading(name='RQ.SRS-007.LDAP.Authentication.Username.Empty', level=3, num='4.2.41'), + Heading(name='RQ.SRS-007.LDAP.Authentication.Username.Long', level=3, num='4.2.42'), + Heading(name='RQ.SRS-007.LDAP.Authentication.Username.UTF8', level=3, num='4.2.43'), + Heading(name='RQ.SRS-007.LDAP.Authentication.Password.Empty', level=3, num='4.2.44'), + Heading(name='RQ.SRS-007.LDAP.Authentication.Password.Long', level=3, num='4.2.45'), + Heading(name='RQ.SRS-007.LDAP.Authentication.Password.UTF8', level=3, num='4.2.46'), + Heading(name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance', level=3, num='4.2.47'), + Heading(name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters', level=3, num='4.2.48'), + Heading(name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword', level=3, num='4.2.49'), Heading(name='References', level=1, num='5'), ), requirements=( @@ -1218,6 +1347,9 @@ SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( RQ_SRS_007_LDAP_Configuration_Server_TLSCACertDir, RQ_SRS_007_LDAP_Configuration_Server_TLSCACertFile, RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite, + RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown, + RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Default, + RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Invalid, RQ_SRS_007_LDAP_Configuration_Server_Syntax, RQ_SRS_007_LDAP_Configuration_User_RBAC, RQ_SRS_007_LDAP_Configuration_User_Syntax, @@ -1234,9 +1366,13 @@ SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( RQ_SRS_007_LDAP_Authentication_Password_Empty, RQ_SRS_007_LDAP_Authentication_Password_Long, RQ_SRS_007_LDAP_Authentication_Password_UTF8, + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Performance, + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters, + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_InvalidPassword, ), content=''' # SRS-007 ClickHouse Authentication of Users via LDAP +# Software Requirements Specification ## Table of Contents @@ -1295,22 +1431,28 @@ SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( * 4.2.25 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir](#rqsrs-007ldapconfigurationservertlscacertdir) * 4.2.26 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile](#rqsrs-007ldapconfigurationservertlscacertfile) * 4.2.27 [RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite](#rqsrs-007ldapconfigurationservertlsciphersuite) - * 4.2.28 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) - * 4.2.29 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) - * 4.2.30 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) - * 4.2.31 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) - * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) - * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) - * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) - * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) - * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) - * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) - * 4.2.38 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) - * 4.2.39 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) - * 4.2.40 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) - * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) - * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) - * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) + * 4.2.28 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown](#rqsrs-007ldapconfigurationserververificationcooldown) + * 4.2.29 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default](#rqsrs-007ldapconfigurationserververificationcooldowndefault) + * 4.2.30 [RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid](#rqsrs-007ldapconfigurationserververificationcooldowninvalid) + * 4.2.31 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) + * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) + * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) + * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) + * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) + * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) + * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) + * 4.2.38 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) + * 4.2.39 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) + * 4.2.40 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) + * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) + * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) + * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) + * 4.2.44 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) + * 4.2.45 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) + * 4.2.46 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) + * 4.2.47 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance](#rqsrs-007ldapauthenticationverificationcooldownperformance) + * 4.2.48 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters](#rqsrs-007ldapauthenticationverificationcooldownresetchangeincoreserverparameters) + * 4.2.49 [RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword](#rqsrs-007ldapauthenticationverificationcooldownresetinvalidpassword) * 5 [References](#references) ## Revision History @@ -1631,9 +1773,44 @@ For example, The available suites SHALL depend on the [OpenSSL] library version and variant used to build [ClickHouse] and therefore might change. -#### RQ.SRS-007.LDAP.Configuration.Server.Syntax +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown version: 1.0 +[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section +that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed +to be successfully authenticated for all consecutive requests without contacting the [LDAP] server. +After period of time since the last successful attempt expires then on the authentication attempt +SHALL result in contacting the [LDAP] server to verify the username and password. + +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default +version: 1.0 + +[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section +SHALL have a default value of `0` that disables caching and forces contacting +the [LDAP] server for each authentication request. + +#### RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid +version: 1.0 + +[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer. + +For example: + +* negative integer +* string +* empty value +* extremely large positive value (overflow) +* extremely large negative value (overflow) + +The error SHALL appear in the log and SHALL be similar to the following: + +```bash + Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value* +``` + +#### RQ.SRS-007.LDAP.Configuration.Server.Syntax +version: 2.0 + [ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml` configuration file or of any configuration file inside the `config.d` directory. @@ -1644,6 +1821,7 @@ configuration file or of any configuration file inside the `config.d` directory. 636 cn= , ou=users, dc=example, dc=com + 0 yes tls1.2 demand @@ -1759,6 +1937,33 @@ version: 1.0 [ClickHouse] SHALL support [UTF-8] characters in passwords used to authenticate users using an [LDAP] server. +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance +version: 1.0 + +[ClickHouse] SHALL provide better login performance of [LDAP] authenticated users +when `verification_cooldown` parameter is set to a positive value when comparing +to the the case when `verification_cooldown` is turned off either for a single user or multiple users +making a large number of repeated requests. + +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters +version: 1.0 + +[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the +`verification_cooldown` parameter in the [LDAP] server configuration section +if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values +change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user +to result in contacting the [LDAP] server to verify user's username and password. + +#### RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword +version: 1.0 + +[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the +`verification_cooldown` parameter in the [LDAP] server configuration section +for the user if the password provided in the current authentication attempt does not match +the valid password provided during the first successful authentication request that was cached +for this exact user. The reset SHALL cause the next authentication attempt for this user +to result in contacting the [LDAP] server to verify user's username and password. + ## References * **ClickHouse:** https://clickhouse.tech From 31e4512c3da40006fd64ec48948e64605e43251b Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 23 Dec 2020 22:36:20 +0300 Subject: [PATCH 108/284] One UNION section. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Объединил все в один раздел UNION. --- docs/en/operations/settings/settings.md | 51 +---------------- .../sql-reference/statements/select/index.md | 2 +- .../select/{union-all.md => union.md} | 53 +++++++++++++++--- .../sql-reference/statements/select/index.md | 2 +- .../select/{union-all.md => union.md} | 2 +- .../sql-reference/statements/select/index.md | 2 +- .../select/{union-all.md => union.md} | 2 +- .../sql-reference/statements/select/index.md | 2 +- .../select/{union-all.md => union.md} | 2 +- .../statements/select/union-all.md | 1 - .../sql-reference/statements/select/union.md | 1 + docs/ru/operations/settings/settings.md | 51 +---------------- .../sql-reference/statements/select/index.md | 2 +- .../select/{union-all.md => union.md} | 55 ++++++++++++++++--- .../statements/select/union-all.md | 1 - .../sql-reference/statements/select/union.md | 1 + .../sql-reference/statements/select/index.md | 2 +- .../select/{union-all.md => union.md} | 2 +- 18 files changed, 109 insertions(+), 125 deletions(-) rename docs/en/sql-reference/statements/select/{union-all.md => union.md} (51%) rename docs/es/sql-reference/statements/select/{union-all.md => union.md} (97%) rename docs/fa/sql-reference/statements/select/{union-all.md => union.md} (97%) rename docs/fr/sql-reference/statements/select/{union-all.md => union.md} (97%) delete mode 120000 docs/ja/sql-reference/statements/select/union-all.md create mode 100644 docs/ja/sql-reference/statements/select/union.md rename docs/ru/sql-reference/statements/select/{union-all.md => union.md} (53%) delete mode 120000 docs/tr/sql-reference/statements/select/union-all.md create mode 100644 docs/tr/sql-reference/statements/select/union.md rename docs/zh/sql-reference/statements/select/{union-all.md => union.md} (97%) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 98ccdab0c2e..4fc5fad7aa9 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2460,7 +2460,7 @@ Default value: `0`. ## union_default_mode {#union-default-mode} -Sets a special mode for combining `SELECT` query results using the [UNION](../../sql-reference/statements/select/union-all.md) expression. +Sets a mode for combining `SELECT` query results. The setting is only used when shared with [UNION](../../sql-reference/statements/select/union.md) without explicitly specifying the `UNION ALL` or `UNION DISTINCT`. Possible values: @@ -2470,53 +2470,6 @@ Possible values: Default value: `'DISTINCT'`. -**Example of using the 'DISTINCT' value** - -Query: - -```sql -SET union_default_mode = 'DISTINCT'; -SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; -``` - -Result: - -```text -┌─1─┐ -│ 1 │ -└───┘ -┌─1─┐ -│ 2 │ -└───┘ -┌─1─┐ -│ 3 │ -└───┘ -``` - -**Example of using the 'ALL' value** - -Query: - -```sql -SET union_default_mode = 'ALL'; -SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; -``` - -Result: - -```text -┌─1─┐ -│ 1 │ -└───┘ -┌─1─┐ -│ 2 │ -└───┘ -┌─1─┐ -│ 2 │ -└───┘ -┌─1─┐ -│ 3 │ -└───┘ -``` +See examples in [UNION](../../sql-reference/statements/select/union.md). [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index 60c769c4660..ed69198ed4d 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -46,7 +46,7 @@ Specifics of each optional clause are covered in separate sections, which are li - [SELECT clause](#select-clause) - [DISTINCT clause](../../../sql-reference/statements/select/distinct.md) - [LIMIT clause](../../../sql-reference/statements/select/limit.md) -- [UNION clause](../../../sql-reference/statements/select/union-all.md) +- [UNION clause](../../../sql-reference/statements/select/union.md) - [INTO OUTFILE clause](../../../sql-reference/statements/select/into-outfile.md) - [FORMAT clause](../../../sql-reference/statements/select/format.md) diff --git a/docs/en/sql-reference/statements/select/union-all.md b/docs/en/sql-reference/statements/select/union.md similarity index 51% rename from docs/en/sql-reference/statements/select/union-all.md rename to docs/en/sql-reference/statements/select/union.md index 85c50450273..099c680d9a7 100644 --- a/docs/en/sql-reference/statements/select/union-all.md +++ b/docs/en/sql-reference/statements/select/union.md @@ -2,7 +2,7 @@ toc_title: UNION --- -# UNION ALL Clause {#union-all-clause} +# UNION Clause {#union-clause} You can use `UNION ALL` to combine any number of `SELECT` queries by extending their results. Example: @@ -25,16 +25,55 @@ Type casting is performed for unions. For example, if two queries being combined Queries that are parts of `UNION ALL` can’t be enclosed in round brackets. [ORDER BY](../../../sql-reference/statements/select/order-by.md) and [LIMIT](../../../sql-reference/statements/select/limit.md) are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with `UNION ALL` in a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause. -# UNION DISTINCT Clause {#union-distinct-clause} +By default, `UNION` has the same behavior as `UNION DISTINCT`. The difference between `UNION ALL` and `UNION DISTINCT` is that `UNION DISTINCT` will do a distinct transform for union result, it is equivalent to `SELECT DISTINCT` from a subquery containing `UNION ALL`. -The difference between `UNION ALL` and `UNION DISTINCT` is that `UNION DISTINCT` will do a distinct transform for union result, it is equivalent to `SELECT DISTINCT` from a subquery containing `UNION ALL`. +If you use `UNION` without explicitly specifying `UNION ALL` or `UNION DISTINCT`, you can specify the union mode using the [union_default_mode](../../../operations/settings/settings.md#union-default-mode) setting. The setting values can be `ALL`, `DISTINCT` or an empty string. However, if you use `UNION` with `union_default_mode` setting to empty string, it will throw an exception. The following examples demonstrate the results of queries with different values setting. -# UNION Clause {#union-clause} +Query: -By default, `UNION` has the same behavior as `UNION DISTINCT`, but you can specify union mode by [union_default_mode](../../../operations/settings/settings.md#union-default-mode) setting, values can be `ALL`, `DISTINCT` or an empty string. However, if you use `UNION` with `union_default_mode` setting to empty string, it will throw an exception. +```sql +SET union_default_mode = 'DISTINCT'; +SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; +``` -## Implementation Details {#implementation-details} +Result: + +```text +┌─1─┐ +│ 1 │ +└───┘ +┌─1─┐ +│ 2 │ +└───┘ +┌─1─┐ +│ 3 │ +└───┘ +``` + +Query: + +```sql +SET union_default_mode = 'ALL'; +SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; +``` + +Result: + +```text +┌─1─┐ +│ 1 │ +└───┘ +┌─1─┐ +│ 2 │ +└───┘ +┌─1─┐ +│ 2 │ +└───┘ +┌─1─┐ +│ 3 │ +└───┘ +``` Queries that are parts of `UNION/UNION ALL/UNION DISTINCT` can be run simultaneously, and their results can be mixed together. -[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/union-all/) +[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/union/) diff --git a/docs/es/sql-reference/statements/select/index.md b/docs/es/sql-reference/statements/select/index.md index a5ff9820a2b..653f737b1d0 100644 --- a/docs/es/sql-reference/statements/select/index.md +++ b/docs/es/sql-reference/statements/select/index.md @@ -44,7 +44,7 @@ Los detalles de cada cláusula opcional se cubren en secciones separadas, que se - [Cláusula HAVING](having.md) - [Cláusula SELECT](#select-clause) - [Cláusula LIMIT](limit.md) -- [UNION ALL cláusula](union-all.md) +- [UNION ALL cláusula](union.md) ## SELECT Cláusula {#select-clause} diff --git a/docs/es/sql-reference/statements/select/union-all.md b/docs/es/sql-reference/statements/select/union.md similarity index 97% rename from docs/es/sql-reference/statements/select/union-all.md rename to docs/es/sql-reference/statements/select/union.md index b2b45ba770e..d3aec34ba4b 100644 --- a/docs/es/sql-reference/statements/select/union-all.md +++ b/docs/es/sql-reference/statements/select/union.md @@ -3,7 +3,7 @@ machine_translated: true machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd --- -# UNION ALL Cláusula {#union-all-clause} +# UNION Cláusula {#union-clause} Usted puede utilizar `UNION ALL` combinar cualquier número de `SELECT` consultas extendiendo sus resultados. Ejemplo: diff --git a/docs/fa/sql-reference/statements/select/index.md b/docs/fa/sql-reference/statements/select/index.md index 2ab3fea2ff1..90541b80636 100644 --- a/docs/fa/sql-reference/statements/select/index.md +++ b/docs/fa/sql-reference/statements/select/index.md @@ -44,7 +44,7 @@ SELECT [DISTINCT] expr_list - [داشتن بند](having.md) - [انتخاب بند](#select-clause) - [بند محدود](limit.md) -- [اتحادیه همه بند](union-all.md) +- [اتحادیه همه بند](union.md) ## انتخاب بند {#select-clause} diff --git a/docs/fa/sql-reference/statements/select/union-all.md b/docs/fa/sql-reference/statements/select/union.md similarity index 97% rename from docs/fa/sql-reference/statements/select/union-all.md rename to docs/fa/sql-reference/statements/select/union.md index 3c4fe5c1546..03d723e2338 100644 --- a/docs/fa/sql-reference/statements/select/union-all.md +++ b/docs/fa/sql-reference/statements/select/union.md @@ -3,7 +3,7 @@ machine_translated: true machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd --- -# اتحادیه همه بند {#union-all-clause} +# اتحادیه همه بند {#union-clause} شما می توانید استفاده کنید `UNION ALL` برای ترکیب هر تعداد از `SELECT` نمایش داده شد با گسترش نتایج خود را. مثال: diff --git a/docs/fr/sql-reference/statements/select/index.md b/docs/fr/sql-reference/statements/select/index.md index 5073469e651..1d53ae80eb4 100644 --- a/docs/fr/sql-reference/statements/select/index.md +++ b/docs/fr/sql-reference/statements/select/index.md @@ -44,7 +44,7 @@ Spécificités de chaque clause facultative, sont couverts dans des sections dis - [Clause HAVING](having.md) - [Clause SELECT](#select-clause) - [Clause LIMIT](limit.md) -- [Clause UNION ALL](union-all.md) +- [Clause UNION ALL](union.md) ## Clause SELECT {#select-clause} diff --git a/docs/fr/sql-reference/statements/select/union-all.md b/docs/fr/sql-reference/statements/select/union.md similarity index 97% rename from docs/fr/sql-reference/statements/select/union-all.md rename to docs/fr/sql-reference/statements/select/union.md index 63e9987965f..9ae65ebcf72 100644 --- a/docs/fr/sql-reference/statements/select/union-all.md +++ b/docs/fr/sql-reference/statements/select/union.md @@ -3,7 +3,7 @@ machine_translated: true machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd --- -# Clause UNION ALL {#union-all-clause} +# Clause UNION ALL {#union-clause} Vous pouvez utiliser `UNION ALL` à combiner `SELECT` requêtes en étendant leurs résultats. Exemple: diff --git a/docs/ja/sql-reference/statements/select/union-all.md b/docs/ja/sql-reference/statements/select/union-all.md deleted file mode 120000 index 837caae2698..00000000000 --- a/docs/ja/sql-reference/statements/select/union-all.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/union-all.md \ No newline at end of file diff --git a/docs/ja/sql-reference/statements/select/union.md b/docs/ja/sql-reference/statements/select/union.md new file mode 100644 index 00000000000..0eb8db0be7a --- /dev/null +++ b/docs/ja/sql-reference/statements/select/union.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/union.md \ No newline at end of file diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index d94f7ddfae8..00f59d9ce48 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2326,7 +2326,7 @@ SELECT number FROM numbers(3) FORMAT JSONEachRow; ## union_default_mode {#union-default-mode} -Устанавливает особый режим объединения результатов `SELECT` запросов, используя выражение [UNION](../../sql-reference/statements/select/union-all.md). +Устанавливает режим объединения результатов `SELECT` запросов. Настройка используется только при совместном использовании с [UNION](../../sql-reference/statements/select/union.md) без явного указания `UNION ALL` или `UNION DISTINCT`. Возможные значения: @@ -2336,53 +2336,6 @@ SELECT number FROM numbers(3) FORMAT JSONEachRow; Значение по умолчанию: `'DISTINCT'`. -**Пример использования 'DISTINCT' в качестве значения настройки** - -Запрос: - -```sql -SET union_default_mode = 'DISTINCT'; -SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; -``` - -Результат: - -```text -┌─1─┐ -│ 1 │ -└───┘ -┌─1─┐ -│ 2 │ -└───┘ -┌─1─┐ -│ 3 │ -└───┘ -``` - -**Пример использования 'ALL' в качестве значения настройки** - -Запрос: - -```sql -SET union_default_mode = 'ALL'; -SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; -``` - -Результат: - -```text -┌─1─┐ -│ 1 │ -└───┘ -┌─1─┐ -│ 2 │ -└───┘ -┌─1─┐ -│ 2 │ -└───┘ -┌─1─┐ -│ 3 │ -└───┘ -``` +Смотрите примеры в разделе [UNION](../../sql-reference/statements/select/union.md). [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) diff --git a/docs/ru/sql-reference/statements/select/index.md b/docs/ru/sql-reference/statements/select/index.md index c2e05f05079..bf4ae44a6f1 100644 --- a/docs/ru/sql-reference/statements/select/index.md +++ b/docs/ru/sql-reference/statements/select/index.md @@ -44,7 +44,7 @@ SELECT [DISTINCT] expr_list - [Секция SELECT](#select-clause) - [Секция DISTINCT](distinct.md) - [Секция LIMIT](limit.md) -- [Секция UNION ALL](union-all.md) +- [Секция UNION ALL](union.md) - [Секция INTO OUTFILE](into-outfile.md) - [Секция FORMAT](format.md) diff --git a/docs/ru/sql-reference/statements/select/union-all.md b/docs/ru/sql-reference/statements/select/union.md similarity index 53% rename from docs/ru/sql-reference/statements/select/union-all.md rename to docs/ru/sql-reference/statements/select/union.md index 72fc7299b30..4d7d96a23db 100644 --- a/docs/ru/sql-reference/statements/select/union-all.md +++ b/docs/ru/sql-reference/statements/select/union.md @@ -1,8 +1,8 @@ --- -toc_title: UNION ALL +toc_title: UNION --- -# Секция UNION ALL {#union-all-clause} +# Секция UNION {#union-clause} Вы можете использовать `UNION ALL` чтобы объединить любое количество `SELECT` запросы путем расширения их результатов. Пример: @@ -25,16 +25,55 @@ SELECT CounterID, 2 AS table, sum(Sign) AS c Запросы, которые являются частью `UNION ALL` не могут быть заключен в круглые скобки. [ORDER BY](order-by.md) и [LIMIT](limit.md) применяются к отдельным запросам, а не к конечному результату. Если вам нужно применить преобразование к конечному результату, вы можете разместить все объединенные с помощью `UNION ALL` запросы в подзапрос в секции [FROM](from.md). -# Секция UNION DISTINCT {#union-distinct-clause} +По умолчанию, `UNION` ведет себя так же, как и `UNION DISTINCT`. Разница между `UNION ALL` и `UNION DISTINCT` в том, что `UNION DISTINCT` выполняет явное преобразование для результата объединения. Это равнозначно выражению `SELECT DISTINCT` из подзапроса, содержащего `UNION ALL`. -Разница между `UNION ALL` и `UNION DISTINCT` в том, что `UNION DISTINCT` выполняет явное преобразование для результата объединения. Это равнозначно выражению `SELECT DISTINCT` из подзапроса, содержащего `UNION ALL`. +Если используете `UNION` без явного указания `UNION ALL` или `UNION DISTINCT`, то вы можете указать режим объединения с помощью настройки [union_default_mode](../../../operations/settings/settings.md#union-default-mode), значениями которой могут быть `ALL`, `DISTINCT` или пустая строка. Однако если вы используете `UNION` с настройкой `union_default_mode`, значением которой является пустая строка, то будет сгенерировано исключение. В следующих примерах продемонстрированы результаты запросов при разных значениях настройки. -# Секция UNION {#union-clause} +Запрос: -По умолчанию, `UNION` ведет себя так же, как и `UNION DISTINCT`. Но вы можете указать особый режим объединения с помощью настройки [union_default_mode](../../../operations/settings/settings.md#union-default-mode), значениями которой могут быть `ALL`, `DISTINCT` или пустая строка. Однако если вы используете `UNION` с настройкой `union_default_mode`, значением которой является пустая строка, то будет сгенерировано исключение. +```sql +SET union_default_mode = 'DISTINCT'; +SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; +``` -## Детали реализации {#implementation-details} +Результат: + +```text +┌─1─┐ +│ 1 │ +└───┘ +┌─1─┐ +│ 2 │ +└───┘ +┌─1─┐ +│ 3 │ +└───┘ +``` + +Запрос: + +```sql +SET union_default_mode = 'ALL'; +SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; +``` + +Результат: + +```text +┌─1─┐ +│ 1 │ +└───┘ +┌─1─┐ +│ 2 │ +└───┘ +┌─1─┐ +│ 2 │ +└───┘ +┌─1─┐ +│ 3 │ +└───┘ +``` Запросы, которые являются частью `UNION ALL`, выполняются параллельно, и их результаты могут быть смешаны вместе. -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/select/union-all/) +[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/select/union/) diff --git a/docs/tr/sql-reference/statements/select/union-all.md b/docs/tr/sql-reference/statements/select/union-all.md deleted file mode 120000 index 837caae2698..00000000000 --- a/docs/tr/sql-reference/statements/select/union-all.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/union-all.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/union.md b/docs/tr/sql-reference/statements/select/union.md new file mode 100644 index 00000000000..0eb8db0be7a --- /dev/null +++ b/docs/tr/sql-reference/statements/select/union.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/union.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/index.md b/docs/zh/sql-reference/statements/select/index.md index cdfd64ff190..689a4f91a0c 100644 --- a/docs/zh/sql-reference/statements/select/index.md +++ b/docs/zh/sql-reference/statements/select/index.md @@ -46,7 +46,7 @@ SELECT [DISTINCT] expr_list - [SELECT 子句](#select-clause) - [DISTINCT 子句](../../../sql-reference/statements/select/distinct.md) - [LIMIT 子句](../../../sql-reference/statements/select/limit.md) -- [UNION ALL 子句](../../../sql-reference/statements/select/union-all.md) +- [UNION ALL 子句](../../../sql-reference/statements/select/union.md) - [INTO OUTFILE 子句](../../../sql-reference/statements/select/into-outfile.md) - [FORMAT 子句](../../../sql-reference/statements/select/format.md) diff --git a/docs/zh/sql-reference/statements/select/union-all.md b/docs/zh/sql-reference/statements/select/union.md similarity index 97% rename from docs/zh/sql-reference/statements/select/union-all.md rename to docs/zh/sql-reference/statements/select/union.md index d32ae896f55..1d88f9674c8 100644 --- a/docs/zh/sql-reference/statements/select/union-all.md +++ b/docs/zh/sql-reference/statements/select/union.md @@ -2,7 +2,7 @@ toc_title: UNION ALL --- -# UNION ALL子句 {#union-all-clause} +# UNION ALL子句 {#union-clause} 你可以使用 `UNION ALL` 结合任意数量的 `SELECT` 来扩展其结果。 示例: From 844239b1c3af0ad970a05f3d1df83ddfeffca3af Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 24 Dec 2020 07:03:33 +0300 Subject: [PATCH 109/284] tmp fixes (split me) --- src/Interpreters/ActionsDAG.cpp | 3 + src/Interpreters/ExpressionAnalyzer.cpp | 63 +++++++++++++++++-- src/Interpreters/ExpressionAnalyzer.h | 2 + .../ExtractExpressionInfoVisitor.cpp | 28 +++++++-- .../ExtractExpressionInfoVisitor.h | 3 +- src/Interpreters/InterpreterSelectQuery.cpp | 3 + .../PredicateExpressionsOptimizer.cpp | 14 ++++- src/Interpreters/PredicateRewriteVisitor.cpp | 2 +- src/Interpreters/TreeOptimizer.cpp | 12 ++++ src/Interpreters/TreeRewriter.cpp | 2 + src/Processors/QueryPlan/AggregatingStep.cpp | 4 +- src/Processors/QueryPlan/AggregatingStep.h | 4 +- src/Processors/QueryPlan/ArrayJoinStep.cpp | 2 +- src/Processors/QueryPlan/ArrayJoinStep.h | 2 +- src/Processors/QueryPlan/CreatingSetsStep.cpp | 4 +- src/Processors/QueryPlan/CreatingSetsStep.h | 4 +- src/Processors/QueryPlan/DistinctStep.cpp | 2 +- src/Processors/QueryPlan/DistinctStep.h | 2 +- src/Processors/QueryPlan/ExpressionStep.cpp | 2 +- src/Processors/QueryPlan/ExpressionStep.h | 2 +- src/Processors/QueryPlan/FillingStep.cpp | 2 +- src/Processors/QueryPlan/FillingStep.h | 2 +- src/Processors/QueryPlan/FilterStep.cpp | 2 +- src/Processors/QueryPlan/FilterStep.h | 2 +- .../QueryPlan/FinishSortingStep.cpp | 2 +- src/Processors/QueryPlan/FinishSortingStep.h | 2 +- src/Processors/QueryPlan/IQueryPlanStep.cpp | 10 ++- src/Processors/QueryPlan/IQueryPlanStep.h | 25 ++++---- src/Processors/QueryPlan/ISourceStep.cpp | 2 +- src/Processors/QueryPlan/ISourceStep.h | 2 +- .../QueryPlan/ITransformingStep.cpp | 2 +- src/Processors/QueryPlan/ITransformingStep.h | 2 +- src/Processors/QueryPlan/LimitByStep.cpp | 2 +- src/Processors/QueryPlan/LimitByStep.h | 2 +- src/Processors/QueryPlan/LimitStep.cpp | 2 +- src/Processors/QueryPlan/LimitStep.h | 2 +- src/Processors/QueryPlan/MergeSortingStep.cpp | 2 +- src/Processors/QueryPlan/MergeSortingStep.h | 2 +- .../QueryPlan/MergingAggregatedStep.cpp | 2 +- .../QueryPlan/MergingAggregatedStep.h | 2 +- src/Processors/QueryPlan/MergingFinal.cpp | 2 +- src/Processors/QueryPlan/MergingFinal.h | 2 +- .../QueryPlan/MergingSortedStep.cpp | 2 +- src/Processors/QueryPlan/MergingSortedStep.h | 2 +- src/Processors/QueryPlan/OffsetStep.cpp | 2 +- src/Processors/QueryPlan/OffsetStep.h | 2 +- .../QueryPlan/PartialSortingStep.cpp | 2 +- src/Processors/QueryPlan/PartialSortingStep.h | 2 +- src/Processors/QueryPlan/QueryPlan.cpp | 23 +++++-- src/Processors/QueryPlan/QueryPlan.h | 6 ++ src/Processors/QueryPlan/TotalsHavingStep.cpp | 2 +- src/Processors/QueryPlan/TotalsHavingStep.h | 2 +- src/Processors/QueryPlan/UnionStep.cpp | 2 +- src/Processors/QueryPlan/UnionStep.h | 2 +- src/Processors/QueryPlan/WindowStep.cpp | 2 +- src/Processors/QueryPlan/WindowStep.h | 2 +- src/Processors/Transforms/WindowTransform.cpp | 5 ++ .../01591_window_functions.reference | 38 +++++++---- .../0_stateless/01591_window_functions.sql | 15 ++++- 59 files changed, 254 insertions(+), 92 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 2fc78261f17..f9505cc4199 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -691,6 +691,9 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions( ActionsDAGPtr ActionsDAG::merge(ActionsDAG && first, ActionsDAG && second) { + fmt::print(stderr, "actiondag merge first '{}'\n, second '{}\n", + first.dumpDAG(), second.dumpDAG()); + /// first: x (1), x (2), y ==> x (2), z, x (3) /// second: x (1), x (2), x (3) ==> x (3), x (2), x (1) /// merge: x (1), x (2), x (3), y =(first)=> x (3), y, x (2), z, x (4) =(second)=> y, z, x (4), x (2), x (3) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 31c12490408..e271f78b699 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1004,15 +1004,17 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments( col.column = col.type->createColumn(); col.name = f.column_name; - step.actions()->addInput(col); +// step.actions()->addInput(col); + columns_after_window.push_back({f.column_name, + f.aggregate_function->getReturnType()}); for (const auto & a : f.function_node->arguments->children) { // 2.1) function arguments; step.required_output.push_back(a->getColumnName()); } - // 2.2) function result; - step.required_output.push_back(f.column_name); +// // 2.2) function result; +// step.required_output.push_back(f.column_name); } // 2.3) PARTITION BY and ORDER BY columns. @@ -1030,6 +1032,8 @@ bool SelectQueryExpressionAnalyzer::appendHaving(ExpressionActionsChain & chain, if (!select_query->having()) return false; + fmt::print(stderr, "appendHaving:\n'{}'\n", select_query->dumpTree()); + ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns); getRootActionsForHaving(select_query->having(), only_types, step.actions()); @@ -1048,6 +1052,15 @@ void SelectQueryExpressionAnalyzer::appendSelect(ExpressionActionsChain & chain, for (const auto & child : select_query->select()->children) { + if (const auto * function = typeid_cast(child.get()); + function + && function->is_window_function) + { + // Skip window function columns here -- they are calculated after + // other SELECT expressions by a special step. + continue; + } + step.required_output.push_back(child->getColumnName()); } } @@ -1421,9 +1434,6 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( /// If there is aggregation, we execute expressions in SELECT and ORDER BY on the initiating server, otherwise on the source servers. query_analyzer.appendSelect(chain, only_types || (need_aggregate ? !second_stage : !first_stage)); - query_analyzer.appendWindowFunctionsArguments(chain, only_types || !first_stage); - - selected_columns = chain.getLastStep().required_output; has_order_by = query.orderBy() != nullptr; before_order_and_select = query_analyzer.appendOrderBy( chain, @@ -1431,6 +1441,44 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( optimize_read_in_order, order_by_elements_actions); + if (has_window) + { + fmt::print(stderr, "chain before window args: '{}'\n", chain.dumpChain()); + + query_analyzer.appendWindowFunctionsArguments(chain, only_types || !first_stage); + + fmt::print(stderr, "after window args chain: '{}'\n", chain.dumpChain()); + + auto select_required_columns = chain.getLastStep().required_output; + + for (const auto & x : chain.getLastActions()->getNamesAndTypesList()) + { + query_analyzer.columns_after_window.push_back(x); + } + + finalize_chain(chain); + //auto & window_step = chain.addStep(); + //window_step.actions_dag.query_analyzer.columns_after_window); + + auto & step = chain.lastStep(query_analyzer.columns_after_window); + step.required_output = select_required_columns; + const auto * select_query = query_analyzer.getSelectQuery(); + for (const auto & child : select_query->select()->children) + { + if (const auto * function + = typeid_cast(child.get()); + function && function->is_window_function) + { + // See its counterpart in appendSelect() + step.required_output.push_back(child->getColumnName()); + } + } + + fmt::print(stderr, "chain after window columns: '{}'\n", chain.dumpChain()); + } + + selected_columns = chain.getLastStep().required_output; + if (query_analyzer.appendLimitBy(chain, only_types || !second_stage)) { before_limit_by = chain.getLastActions(); @@ -1440,8 +1488,11 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( final_projection = query_analyzer.appendProjectResult(chain); finalize_chain(chain); + + fmt::print(stderr, "final chain: '{}'\n", chain.dumpChain()); } + /// Before executing WHERE and HAVING, remove the extra columns from the block (mostly the aggregation keys). removeExtraColumns(); diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index fb0cb4ea4c3..1095689a775 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -55,6 +55,8 @@ struct ExpressionAnalyzerData NamesAndTypesList columns_after_join; /// Columns after ARRAY JOIN, JOIN, and/or aggregation. NamesAndTypesList aggregated_columns; + /// Columns after window functions. + NamesAndTypesList columns_after_window; bool has_aggregation = false; NamesAndTypesList aggregation_keys; diff --git a/src/Interpreters/ExtractExpressionInfoVisitor.cpp b/src/Interpreters/ExtractExpressionInfoVisitor.cpp index 2d9339447b1..64c23cd4fd1 100644 --- a/src/Interpreters/ExtractExpressionInfoVisitor.cpp +++ b/src/Interpreters/ExtractExpressionInfoVisitor.cpp @@ -22,15 +22,22 @@ void ExpressionInfoMatcher::visit(const ASTFunction & ast_function, const ASTPtr { data.is_array_join = true; } - // "is_aggregate_function" doesn't mean much by itself. Apparently here it is - // used to move filters from HAVING to WHERE, and probably for this purpose - // an aggregate function calculated as a window function is not relevant. + // "is_aggregate_function" is used to determine whether we can move a filter + // (1) from HAVING to WHERE or (2) from WHERE of a parent query to HAVING of + // a subquery. + // For aggregate functions we can't do (1) but can do (2). + // For window functions both don't make sense -- they are not allowed in + // WHERE or HAVING. else if (!ast_function.is_window_function && AggregateFunctionFactory::instance().isAggregateFunctionName( ast_function.name)) { data.is_aggregate_function = true; } + else if (ast_function.is_window_function) + { + data.is_window_function = true; + } else { const auto & function = FunctionFactory::instance().tryGet(ast_function.name, data.context); @@ -75,15 +82,26 @@ bool ExpressionInfoMatcher::needChildVisit(const ASTPtr & node, const ASTPtr &) return !node->as(); } -bool hasStatefulFunction(const ASTPtr & node, const Context & context) +bool hasNonRewritableFunction(const ASTPtr & node, const Context & context) { for (const auto & select_expression : node->children) { ExpressionInfoVisitor::Data expression_info{.context = context, .tables = {}}; ExpressionInfoVisitor(expression_info).visit(select_expression); - if (expression_info.is_stateful_function) + if (expression_info.is_stateful_function + || expression_info.is_window_function) + { + // If an outer query has a WHERE on window function, we can't move + // it into the subquery, because window functions are not allowed in + // WHERE and HAVING. Example: + // select * from ( + // select number, + // count(*) over (partition by intDiv(number, 3)) c + // from numbers(3) + // ) where c > 1; return true; + } } return false; diff --git a/src/Interpreters/ExtractExpressionInfoVisitor.h b/src/Interpreters/ExtractExpressionInfoVisitor.h index 0cb43e5b00a..d05415490e6 100644 --- a/src/Interpreters/ExtractExpressionInfoVisitor.h +++ b/src/Interpreters/ExtractExpressionInfoVisitor.h @@ -21,6 +21,7 @@ struct ExpressionInfoMatcher bool is_array_join = false; bool is_stateful_function = false; bool is_aggregate_function = false; + bool is_window_function = false; bool is_deterministic_function = true; std::unordered_set unique_reference_tables_pos = {}; }; @@ -36,6 +37,6 @@ struct ExpressionInfoMatcher using ExpressionInfoVisitor = ConstInDepthNodeVisitor; -bool hasStatefulFunction(const ASTPtr & node, const Context & context); +bool hasNonRewritableFunction(const ASTPtr & node, const Context & context); } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 38cc19a00d6..9a6e6b7593b 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -521,6 +521,9 @@ Block InterpreterSelectQuery::getSampleBlockImpl() filter_info, source_header); + fmt::print(stderr, "ExpressionAnalysisResult: '{}'\nat\n{}\n", + analysis_result.dump(), StackTrace().toString()); + if (options.to_stage == QueryProcessingStage::Enum::FetchColumns) { auto header = source_header; diff --git a/src/Interpreters/PredicateExpressionsOptimizer.cpp b/src/Interpreters/PredicateExpressionsOptimizer.cpp index 614693b5ae0..00b47be408a 100644 --- a/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -90,8 +90,12 @@ std::vector PredicateExpressionsOptimizer::extractTablesPredicates(const A ExpressionInfoVisitor::Data expression_info{.context = context, .tables = tables_with_columns}; ExpressionInfoVisitor(expression_info).visit(predicate_expression); - if (expression_info.is_stateful_function || !expression_info.is_deterministic_function) - return {}; /// Not optimized when predicate contains stateful function or indeterministic function + if (expression_info.is_stateful_function + || !expression_info.is_deterministic_function + || expression_info.is_window_function) + { + return {}; /// Not optimized when predicate contains stateful function or indeterministic function or window functions + } if (!expression_info.is_array_join) { @@ -190,6 +194,12 @@ bool PredicateExpressionsOptimizer::tryMovePredicatesFromHavingToWhere(ASTSelect if (expression_info.is_stateful_function) return false; + if (expression_info.is_window_function) + { + // Window functions are not allowed in either HAVING or WHERE. + return false; + } + if (expression_info.is_aggregate_function) having_predicates.emplace_back(moving_predicate); else diff --git a/src/Interpreters/PredicateRewriteVisitor.cpp b/src/Interpreters/PredicateRewriteVisitor.cpp index 9795675bcc8..5773629d0d1 100644 --- a/src/Interpreters/PredicateRewriteVisitor.cpp +++ b/src/Interpreters/PredicateRewriteVisitor.cpp @@ -88,7 +88,7 @@ bool PredicateRewriteVisitorData::rewriteSubquery(ASTSelectQuery & subquery, con || (!optimize_with && subquery.with()) || subquery.withFill() || subquery.limitBy() || subquery.limitLength() - || hasStatefulFunction(subquery.select(), context)) + || hasNonRewritableFunction(subquery.select(), context)) return false; for (const auto & predicate : predicates) diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index cee19c632fa..c02f68496e2 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -594,9 +594,21 @@ void TreeOptimizer::apply(ASTPtr & query, Aliases & aliases, const NameSet & sou if (settings.optimize_arithmetic_operations_in_aggregate_functions) optimizeAggregationFunctions(query); + const std::string before = select_query->dumpTree(); + fmt::print(stderr, "before predicate pushdown:\n'{}'\n", + select_query->dumpTree()); /// Push the predicate expression down to the subqueries. rewrite_subqueries = PredicateExpressionsOptimizer(context, tables_with_columns, settings).optimize(*select_query); + fmt::print(stderr, "after predicate pushdown:\n'{}'\n", + select_query->dumpTree()); + + if (before != select_query->dumpTree()) + { + fmt::print(stderr, "predicate pushdown made changes at\n'{}'\n", + StackTrace().toString()); + } + /// GROUP BY injective function elimination. optimizeGroupBy(select_query, source_columns_set, context); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 0dee5a8648d..9dfd15a0e77 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -443,6 +443,8 @@ std::vector getAggregates(ASTPtr & query, const ASTSelectQu std::vector getWindowFunctions(ASTPtr & query, const ASTSelectQuery & select_query) { + fmt::print(stderr, "AST at getWindowFunctions:\n'{}'\n", query->dumpTree()); + /// There can not be window functions inside the WHERE and PREWHERE. if (select_query.where()) assertNoWindows(select_query.where(), "in WHERE"); diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index e8d4a262366..8b787859467 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -156,12 +156,12 @@ void AggregatingStep::transformPipeline(QueryPipeline & pipeline) } } -void AggregatingStep::describeActions(FormatSettings & settings) const +void AggregatingStep::describeActions(QueryPlanStepFormatSettings & settings) const { params.explain(settings.out, settings.offset); } -void AggregatingStep::describePipeline(FormatSettings & settings) const +void AggregatingStep::describePipeline(QueryPlanStepFormatSettings & settings) const { if (!aggregating.empty()) IQueryPlanStep::describePipeline(aggregating, settings); diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index 853173895b3..3d12e79a662 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -29,8 +29,8 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings &) const override; - void describePipeline(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings &) const override; + void describePipeline(QueryPlanStepFormatSettings & settings) const override; private: Aggregator::Params params; diff --git a/src/Processors/QueryPlan/ArrayJoinStep.cpp b/src/Processors/QueryPlan/ArrayJoinStep.cpp index b71d2176e52..01cfd7c561a 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.cpp +++ b/src/Processors/QueryPlan/ArrayJoinStep.cpp @@ -69,7 +69,7 @@ void ArrayJoinStep::transformPipeline(QueryPipeline & pipeline) } } -void ArrayJoinStep::describeActions(FormatSettings & settings) const +void ArrayJoinStep::describeActions(QueryPlanStepFormatSettings & settings) const { String prefix(settings.offset, ' '); bool first = true; diff --git a/src/Processors/QueryPlan/ArrayJoinStep.h b/src/Processors/QueryPlan/ArrayJoinStep.h index 92c7e0a1304..338ec7f0583 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.h +++ b/src/Processors/QueryPlan/ArrayJoinStep.h @@ -15,7 +15,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; void updateInputStream(DataStream input_stream, Block result_header); diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 5868a7045f7..c6b77bc46c3 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -47,7 +47,7 @@ void CreatingSetStep::transformPipeline(QueryPipeline & pipeline) pipeline.addCreatingSetsTransform(getOutputStream().header, std::move(subquery_for_set), network_transfer_limits, context); } -void CreatingSetStep::describeActions(FormatSettings & settings) const +void CreatingSetStep::describeActions(QueryPlanStepFormatSettings & settings) const { String prefix(settings.offset, ' '); @@ -102,7 +102,7 @@ QueryPipelinePtr CreatingSetsStep::updatePipeline(QueryPipelines pipelines) return main_pipeline; } -void CreatingSetsStep::describePipeline(FormatSettings & settings) const +void CreatingSetsStep::describePipeline(QueryPlanStepFormatSettings & settings) const { IQueryPlanStep::describePipeline(processors, settings); } diff --git a/src/Processors/QueryPlan/CreatingSetsStep.h b/src/Processors/QueryPlan/CreatingSetsStep.h index ec13ab2052e..f610eeb64d9 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.h +++ b/src/Processors/QueryPlan/CreatingSetsStep.h @@ -22,7 +22,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; private: String description; @@ -40,7 +40,7 @@ public: QueryPipelinePtr updatePipeline(QueryPipelines pipelines) override; - void describePipeline(FormatSettings & settings) const override; + void describePipeline(QueryPlanStepFormatSettings & settings) const override; private: Processors processors; diff --git a/src/Processors/QueryPlan/DistinctStep.cpp b/src/Processors/QueryPlan/DistinctStep.cpp index 60966b08beb..ed2bae002ea 100644 --- a/src/Processors/QueryPlan/DistinctStep.cpp +++ b/src/Processors/QueryPlan/DistinctStep.cpp @@ -79,7 +79,7 @@ void DistinctStep::transformPipeline(QueryPipeline & pipeline) }); } -void DistinctStep::describeActions(FormatSettings & settings) const +void DistinctStep::describeActions(QueryPlanStepFormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Columns: "; diff --git a/src/Processors/QueryPlan/DistinctStep.h b/src/Processors/QueryPlan/DistinctStep.h index 4bfd73ce044..901e199ca39 100644 --- a/src/Processors/QueryPlan/DistinctStep.h +++ b/src/Processors/QueryPlan/DistinctStep.h @@ -20,7 +20,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; private: SizeLimits set_size_limits; diff --git a/src/Processors/QueryPlan/ExpressionStep.cpp b/src/Processors/QueryPlan/ExpressionStep.cpp index f7ca0a972dc..028c9502c71 100644 --- a/src/Processors/QueryPlan/ExpressionStep.cpp +++ b/src/Processors/QueryPlan/ExpressionStep.cpp @@ -87,7 +87,7 @@ void ExpressionStep::transformPipeline(QueryPipeline & pipeline) } } -void ExpressionStep::describeActions(FormatSettings & settings) const +void ExpressionStep::describeActions(QueryPlanStepFormatSettings & settings) const { String prefix(settings.offset, ' '); bool first = true; diff --git a/src/Processors/QueryPlan/ExpressionStep.h b/src/Processors/QueryPlan/ExpressionStep.h index 60f186688b0..2519765dbae 100644 --- a/src/Processors/QueryPlan/ExpressionStep.h +++ b/src/Processors/QueryPlan/ExpressionStep.h @@ -26,7 +26,7 @@ public: void updateInputStream(DataStream input_stream, bool keep_header); - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; const ActionsDAGPtr & getExpression() const { return actions_dag; } diff --git a/src/Processors/QueryPlan/FillingStep.cpp b/src/Processors/QueryPlan/FillingStep.cpp index 1a8fba97ee2..18c71b89601 100644 --- a/src/Processors/QueryPlan/FillingStep.cpp +++ b/src/Processors/QueryPlan/FillingStep.cpp @@ -43,7 +43,7 @@ void FillingStep::transformPipeline(QueryPipeline & pipeline) }); } -void FillingStep::describeActions(FormatSettings & settings) const +void FillingStep::describeActions(QueryPlanStepFormatSettings & settings) const { settings.out << String(settings.offset, ' '); dumpSortDescription(sort_description, input_streams.front().header, settings.out); diff --git a/src/Processors/QueryPlan/FillingStep.h b/src/Processors/QueryPlan/FillingStep.h index 85736464a6c..a7836e4de68 100644 --- a/src/Processors/QueryPlan/FillingStep.h +++ b/src/Processors/QueryPlan/FillingStep.h @@ -15,7 +15,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; private: SortDescription sort_description; diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp index 8fb405e685b..98f111a1529 100644 --- a/src/Processors/QueryPlan/FilterStep.cpp +++ b/src/Processors/QueryPlan/FilterStep.cpp @@ -80,7 +80,7 @@ void FilterStep::transformPipeline(QueryPipeline & pipeline) } } -void FilterStep::describeActions(FormatSettings & settings) const +void FilterStep::describeActions(QueryPlanStepFormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Filter column: " << filter_column_name << '\n'; diff --git a/src/Processors/QueryPlan/FilterStep.h b/src/Processors/QueryPlan/FilterStep.h index 72b02624faa..561fbf347da 100644 --- a/src/Processors/QueryPlan/FilterStep.h +++ b/src/Processors/QueryPlan/FilterStep.h @@ -22,7 +22,7 @@ public: void updateInputStream(DataStream input_stream, bool keep_header); - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; const ActionsDAGPtr & getExpression() const { return actions_dag; } const String & getFilterColumnName() const { return filter_column_name; } diff --git a/src/Processors/QueryPlan/FinishSortingStep.cpp b/src/Processors/QueryPlan/FinishSortingStep.cpp index d883bd0e0dd..7af8fad2980 100644 --- a/src/Processors/QueryPlan/FinishSortingStep.cpp +++ b/src/Processors/QueryPlan/FinishSortingStep.cpp @@ -85,7 +85,7 @@ void FinishSortingStep::transformPipeline(QueryPipeline & pipeline) } } -void FinishSortingStep::describeActions(FormatSettings & settings) const +void FinishSortingStep::describeActions(QueryPlanStepFormatSettings & settings) const { String prefix(settings.offset, ' '); diff --git a/src/Processors/QueryPlan/FinishSortingStep.h b/src/Processors/QueryPlan/FinishSortingStep.h index 4bb62037faa..25dc0b78bda 100644 --- a/src/Processors/QueryPlan/FinishSortingStep.h +++ b/src/Processors/QueryPlan/FinishSortingStep.h @@ -20,7 +20,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; /// Add limit or change it to lower value. void updateLimit(size_t limit_); diff --git a/src/Processors/QueryPlan/IQueryPlanStep.cpp b/src/Processors/QueryPlan/IQueryPlanStep.cpp index 71c4caaa795..a864043ddcb 100644 --- a/src/Processors/QueryPlan/IQueryPlanStep.cpp +++ b/src/Processors/QueryPlan/IQueryPlanStep.cpp @@ -18,7 +18,8 @@ const DataStream & IQueryPlanStep::getOutputStream() const return *output_stream; } -static void doDescribeHeader(const Block & header, size_t count, IQueryPlanStep::FormatSettings & settings) +static void doDescribeHeader(const Block & header, size_t count, + QueryPlanStepFormatSettings & settings) { String prefix(settings.offset, settings.indent_char); prefix += "Header"; @@ -46,11 +47,14 @@ static void doDescribeHeader(const Block & header, size_t count, IQueryPlanStep: first = false; elem.dumpNameAndType(settings.out); + settings.out << ": "; + elem.dumpStructure(settings.out); settings.out << '\n'; } } -static void doDescribeProcessor(const IProcessor & processor, size_t count, IQueryPlanStep::FormatSettings & settings) +static void doDescribeProcessor(const IProcessor & processor, size_t count, + QueryPlanStepFormatSettings & settings) { settings.out << String(settings.offset, settings.indent_char) << processor.getName(); if (count > 1) @@ -87,7 +91,7 @@ static void doDescribeProcessor(const IProcessor & processor, size_t count, IQue settings.offset += settings.indent; } -void IQueryPlanStep::describePipeline(const Processors & processors, FormatSettings & settings) +void IQueryPlanStep::describePipeline(const Processors & processors, QueryPlanStepFormatSettings & settings) { const IProcessor * prev = nullptr; size_t count = 0; diff --git a/src/Processors/QueryPlan/IQueryPlanStep.h b/src/Processors/QueryPlan/IQueryPlanStep.h index dfdbc9df442..1de1bbc438f 100644 --- a/src/Processors/QueryPlan/IQueryPlanStep.h +++ b/src/Processors/QueryPlan/IQueryPlanStep.h @@ -61,6 +61,16 @@ public: using DataStreams = std::vector; +// Not a nested class so that we can forward declare it. +struct QueryPlanStepFormatSettings +{ + WriteBuffer & out; + size_t offset = 0; + const size_t indent = 2; + const char indent_char = ' '; + const bool write_header = false; +}; + /// Single step of query plan. class IQueryPlanStep { @@ -86,20 +96,11 @@ public: const std::string & getStepDescription() const { return step_description; } void setStepDescription(std::string description) { step_description = std::move(description); } - struct FormatSettings - { - WriteBuffer & out; - size_t offset = 0; - const size_t indent = 2; - const char indent_char = ' '; - const bool write_header = false; - }; - /// Get detailed description of step actions. This is shown in EXPLAIN query with options `actions = 1`. - virtual void describeActions(FormatSettings & /*settings*/) const {} + virtual void describeActions(QueryPlanStepFormatSettings & /*settings*/) const {} /// Get description of processors added in current step. Should be called after updatePipeline(). - virtual void describePipeline(FormatSettings & /*settings*/) const {} + virtual void describePipeline(QueryPlanStepFormatSettings & /*settings*/) const {} protected: DataStreams input_streams; @@ -108,7 +109,7 @@ protected: /// Text description about what current step does. std::string step_description; - static void describePipeline(const Processors & processors, FormatSettings & settings); + static void describePipeline(const Processors & processors, QueryPlanStepFormatSettings & settings); }; using QueryPlanStepPtr = std::unique_ptr; diff --git a/src/Processors/QueryPlan/ISourceStep.cpp b/src/Processors/QueryPlan/ISourceStep.cpp index 1796f033896..367be8e8ea5 100644 --- a/src/Processors/QueryPlan/ISourceStep.cpp +++ b/src/Processors/QueryPlan/ISourceStep.cpp @@ -19,7 +19,7 @@ QueryPipelinePtr ISourceStep::updatePipeline(QueryPipelines) return pipeline; } -void ISourceStep::describePipeline(FormatSettings & settings) const +void ISourceStep::describePipeline(QueryPlanStepFormatSettings & settings) const { IQueryPlanStep::describePipeline(processors, settings); } diff --git a/src/Processors/QueryPlan/ISourceStep.h b/src/Processors/QueryPlan/ISourceStep.h index fdb3dd566cb..050e39ac11b 100644 --- a/src/Processors/QueryPlan/ISourceStep.h +++ b/src/Processors/QueryPlan/ISourceStep.h @@ -14,7 +14,7 @@ public: virtual void initializePipeline(QueryPipeline & pipeline) = 0; - void describePipeline(FormatSettings & settings) const override; + void describePipeline(QueryPlanStepFormatSettings & settings) const override; protected: /// We collect processors got after pipeline transformation. diff --git a/src/Processors/QueryPlan/ITransformingStep.cpp b/src/Processors/QueryPlan/ITransformingStep.cpp index cb27bf38278..5430bafa27a 100644 --- a/src/Processors/QueryPlan/ITransformingStep.cpp +++ b/src/Processors/QueryPlan/ITransformingStep.cpp @@ -65,7 +65,7 @@ void ITransformingStep::updateDistinctColumns(const Block & res_header, NameSet } } -void ITransformingStep::describePipeline(FormatSettings & settings) const +void ITransformingStep::describePipeline(QueryPlanStepFormatSettings & settings) const { IQueryPlanStep::describePipeline(processors, settings); } diff --git a/src/Processors/QueryPlan/ITransformingStep.h b/src/Processors/QueryPlan/ITransformingStep.h index bd99478ec37..b1bfebd4571 100644 --- a/src/Processors/QueryPlan/ITransformingStep.h +++ b/src/Processors/QueryPlan/ITransformingStep.h @@ -55,7 +55,7 @@ public: const TransformTraits & getTransformTraits() const { return transform_traits; } const DataStreamTraits & getDataStreamTraits() const { return data_stream_traits; } - void describePipeline(FormatSettings & settings) const override; + void describePipeline(QueryPlanStepFormatSettings & settings) const override; protected: /// Clear distinct_columns if res_header doesn't contain all of them. diff --git a/src/Processors/QueryPlan/LimitByStep.cpp b/src/Processors/QueryPlan/LimitByStep.cpp index 9fcf5b60164..15c64fcea44 100644 --- a/src/Processors/QueryPlan/LimitByStep.cpp +++ b/src/Processors/QueryPlan/LimitByStep.cpp @@ -46,7 +46,7 @@ void LimitByStep::transformPipeline(QueryPipeline & pipeline) }); } -void LimitByStep::describeActions(FormatSettings & settings) const +void LimitByStep::describeActions(QueryPlanStepFormatSettings & settings) const { String prefix(settings.offset, ' '); diff --git a/src/Processors/QueryPlan/LimitByStep.h b/src/Processors/QueryPlan/LimitByStep.h index 9320735640c..01dcff5c912 100644 --- a/src/Processors/QueryPlan/LimitByStep.h +++ b/src/Processors/QueryPlan/LimitByStep.h @@ -16,7 +16,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; private: size_t group_length; diff --git a/src/Processors/QueryPlan/LimitStep.cpp b/src/Processors/QueryPlan/LimitStep.cpp index 565d05956e5..38ec9265d4e 100644 --- a/src/Processors/QueryPlan/LimitStep.cpp +++ b/src/Processors/QueryPlan/LimitStep.cpp @@ -50,7 +50,7 @@ void LimitStep::transformPipeline(QueryPipeline & pipeline) pipeline.addTransform(std::move(transform)); } -void LimitStep::describeActions(FormatSettings & settings) const +void LimitStep::describeActions(QueryPlanStepFormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Limit " << limit << '\n'; diff --git a/src/Processors/QueryPlan/LimitStep.h b/src/Processors/QueryPlan/LimitStep.h index eea186fea4a..a95bf28f896 100644 --- a/src/Processors/QueryPlan/LimitStep.h +++ b/src/Processors/QueryPlan/LimitStep.h @@ -20,7 +20,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; size_t getLimitForSorting() const { diff --git a/src/Processors/QueryPlan/MergeSortingStep.cpp b/src/Processors/QueryPlan/MergeSortingStep.cpp index b30286130b1..e2af023f598 100644 --- a/src/Processors/QueryPlan/MergeSortingStep.cpp +++ b/src/Processors/QueryPlan/MergeSortingStep.cpp @@ -73,7 +73,7 @@ void MergeSortingStep::transformPipeline(QueryPipeline & pipeline) }); } -void MergeSortingStep::describeActions(FormatSettings & settings) const +void MergeSortingStep::describeActions(QueryPlanStepFormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Sort description: "; diff --git a/src/Processors/QueryPlan/MergeSortingStep.h b/src/Processors/QueryPlan/MergeSortingStep.h index a385a8a3e93..c41c209dd55 100644 --- a/src/Processors/QueryPlan/MergeSortingStep.h +++ b/src/Processors/QueryPlan/MergeSortingStep.h @@ -26,7 +26,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; /// Add limit or change it to lower value. void updateLimit(size_t limit_); diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.cpp b/src/Processors/QueryPlan/MergingAggregatedStep.cpp index 85bbbeab59a..2fddc88bbc1 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.cpp +++ b/src/Processors/QueryPlan/MergingAggregatedStep.cpp @@ -63,7 +63,7 @@ void MergingAggregatedStep::transformPipeline(QueryPipeline & pipeline) } } -void MergingAggregatedStep::describeActions(FormatSettings & settings) const +void MergingAggregatedStep::describeActions(QueryPlanStepFormatSettings & settings) const { return params->params.explain(settings.out, settings.offset); } diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.h b/src/Processors/QueryPlan/MergingAggregatedStep.h index 5ffceb7f938..2d9d902a294 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.h +++ b/src/Processors/QueryPlan/MergingAggregatedStep.h @@ -23,7 +23,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; private: AggregatingTransformParamsPtr params; diff --git a/src/Processors/QueryPlan/MergingFinal.cpp b/src/Processors/QueryPlan/MergingFinal.cpp index c1dec231f11..46431e0c767 100644 --- a/src/Processors/QueryPlan/MergingFinal.cpp +++ b/src/Processors/QueryPlan/MergingFinal.cpp @@ -153,7 +153,7 @@ void MergingFinal::transformPipeline(QueryPipeline & pipeline) }); } -void MergingFinal::describeActions(FormatSettings & settings) const +void MergingFinal::describeActions(QueryPlanStepFormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Sort description: "; diff --git a/src/Processors/QueryPlan/MergingFinal.h b/src/Processors/QueryPlan/MergingFinal.h index c01f5c7f9a1..207c191c9cb 100644 --- a/src/Processors/QueryPlan/MergingFinal.h +++ b/src/Processors/QueryPlan/MergingFinal.h @@ -22,7 +22,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; private: size_t num_output_streams; diff --git a/src/Processors/QueryPlan/MergingSortedStep.cpp b/src/Processors/QueryPlan/MergingSortedStep.cpp index c59540b009f..a1249ef412f 100644 --- a/src/Processors/QueryPlan/MergingSortedStep.cpp +++ b/src/Processors/QueryPlan/MergingSortedStep.cpp @@ -62,7 +62,7 @@ void MergingSortedStep::transformPipeline(QueryPipeline & pipeline) } } -void MergingSortedStep::describeActions(FormatSettings & settings) const +void MergingSortedStep::describeActions(QueryPlanStepFormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Sort description: "; diff --git a/src/Processors/QueryPlan/MergingSortedStep.h b/src/Processors/QueryPlan/MergingSortedStep.h index 483cfa5e8a7..0c91d505f3d 100644 --- a/src/Processors/QueryPlan/MergingSortedStep.h +++ b/src/Processors/QueryPlan/MergingSortedStep.h @@ -21,7 +21,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; /// Add limit or change it to lower value. void updateLimit(size_t limit_); diff --git a/src/Processors/QueryPlan/OffsetStep.cpp b/src/Processors/QueryPlan/OffsetStep.cpp index 7ac3d3f2110..1a06f2ba05e 100644 --- a/src/Processors/QueryPlan/OffsetStep.cpp +++ b/src/Processors/QueryPlan/OffsetStep.cpp @@ -36,7 +36,7 @@ void OffsetStep::transformPipeline(QueryPipeline & pipeline) pipeline.addTransform(std::move(transform)); } -void OffsetStep::describeActions(FormatSettings & settings) const +void OffsetStep::describeActions(QueryPlanStepFormatSettings & settings) const { settings.out << String(settings.offset, ' ') << "Offset " << offset << '\n'; } diff --git a/src/Processors/QueryPlan/OffsetStep.h b/src/Processors/QueryPlan/OffsetStep.h index 17949371edf..18ea58635af 100644 --- a/src/Processors/QueryPlan/OffsetStep.h +++ b/src/Processors/QueryPlan/OffsetStep.h @@ -15,7 +15,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; private: size_t offset; diff --git a/src/Processors/QueryPlan/PartialSortingStep.cpp b/src/Processors/QueryPlan/PartialSortingStep.cpp index ce34eca9112..072217b4d0d 100644 --- a/src/Processors/QueryPlan/PartialSortingStep.cpp +++ b/src/Processors/QueryPlan/PartialSortingStep.cpp @@ -70,7 +70,7 @@ void PartialSortingStep::transformPipeline(QueryPipeline & pipeline) }); } -void PartialSortingStep::describeActions(FormatSettings & settings) const +void PartialSortingStep::describeActions(QueryPlanStepFormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Sort description: "; diff --git a/src/Processors/QueryPlan/PartialSortingStep.h b/src/Processors/QueryPlan/PartialSortingStep.h index 172ef25c300..6d9dbe2b9d8 100644 --- a/src/Processors/QueryPlan/PartialSortingStep.h +++ b/src/Processors/QueryPlan/PartialSortingStep.h @@ -20,7 +20,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; /// Add limit or change it to lower value. void updateLimit(size_t limit_); diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index e88090b4819..ee7c5220b4c 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -204,7 +204,7 @@ void QueryPlan::addInterpreterContext(std::shared_ptr context) static void explainStep( const IQueryPlanStep & step, - IQueryPlanStep::FormatSettings & settings, + QueryPlanStepFormatSettings & settings, const QueryPlan::ExplainPlanOptions & options) { std::string prefix(settings.offset, ' '); @@ -247,11 +247,20 @@ static void explainStep( step.describeActions(settings); } +std::string debugExplainStep(const IQueryPlanStep & step) +{ + WriteBufferFromOwnString out; + QueryPlanStepFormatSettings settings{.out = out}; + QueryPlan::ExplainPlanOptions options{.actions = true}; + explainStep(step, settings, options); + return out.str(); +} + void QueryPlan::explainPlan(WriteBuffer & buffer, const ExplainPlanOptions & options) { checkInitialized(); - IQueryPlanStep::FormatSettings settings{.out = buffer, .write_header = options.header}; + QueryPlanStepFormatSettings settings{.out = buffer, .write_header = options.header}; struct Frame { @@ -284,7 +293,7 @@ void QueryPlan::explainPlan(WriteBuffer & buffer, const ExplainPlanOptions & opt } } -static void explainPipelineStep(IQueryPlanStep & step, IQueryPlanStep::FormatSettings & settings) +static void explainPipelineStep(IQueryPlanStep & step, QueryPlanStepFormatSettings & settings) { settings.out << String(settings.offset, settings.indent_char) << "(" << step.getName() << ")\n"; size_t current_offset = settings.offset; @@ -297,7 +306,7 @@ void QueryPlan::explainPipeline(WriteBuffer & buffer, const ExplainPipelineOptio { checkInitialized(); - IQueryPlanStep::FormatSettings settings{.out = buffer, .write_header = options.header}; + QueryPlanStepFormatSettings settings{.out = buffer, .write_header = options.header}; struct Frame { @@ -488,6 +497,12 @@ static bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Node * { auto & parent = parent_node->step; auto & child = child_node->step; + + fmt::print(stderr, + "try merge expressions: parent '{}'\n, child '{}'\n at {}\n", + debugExplainStep(*parent), debugExplainStep(*child), + StackTrace().toString()); + /// TODO: FilterStep auto * parent_expr = typeid_cast(parent.get()); auto * child_expr = typeid_cast(child.get()); diff --git a/src/Processors/QueryPlan/QueryPlan.h b/src/Processors/QueryPlan/QueryPlan.h index cbe487312cc..5ae991a328f 100644 --- a/src/Processors/QueryPlan/QueryPlan.h +++ b/src/Processors/QueryPlan/QueryPlan.h @@ -13,6 +13,7 @@ class DataStream; class IQueryPlanStep; using QueryPlanStepPtr = std::unique_ptr; +struct QueryPlanStepFormatSettings; class QueryPipeline; using QueryPipelinePtr = std::unique_ptr; @@ -97,4 +98,9 @@ private: std::vector> interpreter_context; }; +void debugExplainStep( + const IQueryPlanStep & step, + QueryPlanStepFormatSettings & settings, + const QueryPlan::ExplainPlanOptions & options); + } diff --git a/src/Processors/QueryPlan/TotalsHavingStep.cpp b/src/Processors/QueryPlan/TotalsHavingStep.cpp index 9947875e679..310f657b39d 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.cpp +++ b/src/Processors/QueryPlan/TotalsHavingStep.cpp @@ -74,7 +74,7 @@ static String totalsModeToString(TotalsMode totals_mode, double auto_include_thr __builtin_unreachable(); } -void TotalsHavingStep::describeActions(FormatSettings & settings) const +void TotalsHavingStep::describeActions(QueryPlanStepFormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Filter column: " << filter_column_name << '\n'; diff --git a/src/Processors/QueryPlan/TotalsHavingStep.h b/src/Processors/QueryPlan/TotalsHavingStep.h index 7c1638013e5..88fd9bbb597 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.h +++ b/src/Processors/QueryPlan/TotalsHavingStep.h @@ -26,7 +26,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; private: bool overflow_row; diff --git a/src/Processors/QueryPlan/UnionStep.cpp b/src/Processors/QueryPlan/UnionStep.cpp index 630ff53f47d..413b52c69c2 100644 --- a/src/Processors/QueryPlan/UnionStep.cpp +++ b/src/Processors/QueryPlan/UnionStep.cpp @@ -36,7 +36,7 @@ QueryPipelinePtr UnionStep::updatePipeline(QueryPipelines pipelines) return pipeline; } -void UnionStep::describePipeline(FormatSettings & settings) const +void UnionStep::describePipeline(QueryPlanStepFormatSettings & settings) const { IQueryPlanStep::describePipeline(processors, settings); } diff --git a/src/Processors/QueryPlan/UnionStep.h b/src/Processors/QueryPlan/UnionStep.h index e2e1f2c9efa..aeac7fa54ff 100644 --- a/src/Processors/QueryPlan/UnionStep.h +++ b/src/Processors/QueryPlan/UnionStep.h @@ -15,7 +15,7 @@ public: QueryPipelinePtr updatePipeline(QueryPipelines pipelines) override; - void describePipeline(FormatSettings & settings) const override; + void describePipeline(QueryPlanStepFormatSettings & settings) const override; private: Block header; diff --git a/src/Processors/QueryPlan/WindowStep.cpp b/src/Processors/QueryPlan/WindowStep.cpp index 82c589b8b20..d2d24985937 100644 --- a/src/Processors/QueryPlan/WindowStep.cpp +++ b/src/Processors/QueryPlan/WindowStep.cpp @@ -71,7 +71,7 @@ void WindowStep::transformPipeline(QueryPipeline & pipeline) "WindowStep transform for '" + window_description.window_name + "'"); } -void WindowStep::describeActions(FormatSettings & settings) const +void WindowStep::describeActions(QueryPlanStepFormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Window: ("; diff --git a/src/Processors/QueryPlan/WindowStep.h b/src/Processors/QueryPlan/WindowStep.h index 069d02c655c..25194820a0d 100644 --- a/src/Processors/QueryPlan/WindowStep.h +++ b/src/Processors/QueryPlan/WindowStep.h @@ -22,7 +22,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(FormatSettings & settings) const override; + void describeActions(QueryPlanStepFormatSettings & settings) const override; private: WindowDescription window_description; diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp index 6e8b0ea8e39..b200e306213 100644 --- a/src/Processors/Transforms/WindowTransform.cpp +++ b/src/Processors/Transforms/WindowTransform.cpp @@ -77,6 +77,11 @@ void WindowTransform::transform(Chunk & chunk) ws.argument_columns.clear(); for (const auto column_index : ws.argument_column_indices) { + // Aggregate functions can't work with constant columns, so we have to + // materialize them like the Aggregator does. + columns[column_index] + = std::move(columns[column_index])->convertToFullColumnIfConst(); + ws.argument_columns.push_back(columns[column_index].get()); } diff --git a/tests/queries/0_stateless/01591_window_functions.reference b/tests/queries/0_stateless/01591_window_functions.reference index 6c78108d734..19509de8a41 100644 --- a/tests/queries/0_stateless/01591_window_functions.reference +++ b/tests/queries/0_stateless/01591_window_functions.reference @@ -80,8 +80,19 @@ select number, quantileExact(number) over (partition by intDiv(number, 3)) q fro select q * 10, quantileExact(number) over (partition by intDiv(number, 3)) q from numbers(10); -- { serverError 47 } -- should work in ORDER BY though +-- doesn't work now +-- select number, max(number) over (partition by intDiv(number, 3) order by number desc) m from numbers(10) order by m desc, number; -select number, max(number) over (partition by intDiv(number, 3) order by number desc) m from numbers(10) order by m desc, number; +-- at least it works in ORDER BY if you wrap it in a subquery + +select * from (select count(*) over () c from numbers(3)) order by c; + +-- must work in WHERE if you wrap it in a subquery + +1 +2 +3 +select * from (select count(*) over () c from numbers(3)) where c > 0; -- this one doesn't work yet -- looks like the column names clash, and the -- window count() is overwritten with aggregate count() @@ -91,16 +102,9 @@ select number, max(number) over (partition by intDiv(number, 3) order by number -- an explain test would also be helpful, but it's too immature now and I don't -- want to change reference all the time -9 9 -6 8 -7 8 -8 8 -3 5 -4 5 -5 5 -0 2 -1 2 -2 2 +1 +2 +3 select number, max(number) over (partition by intDiv(number, 3) order by number desc), count(number) over (partition by intDiv(number, 5) order by number) as m from numbers(31) order by number settings max_block_size = 2; -- two functions over the same window @@ -140,6 +144,8 @@ select number, max(number) over (partition by intDiv(number, 3) order by number 30 30 1 select number, max(number) over (partition by intDiv(number, 3) order by number desc), count(number) over (partition by intDiv(number, 3) order by number desc) as m from numbers(7) order by number settings max_block_size = 2; +-- check that we can work with constant columns + 0 2 3 1 2 2 2 2 1 @@ -147,3 +153,13 @@ select number, max(number) over (partition by intDiv(number, 3) order by number 4 5 2 5 5 1 6 6 1 +select median(x) over (partition by x) from (select 1 x); + +-- an empty window definition is valid as well + +1 +select groupArray(number) over () from numbers(3); + +[0] +[0,1] +[0,1,2] diff --git a/tests/queries/0_stateless/01591_window_functions.sql b/tests/queries/0_stateless/01591_window_functions.sql index a28d435d3f8..4bd9f1b7693 100644 --- a/tests/queries/0_stateless/01591_window_functions.sql +++ b/tests/queries/0_stateless/01591_window_functions.sql @@ -25,7 +25,14 @@ select number, quantileExact(number) over (partition by intDiv(number, 3)) q fro select q * 10, quantileExact(number) over (partition by intDiv(number, 3)) q from numbers(10); -- { serverError 47 } -- should work in ORDER BY though -select number, max(number) over (partition by intDiv(number, 3) order by number desc) m from numbers(10) order by m desc, number; +-- doesn't work now +-- select number, max(number) over (partition by intDiv(number, 3) order by number desc) m from numbers(10) order by m desc, number; + +-- at least it works in ORDER BY if you wrap it in a subquery +select * from (select count(*) over () c from numbers(3)) order by c; + +-- must work in WHERE if you wrap it in a subquery +select * from (select count(*) over () c from numbers(3)) where c > 0; -- this one doesn't work yet -- looks like the column names clash, and the -- window count() is overwritten with aggregate count() @@ -40,3 +47,9 @@ select number, max(number) over (partition by intDiv(number, 3) order by number -- an explain test would also be helpful, but it's too immature now and I don't -- want to change reference all the time select number, max(number) over (partition by intDiv(number, 3) order by number desc), count(number) over (partition by intDiv(number, 3) order by number desc) as m from numbers(7) order by number settings max_block_size = 2; + +-- check that we can work with constant columns +select median(x) over (partition by x) from (select 1 x); + +-- an empty window definition is valid as well +select groupArray(number) over () from numbers(3); From 23d77a1698e90c8cd5f3f873fd95c313f012d5c9 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 24 Dec 2020 09:59:38 +0300 Subject: [PATCH 110/284] remove debug output --- src/Interpreters/ActionsDAG.cpp | 3 --- src/Interpreters/ExpressionAnalyzer.cpp | 12 ------------ src/Interpreters/InterpreterSelectQuery.cpp | 3 --- src/Interpreters/TreeOptimizer.cpp | 12 ------------ src/Interpreters/TreeRewriter.cpp | 2 -- src/Processors/QueryPlan/QueryPlan.cpp | 5 ----- 6 files changed, 37 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index f9505cc4199..2fc78261f17 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -691,9 +691,6 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions( ActionsDAGPtr ActionsDAG::merge(ActionsDAG && first, ActionsDAG && second) { - fmt::print(stderr, "actiondag merge first '{}'\n, second '{}\n", - first.dumpDAG(), second.dumpDAG()); - /// first: x (1), x (2), y ==> x (2), z, x (3) /// second: x (1), x (2), x (3) ==> x (3), x (2), x (1) /// merge: x (1), x (2), x (3), y =(first)=> x (3), y, x (2), z, x (4) =(second)=> y, z, x (4), x (2), x (3) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index e271f78b699..37f2791d8bd 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1032,8 +1032,6 @@ bool SelectQueryExpressionAnalyzer::appendHaving(ExpressionActionsChain & chain, if (!select_query->having()) return false; - fmt::print(stderr, "appendHaving:\n'{}'\n", select_query->dumpTree()); - ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns); getRootActionsForHaving(select_query->having(), only_types, step.actions()); @@ -1443,12 +1441,8 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( if (has_window) { - fmt::print(stderr, "chain before window args: '{}'\n", chain.dumpChain()); - query_analyzer.appendWindowFunctionsArguments(chain, only_types || !first_stage); - fmt::print(stderr, "after window args chain: '{}'\n", chain.dumpChain()); - auto select_required_columns = chain.getLastStep().required_output; for (const auto & x : chain.getLastActions()->getNamesAndTypesList()) @@ -1457,8 +1451,6 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( } finalize_chain(chain); - //auto & window_step = chain.addStep(); - //window_step.actions_dag.query_analyzer.columns_after_window); auto & step = chain.lastStep(query_analyzer.columns_after_window); step.required_output = select_required_columns; @@ -1473,8 +1465,6 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( step.required_output.push_back(child->getColumnName()); } } - - fmt::print(stderr, "chain after window columns: '{}'\n", chain.dumpChain()); } selected_columns = chain.getLastStep().required_output; @@ -1488,8 +1478,6 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( final_projection = query_analyzer.appendProjectResult(chain); finalize_chain(chain); - - fmt::print(stderr, "final chain: '{}'\n", chain.dumpChain()); } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 9a6e6b7593b..38cc19a00d6 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -521,9 +521,6 @@ Block InterpreterSelectQuery::getSampleBlockImpl() filter_info, source_header); - fmt::print(stderr, "ExpressionAnalysisResult: '{}'\nat\n{}\n", - analysis_result.dump(), StackTrace().toString()); - if (options.to_stage == QueryProcessingStage::Enum::FetchColumns) { auto header = source_header; diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index c02f68496e2..cee19c632fa 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -594,21 +594,9 @@ void TreeOptimizer::apply(ASTPtr & query, Aliases & aliases, const NameSet & sou if (settings.optimize_arithmetic_operations_in_aggregate_functions) optimizeAggregationFunctions(query); - const std::string before = select_query->dumpTree(); - fmt::print(stderr, "before predicate pushdown:\n'{}'\n", - select_query->dumpTree()); /// Push the predicate expression down to the subqueries. rewrite_subqueries = PredicateExpressionsOptimizer(context, tables_with_columns, settings).optimize(*select_query); - fmt::print(stderr, "after predicate pushdown:\n'{}'\n", - select_query->dumpTree()); - - if (before != select_query->dumpTree()) - { - fmt::print(stderr, "predicate pushdown made changes at\n'{}'\n", - StackTrace().toString()); - } - /// GROUP BY injective function elimination. optimizeGroupBy(select_query, source_columns_set, context); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 9dfd15a0e77..0dee5a8648d 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -443,8 +443,6 @@ std::vector getAggregates(ASTPtr & query, const ASTSelectQu std::vector getWindowFunctions(ASTPtr & query, const ASTSelectQuery & select_query) { - fmt::print(stderr, "AST at getWindowFunctions:\n'{}'\n", query->dumpTree()); - /// There can not be window functions inside the WHERE and PREWHERE. if (select_query.where()) assertNoWindows(select_query.where(), "in WHERE"); diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index ee7c5220b4c..a2fc8c689ff 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -498,11 +498,6 @@ static bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Node * auto & parent = parent_node->step; auto & child = child_node->step; - fmt::print(stderr, - "try merge expressions: parent '{}'\n, child '{}'\n at {}\n", - debugExplainStep(*parent), debugExplainStep(*child), - StackTrace().toString()); - /// TODO: FilterStep auto * parent_expr = typeid_cast(parent.get()); auto * child_expr = typeid_cast(child.get()); From 912995cbaeedd487b72a7a4954e19637fc99dec1 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 24 Dec 2020 11:49:55 +0300 Subject: [PATCH 111/284] some provision for aggregate fns as window fn args (doesn't work yet) also a perf test w/LIMIT BY --- src/Interpreters/ActionsVisitor.cpp | 13 +++++++ src/Interpreters/GetAggregatesVisitor.h | 7 ++-- src/Interpreters/TreeRewriter.cpp | 28 +++++++++++++-- tests/performance/window_functions.xml | 36 +++++++++++++++++++ .../01591_window_functions.reference | 6 ++++ .../0_stateless/01591_window_functions.sql | 4 +++ 6 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 tests/performance/window_functions.xml diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index c7bbc019518..bb47129f58c 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -749,6 +749,19 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & visit(node.window_order_by->clone(), data); } + // Also manually add columns for arguments of the window function itself. + // ActionVisitor is written in such a way that this method must itself + // descend into all needed function children. Window functions can't have + // any special functions as argument, so the code below that handles + // special arguments is not needed. This is analogous to the + // appendWindowFunctionsArguments() in SelectQueryExpressionAnalyzer and + // partially duplicates its code. Probably we can remove most of the + // logic from that function, but I don't yet have it all figured out... + for (const auto & arg : node.arguments->children) + { + visit(arg, data); + } + // Don't need to do anything more for window functions here -- the // resulting column is added in ExpressionAnalyzer, similar to the // aggregate functions. diff --git a/src/Interpreters/GetAggregatesVisitor.h b/src/Interpreters/GetAggregatesVisitor.h index d416a5f240e..266187aaffb 100644 --- a/src/Interpreters/GetAggregatesVisitor.h +++ b/src/Interpreters/GetAggregatesVisitor.h @@ -33,11 +33,14 @@ public: return false; if (auto * func = node->as()) { - if (isAggregateFunction(*func) - || func->is_window_function) + if (isAggregateFunction(*func)) { return false; } + + // Window functions can contain aggregation results as arguments + // to the window functions, or columns of PARTITION BY or ORDER BY + // of the window. } return true; } diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 9816d1a3940..70d35fd40a8 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -29,6 +29,7 @@ #include #include +#include #include #include @@ -445,6 +446,8 @@ std::vector getAggregates(ASTPtr & query, const ASTSelectQu for (auto & arg : node->arguments->children) { assertNoAggregates(arg, "inside another aggregate function"); + // We also can't have window functions inside aggregate functions, + // because the window functions are calculated later. assertNoWindows(arg, "inside an aggregate function"); } } @@ -454,7 +457,9 @@ std::vector getAggregates(ASTPtr & query, const ASTSelectQu std::vector getWindowFunctions(ASTPtr & query, const ASTSelectQuery & select_query) { - /// There can not be window functions inside the WHERE and PREWHERE. + /// There can not be window functions inside the WHERE, PREWHERE and HAVING + if (select_query.having()) + assertNoWindows(select_query.having(), "in HAVING"); if (select_query.where()) assertNoWindows(select_query.where(), "in WHERE"); if (select_query.prewhere()) @@ -463,17 +468,34 @@ std::vector getWindowFunctions(ASTPtr & query, const ASTSel GetAggregatesVisitor::Data data; GetAggregatesVisitor(data).visit(query); - /// There can not be other window functions within the aggregate functions. + /// Window functions cannot be inside aggregates or other window functions. + /// Aggregate functions can be inside window functions because they are + /// calculated earlier. for (const ASTFunction * node : data.window_functions) { if (node->arguments) { for (auto & arg : node->arguments->children) { - assertNoAggregates(arg, "inside a window function"); assertNoWindows(arg, "inside another window function"); } } + + if (node->window_partition_by) + { + for (auto & arg : node->window_partition_by->children) + { + assertNoWindows(arg, "inside PARTITION BY of a window"); + } + } + + if (node->window_order_by) + { + for (auto & arg : node->window_order_by->children) + { + assertNoWindows(arg, "inside ORDER BY of a window"); + } + } } return data.window_functions; diff --git a/tests/performance/window_functions.xml b/tests/performance/window_functions.xml new file mode 100644 index 00000000000..e4578a575a0 --- /dev/null +++ b/tests/performance/window_functions.xml @@ -0,0 +1,36 @@ + + + hits_100m_single + + + + + + + + diff --git a/tests/queries/0_stateless/01591_window_functions.reference b/tests/queries/0_stateless/01591_window_functions.reference index 19509de8a41..704c5a98811 100644 --- a/tests/queries/0_stateless/01591_window_functions.reference +++ b/tests/queries/0_stateless/01591_window_functions.reference @@ -160,6 +160,12 @@ select median(x) over (partition by x) from (select 1 x); 1 select groupArray(number) over () from numbers(3); +-- This one tests we properly process the window function arguments. +-- Seen errors like 'column `1` not found' from count(1). + [0] [0,1] [0,1,2] +select count(1) over (), max(number + 1) over () from numbers(3); + +1 3 diff --git a/tests/queries/0_stateless/01591_window_functions.sql b/tests/queries/0_stateless/01591_window_functions.sql index 4bd9f1b7693..fd2831ea2ac 100644 --- a/tests/queries/0_stateless/01591_window_functions.sql +++ b/tests/queries/0_stateless/01591_window_functions.sql @@ -53,3 +53,7 @@ select median(x) over (partition by x) from (select 1 x); -- an empty window definition is valid as well select groupArray(number) over () from numbers(3); + +-- This one tests we properly process the window function arguments. +-- Seen errors like 'column `1` not found' from count(1). +select count(1) over (), max(number + 1) over () from numbers(3); From 34cd4fea0e6e94c1275c34b7797d78403b1fdcb7 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 24 Dec 2020 12:05:14 +0300 Subject: [PATCH 112/284] Fixing integrational test. --- tests/integration/README.md | 2 +- tests/integration/test_grpc_protocol/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index cdfb6b1a70a..346926677bd 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -12,7 +12,7 @@ You must install latest Docker from https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#set-up-the-repository Don't use Docker from your system repository. -* [pip](https://pypi.python.org/pypi/pip) and `libpq-dev`. To install: `sudo apt-get install python3-pip libpq-dev zlib1g-dev libcrypto++-dev libssl-dev` +* [pip](https://pypi.python.org/pypi/pip) and `libpq-dev`. To install: `sudo apt-get install python3-pip libpq-dev zlib1g-dev libcrypto++-dev libssl-dev libkrb5-dev` * [py.test](https://docs.pytest.org/) testing framework. To install: `sudo -H pip install pytest` * [docker-compose](https://docs.docker.com/compose/) and additional python libraries. To install: diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index 3a3dc464155..ae4f4473475 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -239,7 +239,7 @@ def test_progress(): , output: "6\\t0\\n7\\t0\\n" , stats { rows: 8 - blocks: 4 + blocks: 5 allocated_bytes: 324 applied_limit: true rows_before_limit: 8 From 2bb0f5772e7c6003eeee11de49244180c18e181f Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 24 Dec 2020 12:06:12 +0300 Subject: [PATCH 113/284] proper signature --- src/Processors/QueryPlan/QueryPlan.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Processors/QueryPlan/QueryPlan.h b/src/Processors/QueryPlan/QueryPlan.h index 5ae991a328f..4c55718a062 100644 --- a/src/Processors/QueryPlan/QueryPlan.h +++ b/src/Processors/QueryPlan/QueryPlan.h @@ -98,9 +98,6 @@ private: std::vector> interpreter_context; }; -void debugExplainStep( - const IQueryPlanStep & step, - QueryPlanStepFormatSettings & settings, - const QueryPlan::ExplainPlanOptions & options); +std::string debugExplainStep(const IQueryPlanStep & step); } From c6c2b3ba00bfbd85e4ac309c306b7619dc96f40a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 24 Dec 2020 12:09:35 +0300 Subject: [PATCH 114/284] Remove the forward declaration changes --- src/Processors/QueryPlan/AggregatingStep.cpp | 4 +-- src/Processors/QueryPlan/AggregatingStep.h | 4 +-- src/Processors/QueryPlan/ArrayJoinStep.cpp | 2 +- src/Processors/QueryPlan/ArrayJoinStep.h | 2 +- src/Processors/QueryPlan/CreatingSetsStep.cpp | 4 +-- src/Processors/QueryPlan/CreatingSetsStep.h | 4 +-- src/Processors/QueryPlan/DistinctStep.cpp | 2 +- src/Processors/QueryPlan/DistinctStep.h | 2 +- src/Processors/QueryPlan/ExpressionStep.cpp | 2 +- src/Processors/QueryPlan/ExpressionStep.h | 2 +- src/Processors/QueryPlan/FillingStep.cpp | 2 +- src/Processors/QueryPlan/FillingStep.h | 2 +- src/Processors/QueryPlan/FilterStep.cpp | 2 +- src/Processors/QueryPlan/FilterStep.h | 2 +- .../QueryPlan/FinishSortingStep.cpp | 2 +- src/Processors/QueryPlan/FinishSortingStep.h | 2 +- src/Processors/QueryPlan/IQueryPlanStep.cpp | 8 +++--- src/Processors/QueryPlan/IQueryPlanStep.h | 25 +++++++++---------- src/Processors/QueryPlan/ISourceStep.cpp | 2 +- src/Processors/QueryPlan/ISourceStep.h | 2 +- .../QueryPlan/ITransformingStep.cpp | 2 +- src/Processors/QueryPlan/ITransformingStep.h | 2 +- src/Processors/QueryPlan/LimitByStep.cpp | 2 +- src/Processors/QueryPlan/LimitByStep.h | 2 +- src/Processors/QueryPlan/LimitStep.cpp | 2 +- src/Processors/QueryPlan/LimitStep.h | 2 +- src/Processors/QueryPlan/MergeSortingStep.cpp | 2 +- src/Processors/QueryPlan/MergeSortingStep.h | 2 +- .../QueryPlan/MergingAggregatedStep.cpp | 2 +- .../QueryPlan/MergingAggregatedStep.h | 2 +- src/Processors/QueryPlan/MergingFinal.cpp | 2 +- src/Processors/QueryPlan/MergingFinal.h | 2 +- .../QueryPlan/MergingSortedStep.cpp | 2 +- src/Processors/QueryPlan/MergingSortedStep.h | 2 +- src/Processors/QueryPlan/OffsetStep.cpp | 2 +- src/Processors/QueryPlan/OffsetStep.h | 2 +- .../QueryPlan/PartialSortingStep.cpp | 2 +- src/Processors/QueryPlan/PartialSortingStep.h | 2 +- src/Processors/QueryPlan/TotalsHavingStep.cpp | 2 +- src/Processors/QueryPlan/TotalsHavingStep.h | 2 +- src/Processors/QueryPlan/UnionStep.cpp | 2 +- src/Processors/QueryPlan/UnionStep.h | 2 +- src/Processors/QueryPlan/WindowStep.cpp | 2 +- src/Processors/QueryPlan/WindowStep.h | 2 +- 44 files changed, 61 insertions(+), 64 deletions(-) diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 8b787859467..e8d4a262366 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -156,12 +156,12 @@ void AggregatingStep::transformPipeline(QueryPipeline & pipeline) } } -void AggregatingStep::describeActions(QueryPlanStepFormatSettings & settings) const +void AggregatingStep::describeActions(FormatSettings & settings) const { params.explain(settings.out, settings.offset); } -void AggregatingStep::describePipeline(QueryPlanStepFormatSettings & settings) const +void AggregatingStep::describePipeline(FormatSettings & settings) const { if (!aggregating.empty()) IQueryPlanStep::describePipeline(aggregating, settings); diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index 3d12e79a662..853173895b3 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -29,8 +29,8 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings &) const override; - void describePipeline(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings &) const override; + void describePipeline(FormatSettings & settings) const override; private: Aggregator::Params params; diff --git a/src/Processors/QueryPlan/ArrayJoinStep.cpp b/src/Processors/QueryPlan/ArrayJoinStep.cpp index 01cfd7c561a..b71d2176e52 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.cpp +++ b/src/Processors/QueryPlan/ArrayJoinStep.cpp @@ -69,7 +69,7 @@ void ArrayJoinStep::transformPipeline(QueryPipeline & pipeline) } } -void ArrayJoinStep::describeActions(QueryPlanStepFormatSettings & settings) const +void ArrayJoinStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); bool first = true; diff --git a/src/Processors/QueryPlan/ArrayJoinStep.h b/src/Processors/QueryPlan/ArrayJoinStep.h index 338ec7f0583..92c7e0a1304 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.h +++ b/src/Processors/QueryPlan/ArrayJoinStep.h @@ -15,7 +15,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; void updateInputStream(DataStream input_stream, Block result_header); diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index c6b77bc46c3..5868a7045f7 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -47,7 +47,7 @@ void CreatingSetStep::transformPipeline(QueryPipeline & pipeline) pipeline.addCreatingSetsTransform(getOutputStream().header, std::move(subquery_for_set), network_transfer_limits, context); } -void CreatingSetStep::describeActions(QueryPlanStepFormatSettings & settings) const +void CreatingSetStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); @@ -102,7 +102,7 @@ QueryPipelinePtr CreatingSetsStep::updatePipeline(QueryPipelines pipelines) return main_pipeline; } -void CreatingSetsStep::describePipeline(QueryPlanStepFormatSettings & settings) const +void CreatingSetsStep::describePipeline(FormatSettings & settings) const { IQueryPlanStep::describePipeline(processors, settings); } diff --git a/src/Processors/QueryPlan/CreatingSetsStep.h b/src/Processors/QueryPlan/CreatingSetsStep.h index f610eeb64d9..ec13ab2052e 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.h +++ b/src/Processors/QueryPlan/CreatingSetsStep.h @@ -22,7 +22,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; private: String description; @@ -40,7 +40,7 @@ public: QueryPipelinePtr updatePipeline(QueryPipelines pipelines) override; - void describePipeline(QueryPlanStepFormatSettings & settings) const override; + void describePipeline(FormatSettings & settings) const override; private: Processors processors; diff --git a/src/Processors/QueryPlan/DistinctStep.cpp b/src/Processors/QueryPlan/DistinctStep.cpp index ed2bae002ea..60966b08beb 100644 --- a/src/Processors/QueryPlan/DistinctStep.cpp +++ b/src/Processors/QueryPlan/DistinctStep.cpp @@ -79,7 +79,7 @@ void DistinctStep::transformPipeline(QueryPipeline & pipeline) }); } -void DistinctStep::describeActions(QueryPlanStepFormatSettings & settings) const +void DistinctStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Columns: "; diff --git a/src/Processors/QueryPlan/DistinctStep.h b/src/Processors/QueryPlan/DistinctStep.h index 901e199ca39..4bfd73ce044 100644 --- a/src/Processors/QueryPlan/DistinctStep.h +++ b/src/Processors/QueryPlan/DistinctStep.h @@ -20,7 +20,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; private: SizeLimits set_size_limits; diff --git a/src/Processors/QueryPlan/ExpressionStep.cpp b/src/Processors/QueryPlan/ExpressionStep.cpp index 028c9502c71..f7ca0a972dc 100644 --- a/src/Processors/QueryPlan/ExpressionStep.cpp +++ b/src/Processors/QueryPlan/ExpressionStep.cpp @@ -87,7 +87,7 @@ void ExpressionStep::transformPipeline(QueryPipeline & pipeline) } } -void ExpressionStep::describeActions(QueryPlanStepFormatSettings & settings) const +void ExpressionStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); bool first = true; diff --git a/src/Processors/QueryPlan/ExpressionStep.h b/src/Processors/QueryPlan/ExpressionStep.h index 2519765dbae..60f186688b0 100644 --- a/src/Processors/QueryPlan/ExpressionStep.h +++ b/src/Processors/QueryPlan/ExpressionStep.h @@ -26,7 +26,7 @@ public: void updateInputStream(DataStream input_stream, bool keep_header); - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; const ActionsDAGPtr & getExpression() const { return actions_dag; } diff --git a/src/Processors/QueryPlan/FillingStep.cpp b/src/Processors/QueryPlan/FillingStep.cpp index 18c71b89601..1a8fba97ee2 100644 --- a/src/Processors/QueryPlan/FillingStep.cpp +++ b/src/Processors/QueryPlan/FillingStep.cpp @@ -43,7 +43,7 @@ void FillingStep::transformPipeline(QueryPipeline & pipeline) }); } -void FillingStep::describeActions(QueryPlanStepFormatSettings & settings) const +void FillingStep::describeActions(FormatSettings & settings) const { settings.out << String(settings.offset, ' '); dumpSortDescription(sort_description, input_streams.front().header, settings.out); diff --git a/src/Processors/QueryPlan/FillingStep.h b/src/Processors/QueryPlan/FillingStep.h index a7836e4de68..85736464a6c 100644 --- a/src/Processors/QueryPlan/FillingStep.h +++ b/src/Processors/QueryPlan/FillingStep.h @@ -15,7 +15,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; private: SortDescription sort_description; diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp index 98f111a1529..8fb405e685b 100644 --- a/src/Processors/QueryPlan/FilterStep.cpp +++ b/src/Processors/QueryPlan/FilterStep.cpp @@ -80,7 +80,7 @@ void FilterStep::transformPipeline(QueryPipeline & pipeline) } } -void FilterStep::describeActions(QueryPlanStepFormatSettings & settings) const +void FilterStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Filter column: " << filter_column_name << '\n'; diff --git a/src/Processors/QueryPlan/FilterStep.h b/src/Processors/QueryPlan/FilterStep.h index 561fbf347da..72b02624faa 100644 --- a/src/Processors/QueryPlan/FilterStep.h +++ b/src/Processors/QueryPlan/FilterStep.h @@ -22,7 +22,7 @@ public: void updateInputStream(DataStream input_stream, bool keep_header); - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; const ActionsDAGPtr & getExpression() const { return actions_dag; } const String & getFilterColumnName() const { return filter_column_name; } diff --git a/src/Processors/QueryPlan/FinishSortingStep.cpp b/src/Processors/QueryPlan/FinishSortingStep.cpp index 7af8fad2980..d883bd0e0dd 100644 --- a/src/Processors/QueryPlan/FinishSortingStep.cpp +++ b/src/Processors/QueryPlan/FinishSortingStep.cpp @@ -85,7 +85,7 @@ void FinishSortingStep::transformPipeline(QueryPipeline & pipeline) } } -void FinishSortingStep::describeActions(QueryPlanStepFormatSettings & settings) const +void FinishSortingStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); diff --git a/src/Processors/QueryPlan/FinishSortingStep.h b/src/Processors/QueryPlan/FinishSortingStep.h index 25dc0b78bda..4bb62037faa 100644 --- a/src/Processors/QueryPlan/FinishSortingStep.h +++ b/src/Processors/QueryPlan/FinishSortingStep.h @@ -20,7 +20,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; /// Add limit or change it to lower value. void updateLimit(size_t limit_); diff --git a/src/Processors/QueryPlan/IQueryPlanStep.cpp b/src/Processors/QueryPlan/IQueryPlanStep.cpp index a864043ddcb..f06897e8488 100644 --- a/src/Processors/QueryPlan/IQueryPlanStep.cpp +++ b/src/Processors/QueryPlan/IQueryPlanStep.cpp @@ -18,8 +18,7 @@ const DataStream & IQueryPlanStep::getOutputStream() const return *output_stream; } -static void doDescribeHeader(const Block & header, size_t count, - QueryPlanStepFormatSettings & settings) +static void doDescribeHeader(const Block & header, size_t count, IQueryPlanStep::FormatSettings & settings) { String prefix(settings.offset, settings.indent_char); prefix += "Header"; @@ -53,8 +52,7 @@ static void doDescribeHeader(const Block & header, size_t count, } } -static void doDescribeProcessor(const IProcessor & processor, size_t count, - QueryPlanStepFormatSettings & settings) +static void doDescribeProcessor(const IProcessor & processor, size_t count, IQueryPlanStep::FormatSettings & settings) { settings.out << String(settings.offset, settings.indent_char) << processor.getName(); if (count > 1) @@ -91,7 +89,7 @@ static void doDescribeProcessor(const IProcessor & processor, size_t count, settings.offset += settings.indent; } -void IQueryPlanStep::describePipeline(const Processors & processors, QueryPlanStepFormatSettings & settings) +void IQueryPlanStep::describePipeline(const Processors & processors, FormatSettings & settings) { const IProcessor * prev = nullptr; size_t count = 0; diff --git a/src/Processors/QueryPlan/IQueryPlanStep.h b/src/Processors/QueryPlan/IQueryPlanStep.h index 1de1bbc438f..dfdbc9df442 100644 --- a/src/Processors/QueryPlan/IQueryPlanStep.h +++ b/src/Processors/QueryPlan/IQueryPlanStep.h @@ -61,16 +61,6 @@ public: using DataStreams = std::vector; -// Not a nested class so that we can forward declare it. -struct QueryPlanStepFormatSettings -{ - WriteBuffer & out; - size_t offset = 0; - const size_t indent = 2; - const char indent_char = ' '; - const bool write_header = false; -}; - /// Single step of query plan. class IQueryPlanStep { @@ -96,11 +86,20 @@ public: const std::string & getStepDescription() const { return step_description; } void setStepDescription(std::string description) { step_description = std::move(description); } + struct FormatSettings + { + WriteBuffer & out; + size_t offset = 0; + const size_t indent = 2; + const char indent_char = ' '; + const bool write_header = false; + }; + /// Get detailed description of step actions. This is shown in EXPLAIN query with options `actions = 1`. - virtual void describeActions(QueryPlanStepFormatSettings & /*settings*/) const {} + virtual void describeActions(FormatSettings & /*settings*/) const {} /// Get description of processors added in current step. Should be called after updatePipeline(). - virtual void describePipeline(QueryPlanStepFormatSettings & /*settings*/) const {} + virtual void describePipeline(FormatSettings & /*settings*/) const {} protected: DataStreams input_streams; @@ -109,7 +108,7 @@ protected: /// Text description about what current step does. std::string step_description; - static void describePipeline(const Processors & processors, QueryPlanStepFormatSettings & settings); + static void describePipeline(const Processors & processors, FormatSettings & settings); }; using QueryPlanStepPtr = std::unique_ptr; diff --git a/src/Processors/QueryPlan/ISourceStep.cpp b/src/Processors/QueryPlan/ISourceStep.cpp index 367be8e8ea5..1796f033896 100644 --- a/src/Processors/QueryPlan/ISourceStep.cpp +++ b/src/Processors/QueryPlan/ISourceStep.cpp @@ -19,7 +19,7 @@ QueryPipelinePtr ISourceStep::updatePipeline(QueryPipelines) return pipeline; } -void ISourceStep::describePipeline(QueryPlanStepFormatSettings & settings) const +void ISourceStep::describePipeline(FormatSettings & settings) const { IQueryPlanStep::describePipeline(processors, settings); } diff --git a/src/Processors/QueryPlan/ISourceStep.h b/src/Processors/QueryPlan/ISourceStep.h index 050e39ac11b..fdb3dd566cb 100644 --- a/src/Processors/QueryPlan/ISourceStep.h +++ b/src/Processors/QueryPlan/ISourceStep.h @@ -14,7 +14,7 @@ public: virtual void initializePipeline(QueryPipeline & pipeline) = 0; - void describePipeline(QueryPlanStepFormatSettings & settings) const override; + void describePipeline(FormatSettings & settings) const override; protected: /// We collect processors got after pipeline transformation. diff --git a/src/Processors/QueryPlan/ITransformingStep.cpp b/src/Processors/QueryPlan/ITransformingStep.cpp index 5430bafa27a..cb27bf38278 100644 --- a/src/Processors/QueryPlan/ITransformingStep.cpp +++ b/src/Processors/QueryPlan/ITransformingStep.cpp @@ -65,7 +65,7 @@ void ITransformingStep::updateDistinctColumns(const Block & res_header, NameSet } } -void ITransformingStep::describePipeline(QueryPlanStepFormatSettings & settings) const +void ITransformingStep::describePipeline(FormatSettings & settings) const { IQueryPlanStep::describePipeline(processors, settings); } diff --git a/src/Processors/QueryPlan/ITransformingStep.h b/src/Processors/QueryPlan/ITransformingStep.h index b1bfebd4571..bd99478ec37 100644 --- a/src/Processors/QueryPlan/ITransformingStep.h +++ b/src/Processors/QueryPlan/ITransformingStep.h @@ -55,7 +55,7 @@ public: const TransformTraits & getTransformTraits() const { return transform_traits; } const DataStreamTraits & getDataStreamTraits() const { return data_stream_traits; } - void describePipeline(QueryPlanStepFormatSettings & settings) const override; + void describePipeline(FormatSettings & settings) const override; protected: /// Clear distinct_columns if res_header doesn't contain all of them. diff --git a/src/Processors/QueryPlan/LimitByStep.cpp b/src/Processors/QueryPlan/LimitByStep.cpp index 15c64fcea44..9fcf5b60164 100644 --- a/src/Processors/QueryPlan/LimitByStep.cpp +++ b/src/Processors/QueryPlan/LimitByStep.cpp @@ -46,7 +46,7 @@ void LimitByStep::transformPipeline(QueryPipeline & pipeline) }); } -void LimitByStep::describeActions(QueryPlanStepFormatSettings & settings) const +void LimitByStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); diff --git a/src/Processors/QueryPlan/LimitByStep.h b/src/Processors/QueryPlan/LimitByStep.h index 01dcff5c912..9320735640c 100644 --- a/src/Processors/QueryPlan/LimitByStep.h +++ b/src/Processors/QueryPlan/LimitByStep.h @@ -16,7 +16,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; private: size_t group_length; diff --git a/src/Processors/QueryPlan/LimitStep.cpp b/src/Processors/QueryPlan/LimitStep.cpp index 38ec9265d4e..565d05956e5 100644 --- a/src/Processors/QueryPlan/LimitStep.cpp +++ b/src/Processors/QueryPlan/LimitStep.cpp @@ -50,7 +50,7 @@ void LimitStep::transformPipeline(QueryPipeline & pipeline) pipeline.addTransform(std::move(transform)); } -void LimitStep::describeActions(QueryPlanStepFormatSettings & settings) const +void LimitStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Limit " << limit << '\n'; diff --git a/src/Processors/QueryPlan/LimitStep.h b/src/Processors/QueryPlan/LimitStep.h index a95bf28f896..eea186fea4a 100644 --- a/src/Processors/QueryPlan/LimitStep.h +++ b/src/Processors/QueryPlan/LimitStep.h @@ -20,7 +20,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; size_t getLimitForSorting() const { diff --git a/src/Processors/QueryPlan/MergeSortingStep.cpp b/src/Processors/QueryPlan/MergeSortingStep.cpp index e2af023f598..b30286130b1 100644 --- a/src/Processors/QueryPlan/MergeSortingStep.cpp +++ b/src/Processors/QueryPlan/MergeSortingStep.cpp @@ -73,7 +73,7 @@ void MergeSortingStep::transformPipeline(QueryPipeline & pipeline) }); } -void MergeSortingStep::describeActions(QueryPlanStepFormatSettings & settings) const +void MergeSortingStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Sort description: "; diff --git a/src/Processors/QueryPlan/MergeSortingStep.h b/src/Processors/QueryPlan/MergeSortingStep.h index c41c209dd55..a385a8a3e93 100644 --- a/src/Processors/QueryPlan/MergeSortingStep.h +++ b/src/Processors/QueryPlan/MergeSortingStep.h @@ -26,7 +26,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; /// Add limit or change it to lower value. void updateLimit(size_t limit_); diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.cpp b/src/Processors/QueryPlan/MergingAggregatedStep.cpp index 2fddc88bbc1..85bbbeab59a 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.cpp +++ b/src/Processors/QueryPlan/MergingAggregatedStep.cpp @@ -63,7 +63,7 @@ void MergingAggregatedStep::transformPipeline(QueryPipeline & pipeline) } } -void MergingAggregatedStep::describeActions(QueryPlanStepFormatSettings & settings) const +void MergingAggregatedStep::describeActions(FormatSettings & settings) const { return params->params.explain(settings.out, settings.offset); } diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.h b/src/Processors/QueryPlan/MergingAggregatedStep.h index 2d9d902a294..5ffceb7f938 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.h +++ b/src/Processors/QueryPlan/MergingAggregatedStep.h @@ -23,7 +23,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; private: AggregatingTransformParamsPtr params; diff --git a/src/Processors/QueryPlan/MergingFinal.cpp b/src/Processors/QueryPlan/MergingFinal.cpp index 46431e0c767..c1dec231f11 100644 --- a/src/Processors/QueryPlan/MergingFinal.cpp +++ b/src/Processors/QueryPlan/MergingFinal.cpp @@ -153,7 +153,7 @@ void MergingFinal::transformPipeline(QueryPipeline & pipeline) }); } -void MergingFinal::describeActions(QueryPlanStepFormatSettings & settings) const +void MergingFinal::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Sort description: "; diff --git a/src/Processors/QueryPlan/MergingFinal.h b/src/Processors/QueryPlan/MergingFinal.h index 207c191c9cb..c01f5c7f9a1 100644 --- a/src/Processors/QueryPlan/MergingFinal.h +++ b/src/Processors/QueryPlan/MergingFinal.h @@ -22,7 +22,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; private: size_t num_output_streams; diff --git a/src/Processors/QueryPlan/MergingSortedStep.cpp b/src/Processors/QueryPlan/MergingSortedStep.cpp index a1249ef412f..c59540b009f 100644 --- a/src/Processors/QueryPlan/MergingSortedStep.cpp +++ b/src/Processors/QueryPlan/MergingSortedStep.cpp @@ -62,7 +62,7 @@ void MergingSortedStep::transformPipeline(QueryPipeline & pipeline) } } -void MergingSortedStep::describeActions(QueryPlanStepFormatSettings & settings) const +void MergingSortedStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Sort description: "; diff --git a/src/Processors/QueryPlan/MergingSortedStep.h b/src/Processors/QueryPlan/MergingSortedStep.h index 0c91d505f3d..483cfa5e8a7 100644 --- a/src/Processors/QueryPlan/MergingSortedStep.h +++ b/src/Processors/QueryPlan/MergingSortedStep.h @@ -21,7 +21,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; /// Add limit or change it to lower value. void updateLimit(size_t limit_); diff --git a/src/Processors/QueryPlan/OffsetStep.cpp b/src/Processors/QueryPlan/OffsetStep.cpp index 1a06f2ba05e..7ac3d3f2110 100644 --- a/src/Processors/QueryPlan/OffsetStep.cpp +++ b/src/Processors/QueryPlan/OffsetStep.cpp @@ -36,7 +36,7 @@ void OffsetStep::transformPipeline(QueryPipeline & pipeline) pipeline.addTransform(std::move(transform)); } -void OffsetStep::describeActions(QueryPlanStepFormatSettings & settings) const +void OffsetStep::describeActions(FormatSettings & settings) const { settings.out << String(settings.offset, ' ') << "Offset " << offset << '\n'; } diff --git a/src/Processors/QueryPlan/OffsetStep.h b/src/Processors/QueryPlan/OffsetStep.h index 18ea58635af..17949371edf 100644 --- a/src/Processors/QueryPlan/OffsetStep.h +++ b/src/Processors/QueryPlan/OffsetStep.h @@ -15,7 +15,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; private: size_t offset; diff --git a/src/Processors/QueryPlan/PartialSortingStep.cpp b/src/Processors/QueryPlan/PartialSortingStep.cpp index 072217b4d0d..ce34eca9112 100644 --- a/src/Processors/QueryPlan/PartialSortingStep.cpp +++ b/src/Processors/QueryPlan/PartialSortingStep.cpp @@ -70,7 +70,7 @@ void PartialSortingStep::transformPipeline(QueryPipeline & pipeline) }); } -void PartialSortingStep::describeActions(QueryPlanStepFormatSettings & settings) const +void PartialSortingStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Sort description: "; diff --git a/src/Processors/QueryPlan/PartialSortingStep.h b/src/Processors/QueryPlan/PartialSortingStep.h index 6d9dbe2b9d8..172ef25c300 100644 --- a/src/Processors/QueryPlan/PartialSortingStep.h +++ b/src/Processors/QueryPlan/PartialSortingStep.h @@ -20,7 +20,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; /// Add limit or change it to lower value. void updateLimit(size_t limit_); diff --git a/src/Processors/QueryPlan/TotalsHavingStep.cpp b/src/Processors/QueryPlan/TotalsHavingStep.cpp index 310f657b39d..9947875e679 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.cpp +++ b/src/Processors/QueryPlan/TotalsHavingStep.cpp @@ -74,7 +74,7 @@ static String totalsModeToString(TotalsMode totals_mode, double auto_include_thr __builtin_unreachable(); } -void TotalsHavingStep::describeActions(QueryPlanStepFormatSettings & settings) const +void TotalsHavingStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Filter column: " << filter_column_name << '\n'; diff --git a/src/Processors/QueryPlan/TotalsHavingStep.h b/src/Processors/QueryPlan/TotalsHavingStep.h index 88fd9bbb597..7c1638013e5 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.h +++ b/src/Processors/QueryPlan/TotalsHavingStep.h @@ -26,7 +26,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; private: bool overflow_row; diff --git a/src/Processors/QueryPlan/UnionStep.cpp b/src/Processors/QueryPlan/UnionStep.cpp index 413b52c69c2..630ff53f47d 100644 --- a/src/Processors/QueryPlan/UnionStep.cpp +++ b/src/Processors/QueryPlan/UnionStep.cpp @@ -36,7 +36,7 @@ QueryPipelinePtr UnionStep::updatePipeline(QueryPipelines pipelines) return pipeline; } -void UnionStep::describePipeline(QueryPlanStepFormatSettings & settings) const +void UnionStep::describePipeline(FormatSettings & settings) const { IQueryPlanStep::describePipeline(processors, settings); } diff --git a/src/Processors/QueryPlan/UnionStep.h b/src/Processors/QueryPlan/UnionStep.h index aeac7fa54ff..e2e1f2c9efa 100644 --- a/src/Processors/QueryPlan/UnionStep.h +++ b/src/Processors/QueryPlan/UnionStep.h @@ -15,7 +15,7 @@ public: QueryPipelinePtr updatePipeline(QueryPipelines pipelines) override; - void describePipeline(QueryPlanStepFormatSettings & settings) const override; + void describePipeline(FormatSettings & settings) const override; private: Block header; diff --git a/src/Processors/QueryPlan/WindowStep.cpp b/src/Processors/QueryPlan/WindowStep.cpp index d2d24985937..82c589b8b20 100644 --- a/src/Processors/QueryPlan/WindowStep.cpp +++ b/src/Processors/QueryPlan/WindowStep.cpp @@ -71,7 +71,7 @@ void WindowStep::transformPipeline(QueryPipeline & pipeline) "WindowStep transform for '" + window_description.window_name + "'"); } -void WindowStep::describeActions(QueryPlanStepFormatSettings & settings) const +void WindowStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); settings.out << prefix << "Window: ("; diff --git a/src/Processors/QueryPlan/WindowStep.h b/src/Processors/QueryPlan/WindowStep.h index 25194820a0d..069d02c655c 100644 --- a/src/Processors/QueryPlan/WindowStep.h +++ b/src/Processors/QueryPlan/WindowStep.h @@ -22,7 +22,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; - void describeActions(QueryPlanStepFormatSettings & settings) const override; + void describeActions(FormatSettings & settings) const override; private: WindowDescription window_description; From 61a35cdafd056e32879ab6389fbf33c9d89d58d9 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 24 Dec 2020 12:11:37 +0300 Subject: [PATCH 115/284] final cleanup --- src/Processors/QueryPlan/QueryPlan.cpp | 10 +++++----- src/Processors/QueryPlan/QueryPlan.h | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index a2fc8c689ff..1b3ea16a213 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -204,7 +204,7 @@ void QueryPlan::addInterpreterContext(std::shared_ptr context) static void explainStep( const IQueryPlanStep & step, - QueryPlanStepFormatSettings & settings, + IQueryPlanStep::FormatSettings & settings, const QueryPlan::ExplainPlanOptions & options) { std::string prefix(settings.offset, ' '); @@ -250,7 +250,7 @@ static void explainStep( std::string debugExplainStep(const IQueryPlanStep & step) { WriteBufferFromOwnString out; - QueryPlanStepFormatSettings settings{.out = out}; + IQueryPlanStep::FormatSettings settings{.out = out}; QueryPlan::ExplainPlanOptions options{.actions = true}; explainStep(step, settings, options); return out.str(); @@ -260,7 +260,7 @@ void QueryPlan::explainPlan(WriteBuffer & buffer, const ExplainPlanOptions & opt { checkInitialized(); - QueryPlanStepFormatSettings settings{.out = buffer, .write_header = options.header}; + IQueryPlanStep::FormatSettings settings{.out = buffer, .write_header = options.header}; struct Frame { @@ -293,7 +293,7 @@ void QueryPlan::explainPlan(WriteBuffer & buffer, const ExplainPlanOptions & opt } } -static void explainPipelineStep(IQueryPlanStep & step, QueryPlanStepFormatSettings & settings) +static void explainPipelineStep(IQueryPlanStep & step, IQueryPlanStep::FormatSettings & settings) { settings.out << String(settings.offset, settings.indent_char) << "(" << step.getName() << ")\n"; size_t current_offset = settings.offset; @@ -306,7 +306,7 @@ void QueryPlan::explainPipeline(WriteBuffer & buffer, const ExplainPipelineOptio { checkInitialized(); - QueryPlanStepFormatSettings settings{.out = buffer, .write_header = options.header}; + IQueryPlanStep::FormatSettings settings{.out = buffer, .write_header = options.header}; struct Frame { diff --git a/src/Processors/QueryPlan/QueryPlan.h b/src/Processors/QueryPlan/QueryPlan.h index 4c55718a062..9d2d7d93a36 100644 --- a/src/Processors/QueryPlan/QueryPlan.h +++ b/src/Processors/QueryPlan/QueryPlan.h @@ -13,7 +13,6 @@ class DataStream; class IQueryPlanStep; using QueryPlanStepPtr = std::unique_ptr; -struct QueryPlanStepFormatSettings; class QueryPipeline; using QueryPipelinePtr = std::unique_ptr; From 5e8cf89b9d6be6f58cfcda3b9cd1d0307e2eafa5 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 24 Dec 2020 12:13:54 +0300 Subject: [PATCH 116/284] cleanup --- src/Interpreters/ExpressionAnalyzer.cpp | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 37f2791d8bd..0a380c66d23 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -981,6 +981,7 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments( getRootActionsNoMakeSet(f.function_node->clone(), true /* no_subqueries */, step.actions()); + // FIXME rewrite this comment // 1.2) result of window function: an empty INPUT. // It is an aggregate function, so it won't be added by getRootActions. // This is something of a hack. Other options: @@ -999,12 +1000,6 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments( // Expression pipelines. This is a "proper" way that would allow // us to depend on window functions in other functions. But it's // complicated so I avoid doing it for now. - ColumnWithTypeAndName col; - col.type = f.aggregate_function->getReturnType(); - col.column = col.type->createColumn(); - col.name = f.column_name; - -// step.actions()->addInput(col); columns_after_window.push_back({f.column_name, f.aggregate_function->getReturnType()}); @@ -1013,8 +1008,6 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments( // 2.1) function arguments; step.required_output.push_back(a->getColumnName()); } -// // 2.2) function result; -// step.required_output.push_back(f.column_name); } // 2.3) PARTITION BY and ORDER BY columns. From 8c72fa063df28b25c04acdf24405f7347a04fd6c Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 24 Dec 2020 13:59:58 +0300 Subject: [PATCH 117/284] boop the CI --- src/Interpreters/ExpressionAnalyzer.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 0a380c66d23..4af2a17e164 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1426,6 +1426,12 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( query_analyzer.appendSelect(chain, only_types || (need_aggregate ? !second_stage : !first_stage)); has_order_by = query.orderBy() != nullptr; + // FIXME selected columns was set here. Should split order by and select + // and insert window functions in between. + // Will fix: + // 1) window function in DISTINCT + // 2) totally broken DISTINCT + // 3) window functions in order by before_order_and_select = query_analyzer.appendOrderBy( chain, only_types || (need_aggregate ? !second_stage : !first_stage), From e7ffc135247c9fdf6d89f7c2d679081f8e28617c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 24 Dec 2020 18:49:05 +0300 Subject: [PATCH 118/284] Renames. --- src/Functions/FunctionsStringHash.cpp | 226 ++++++++++++-------------- 1 file changed, 106 insertions(+), 120 deletions(-) diff --git a/src/Functions/FunctionsStringHash.cpp b/src/Functions/FunctionsStringHash.cpp index 7062a47fa22..2f1f900bda9 100644 --- a/src/Functions/FunctionsStringHash.cpp +++ b/src/Functions/FunctionsStringHash.cpp @@ -3,8 +3,6 @@ #include #include #include -#include -#include #include #include @@ -141,22 +139,19 @@ struct Hash } }; -// Simhash String -> UInt64 +// SimHash String -> UInt64 // N: the length of ngram or words shingles // CodePoint: UInt8(ASCII) or UInt32(UTF8) // UTF8: means ASCII or UTF8, these two parameters CodePoint and UTF8 can only be (UInt8, false) or (UInt32, true) // Ngram: means ngram(true) or words shingles(false) // CaseInsensitive: means should we consider about letter case or not template -struct SimhashImpl +struct SimHashImpl { using StrOp = ExtractStringImpl; - // we made an assumption that the size of one word can't exceed 128, which may not true - // if some word's size exceed 128, it would be cut up to several word - static constexpr size_t max_string_size = 1u << 15; static constexpr size_t simultaneously_codepoints_num = StrOp::buffer_size; - // Simhash ngram calculate function: String ->UInt64 + // SimHash ngram calculate function: String ->UInt64 // this function extracting ngram from input string, and maintain a 64-dimensions vector // for each ngram, calculate a 64 bit hash value, and update the vector according the hash value // finally return a 64 bit value(UInt64), i'th bit is 1 means vector[i] > 0, otherwise, vector[i] < 0 @@ -203,7 +198,7 @@ struct SimhashImpl return res_bit.to_ullong(); } - // Simhash word shingle calculate function: String -> UInt64 + // SimHash word shingle calculate function: String -> UInt64 // this function extracting n word shingle from input string, and maintain a 64-dimensions vector as well // for each word shingle, calculate a 64 bit hash value, and update the vector according the hash value // finally return a 64 bit value(UInt64), i'th bit is 1 means vector[i] > 0, otherwise, vector[i] < 0 @@ -283,25 +278,21 @@ struct SimhashImpl { const char * one_data = reinterpret_cast(&data[offsets[i - 1]]); const size_t data_size = offsets[i] - offsets[i - 1] - 1; - if (data_size <= max_string_size) + + if constexpr (Ngram) { - if constexpr (Ngram) - { - if constexpr (!UTF8) - res[i] = ngramCalculateHashValue(one_data, data_size, StrOp::readASCIICodePoints, Hash::ngramASCIIHash); - else - res[i] = ngramCalculateHashValue(one_data, data_size, StrOp::readUTF8CodePoints, Hash::ngramUTF8Hash); - } + if constexpr (!UTF8) + res[i] = ngramCalculateHashValue(one_data, data_size, StrOp::readASCIICodePoints, Hash::ngramASCIIHash); else - { - if constexpr (!UTF8) - res[i] = wordShinglesCalculateHashValue(one_data, data_size, StrOp::readOneASCIIWord, Hash::wordShinglesHash); - else - res[i] = wordShinglesCalculateHashValue(one_data, data_size, StrOp::readOneUTF8Word, Hash::wordShinglesHash); - } + res[i] = ngramCalculateHashValue(one_data, data_size, StrOp::readUTF8CodePoints, Hash::ngramUTF8Hash); } else - res[i] = -1ull; + { + if constexpr (!UTF8) + res[i] = wordShinglesCalculateHashValue(one_data, data_size, StrOp::readOneASCIIWord, Hash::wordShinglesHash); + else + res[i] = wordShinglesCalculateHashValue(one_data, data_size, StrOp::readOneUTF8Word, Hash::wordShinglesHash); + } } } }; @@ -333,7 +324,7 @@ private: }; -// Minhash: String -> Tuple(UInt64, UInt64) +// MinHash: String -> Tuple(UInt64, UInt64) // for each string, we extract ngram or word shingle, // for each ngram or word shingle, calculate a hash value, // then we take the K minimum hash values to calculate a hashsum, @@ -347,17 +338,16 @@ private: // Ngram: means ngram(true) or words shingles(false) // CaseInsensitive: means should we consider about letter case or not template -struct MinhashImpl +struct MinHashImpl { using Less = std::less; using Greater = std::greater; using MaxHeap = FixedHeap, K, -1ULL>; using MinHeap = FixedHeap, K, 0>; using StrOp = ExtractStringImpl; - static constexpr size_t max_string_size = 1u << 15; static constexpr size_t simultaneously_codepoints_num = StrOp::buffer_size; - // Minhash ngram calculate function, String -> Tuple(UInt64, UInt64) + // MinHash ngram calculate function, String -> Tuple(UInt64, UInt64) // we extract ngram from input string, and calculate a hash value for each ngram // then we take the K minimum hash values to calculate a hashsum, // and take the K maximum hash values to calculate another hashsum, @@ -397,7 +387,7 @@ struct MinhashImpl return std::make_tuple(res1, res2); } - // Minhash word shingle hash value calculate function: String ->Tuple(UInt64, UInt64) + // MinHash word shingle hash value calculate function: String ->Tuple(UInt64, UInt64) // for each word shingle, we calculate a hash value, but in fact, we just maintain the // K minimum and K maximum hash value static ALWAYS_INLINE inline std::tuple wordShinglesCalculateHashValue( @@ -416,7 +406,7 @@ struct MinhashImpl // word buffer to store one word PaddedPODArray word_buf; // how word shingle hash value calculation and word hash storation is same as we - // have descripted in Simhash wordShinglesCalculateHashValue function + // have descripted in SimHash wordShinglesCalculateHashValue function for (size_t i = 0; i < N && start < end; ++i) { read_one_word(word_buf, start, end); @@ -456,171 +446,167 @@ struct MinhashImpl { const char * one_data = reinterpret_cast(&data[offsets[i - 1]]); const size_t data_size = offsets[i] - offsets[i - 1] - 1; - if (data_size <= max_string_size) + + if constexpr (Ngram) { - if constexpr (Ngram) - { - if constexpr (!UTF8) - std::tie(res1[i], res2[i]) = ngramCalculateHashValue(one_data, data_size, StrOp::readASCIICodePoints, Hash::ngramASCIIHash); - else - std::tie(res1[i], res2[i]) = ngramCalculateHashValue(one_data, data_size, StrOp::readUTF8CodePoints, Hash::ngramUTF8Hash); - } + if constexpr (!UTF8) + std::tie(res1[i], res2[i]) = ngramCalculateHashValue(one_data, data_size, StrOp::readASCIICodePoints, Hash::ngramASCIIHash); else - { - if constexpr (!UTF8) - std::tie(res1[i], res2[i]) = wordShinglesCalculateHashValue(one_data, data_size, StrOp::readOneASCIIWord, Hash::wordShinglesHash); - else - std::tie(res1[i], res2[i]) = wordShinglesCalculateHashValue(one_data, data_size, StrOp::readOneUTF8Word, Hash::wordShinglesHash); - } + std::tie(res1[i], res2[i]) = ngramCalculateHashValue(one_data, data_size, StrOp::readUTF8CodePoints, Hash::ngramUTF8Hash); } else - std::tie(res1[i], res2[i]) = std::make_tuple(-1ull, -1ull); + { + if constexpr (!UTF8) + std::tie(res1[i], res2[i]) = wordShinglesCalculateHashValue(one_data, data_size, StrOp::readOneASCIIWord, Hash::wordShinglesHash); + else + std::tie(res1[i], res2[i]) = wordShinglesCalculateHashValue(one_data, data_size, StrOp::readOneUTF8Word, Hash::wordShinglesHash); + } } } }; -struct NameNgramSimhash +struct NameNgramSimHash { - static constexpr auto name = "ngramSimhash"; + static constexpr auto name = "ngramSimHash"; }; -struct NameNgramSimhashCaseInsensitive +struct NameNgramSimHashCaseInsensitive { - static constexpr auto name = "ngramSimhashCaseInsensitive"; + static constexpr auto name = "ngramSimHashCaseInsensitive"; }; -struct NameNgramSimhashUTF8 +struct NameNgramSimHashUTF8 { - static constexpr auto name = "ngramSimhashUTF8"; + static constexpr auto name = "ngramSimHashUTF8"; }; -struct NameNgramSimhashCaseInsensitiveUTF8 +struct NameNgramSimHashCaseInsensitiveUTF8 { - static constexpr auto name = "ngramSimhashCaseInsensitiveUTF8"; + static constexpr auto name = "ngramSimHashCaseInsensitiveUTF8"; }; -struct NameWordShingleSimhash +struct NameWordShingleSimHash { - static constexpr auto name = "wordShingleSimhash"; + static constexpr auto name = "wordShingleSimHash"; }; -struct NameWordShingleSimhashCaseInsensitive +struct NameWordShingleSimHashCaseInsensitive { - static constexpr auto name = "wordShingleSimhashCaseInsensitive"; + static constexpr auto name = "wordShingleSimHashCaseInsensitive"; }; -struct NameWordShingleSimhashUTF8 +struct NameWordShingleSimHashUTF8 { - static constexpr auto name = "wordShingleSimhashUTF8"; + static constexpr auto name = "wordShingleSimHashUTF8"; }; -struct NameWordShingleSimhashCaseInsensitiveUTF8 +struct NameWordShingleSimHashCaseInsensitiveUTF8 { - static constexpr auto name = "wordShingleSimhashCaseInsensitiveUTF8"; + static constexpr auto name = "wordShingleSimHashCaseInsensitiveUTF8"; }; -struct NameNgramMinhash +struct NameNgramMinHash { - static constexpr auto name = "ngramMinhash"; + static constexpr auto name = "ngramMinHash"; }; -struct NameNgramMinhashCaseInsensitive +struct NameNgramMinHashCaseInsensitive { - static constexpr auto name = "ngramMinhashCaseInsensitive"; + static constexpr auto name = "ngramMinHashCaseInsensitive"; }; -struct NameNgramMinhashUTF8 +struct NameNgramMinHashUTF8 { - static constexpr auto name = "ngramMinhashUTF8"; + static constexpr auto name = "ngramMinHashUTF8"; }; -struct NameNgramMinhashCaseInsensitiveUTF8 +struct NameNgramMinHashCaseInsensitiveUTF8 { - static constexpr auto name = "ngramMinhashCaseInsensitiveUTF8"; + static constexpr auto name = "ngramMinHashCaseInsensitiveUTF8"; }; -struct NameWordShingleMinhash +struct NameWordShingleMinHash { - static constexpr auto name = "wordShingleMinhash"; + static constexpr auto name = "wordShingleMinHash"; }; -struct NameWordShingleMinhashCaseInsensitive +struct NameWordShingleMinHashCaseInsensitive { - static constexpr auto name = "wordShingleMinhashCaseInsensitive"; + static constexpr auto name = "wordShingleMinHashCaseInsensitive"; }; -struct NameWordShingleMinhashUTF8 +struct NameWordShingleMinHashUTF8 { - static constexpr auto name = "wordShingleMinhashUTF8"; + static constexpr auto name = "wordShingleMinHashUTF8"; }; -struct NameWordShingleMinhashCaseInsensitiveUTF8 +struct NameWordShingleMinHashCaseInsensitiveUTF8 { - static constexpr auto name = "wordShingleMinhashCaseInsensitiveUTF8"; + static constexpr auto name = "wordShingleMinHashCaseInsensitiveUTF8"; }; -// Simhash -using FunctionNgramSimhash = FunctionsStringHash, NameNgramSimhash, true>; +// SimHash +using FunctionNgramSimHash = FunctionsStringHash, NameNgramSimHash, true>; -using FunctionNgramSimhashCaseInsensitive - = FunctionsStringHash, NameNgramSimhashCaseInsensitive, true>; +using FunctionNgramSimHashCaseInsensitive + = FunctionsStringHash, NameNgramSimHashCaseInsensitive, true>; -using FunctionNgramSimhashUTF8 = FunctionsStringHash, NameNgramSimhashUTF8, true>; +using FunctionNgramSimHashUTF8 = FunctionsStringHash, NameNgramSimHashUTF8, true>; -using FunctionNgramSimhashCaseInsensitiveUTF8 - = FunctionsStringHash, NameNgramSimhashCaseInsensitiveUTF8, true>; +using FunctionNgramSimHashCaseInsensitiveUTF8 + = FunctionsStringHash, NameNgramSimHashCaseInsensitiveUTF8, true>; -using FunctionWordShingleSimhash = FunctionsStringHash, NameWordShingleSimhash, true>; +using FunctionWordShingleSimHash = FunctionsStringHash, NameWordShingleSimHash, true>; -using FunctionWordShingleSimhashCaseInsensitive - = FunctionsStringHash, NameWordShingleSimhashCaseInsensitive, true>; +using FunctionWordShingleSimHashCaseInsensitive + = FunctionsStringHash, NameWordShingleSimHashCaseInsensitive, true>; -using FunctionWordShingleSimhashUTF8 = FunctionsStringHash, NameWordShingleSimhashUTF8, true>; +using FunctionWordShingleSimHashUTF8 = FunctionsStringHash, NameWordShingleSimHashUTF8, true>; -using FunctionWordShingleSimhashCaseInsensitiveUTF8 - = FunctionsStringHash, NameWordShingleSimhashCaseInsensitiveUTF8, true>; +using FunctionWordShingleSimHashCaseInsensitiveUTF8 + = FunctionsStringHash, NameWordShingleSimHashCaseInsensitiveUTF8, true>; -// Minhash -using FunctionNgramMinhash = FunctionsStringHash, NameNgramMinhash, false>; +// MinHash +using FunctionNgramMinHash = FunctionsStringHash, NameNgramMinHash, false>; -using FunctionNgramMinhashCaseInsensitive - = FunctionsStringHash, NameNgramMinhashCaseInsensitive, false>; +using FunctionNgramMinHashCaseInsensitive + = FunctionsStringHash, NameNgramMinHashCaseInsensitive, false>; -using FunctionNgramMinhashUTF8 = FunctionsStringHash, NameNgramMinhashUTF8, false>; +using FunctionNgramMinHashUTF8 = FunctionsStringHash, NameNgramMinHashUTF8, false>; -using FunctionNgramMinhashCaseInsensitiveUTF8 - = FunctionsStringHash, NameNgramMinhashCaseInsensitiveUTF8, false>; +using FunctionNgramMinHashCaseInsensitiveUTF8 + = FunctionsStringHash, NameNgramMinHashCaseInsensitiveUTF8, false>; -using FunctionWordShingleMinhash = FunctionsStringHash, NameWordShingleMinhash, false>; +using FunctionWordShingleMinHash = FunctionsStringHash, NameWordShingleMinHash, false>; -using FunctionWordShingleMinhashCaseInsensitive - = FunctionsStringHash, NameWordShingleMinhashCaseInsensitive, false>; +using FunctionWordShingleMinHashCaseInsensitive + = FunctionsStringHash, NameWordShingleMinHashCaseInsensitive, false>; -using FunctionWordShingleMinhashUTF8 - = FunctionsStringHash, NameWordShingleMinhashUTF8, false>; +using FunctionWordShingleMinHashUTF8 + = FunctionsStringHash, NameWordShingleMinHashUTF8, false>; -using FunctionWordShingleMinhashCaseInsensitiveUTF8 - = FunctionsStringHash, NameWordShingleMinhashCaseInsensitiveUTF8, false>; +using FunctionWordShingleMinHashCaseInsensitiveUTF8 + = FunctionsStringHash, NameWordShingleMinHashCaseInsensitiveUTF8, false>; void registerFunctionsStringHash(FunctionFactory & factory) { - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); } } From 06819c187bd24a9c437fe3370f37405307a97952 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Thu, 24 Dec 2020 20:06:11 +0300 Subject: [PATCH 119/284] Update countMatches function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Поставил ';' в конце запросов. --- docs/en/sql-reference/functions/string-search-functions.md | 6 +++--- docs/ru/sql-reference/functions/string-search-functions.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index ef7e378ad11..96a2e74e1b7 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -618,7 +618,7 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Query: ``` sql -SELECT countMatches('foobar.com', 'o+') +SELECT countMatches('foobar.com', 'o+'); ``` Result: @@ -632,13 +632,13 @@ Result: Query: ``` sql -SELECT countMatches('aaaa', 'aa') +SELECT countMatches('aaaa', 'aa'); ``` Result: ``` text -┌─countMatches('aaaa', 'aa')─┐ +┌─countMatches('aaaa', 'aa')────┐ │ 2 │ └───────────────────────────────┘ ``` diff --git a/docs/ru/sql-reference/functions/string-search-functions.md b/docs/ru/sql-reference/functions/string-search-functions.md index 8bc2bb12382..996937c0d83 100644 --- a/docs/ru/sql-reference/functions/string-search-functions.md +++ b/docs/ru/sql-reference/functions/string-search-functions.md @@ -548,7 +548,7 @@ countMatches(haystack, pattern) Запрос: ``` sql -SELECT countMatches('foobar.com', 'o+') +SELECT countMatches('foobar.com', 'o+'); ``` Результат: @@ -562,13 +562,13 @@ SELECT countMatches('foobar.com', 'o+') Запрос: ``` sql -SELECT countMatches('aaaa', 'aa') +SELECT countMatches('aaaa', 'aa'); ``` Результат: ``` text -┌─countMatches('aaaa', 'aa')─┐ +┌─countMatches('aaaa', 'aa')────┐ │ 2 │ └───────────────────────────────┘ ``` From 21c3fc0e164429dfeb2e1d3d126afdddf5653efc Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 24 Dec 2020 20:48:54 +0300 Subject: [PATCH 120/284] check settings constraints in setProfile(...) --- src/Access/SettingsConstraints.cpp | 16 ---------------- src/Interpreters/Context.cpp | 12 +++++++++++- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index 7dce4d96b33..958075541c8 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -157,23 +157,7 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh const String & setting_name = change.name; if (setting_name == "profile") - { - /// TODO Check profile settings in Context::setProfile(...), not here. It will be backward incompatible. - const String & profile_name = change.value.safeGet(); - const auto & profile_settings_changes = manager->getProfileSettings(profile_name); - try - { - /// NOTE We cannot use CLAMP_ON_VIOLATION here, because we cannot modify elements of profile_settings_changes - for (auto change_copy : *profile_settings_changes) - checkImpl(current_settings, change_copy, THROW_ON_VIOLATION); - } - catch (Exception & e) - { - e.addMessage(", while trying to set settings profile {}", profile_name); - throw; - } return true; - } bool cannot_cast; auto cast_value = [&](const Field & x) -> Field diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 482bfa83632..e3a837617ec 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -849,7 +849,17 @@ std::optional Context::getQuotaUsage() const void Context::setProfile(const String & profile_name) { - applySettingsChanges(*getAccessControlManager().getProfileSettings(profile_name)); + SettingsChanges profile_settings_changes = *getAccessControlManager().getProfileSettings(profile_name); + try + { + checkSettingsConstraints(profile_settings_changes); + } + catch (Exception & e) + { + e.addMessage(", while trying to set settings profile {}", profile_name); + throw; + } + applySettingsChanges(profile_settings_changes); } From 30d38bf9c43fda94d9b70994e4decb07dc3cc96d Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Thu, 24 Dec 2020 21:04:32 +0300 Subject: [PATCH 121/284] Update union.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Переструктурировал текст. --- docs/en/sql-reference/statements/select/union.md | 6 ++++-- docs/ru/sql-reference/statements/select/union.md | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/en/sql-reference/statements/select/union.md b/docs/en/sql-reference/statements/select/union.md index 099c680d9a7..18b132bfac7 100644 --- a/docs/en/sql-reference/statements/select/union.md +++ b/docs/en/sql-reference/statements/select/union.md @@ -4,6 +4,10 @@ toc_title: UNION # UNION Clause {#union-clause} +You can use `UNION` with explicitly specifying `UNION ALL` or `UNION DISTINCT`. + +By default, `UNION` has the same behavior as `UNION DISTINCT`. The difference between `UNION ALL` and `UNION DISTINCT` is that `UNION DISTINCT` will do a distinct transform for union result, it is equivalent to `SELECT DISTINCT` from a subquery containing `UNION ALL`. + You can use `UNION ALL` to combine any number of `SELECT` queries by extending their results. Example: ``` sql @@ -25,8 +29,6 @@ Type casting is performed for unions. For example, if two queries being combined Queries that are parts of `UNION ALL` can’t be enclosed in round brackets. [ORDER BY](../../../sql-reference/statements/select/order-by.md) and [LIMIT](../../../sql-reference/statements/select/limit.md) are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with `UNION ALL` in a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause. -By default, `UNION` has the same behavior as `UNION DISTINCT`. The difference between `UNION ALL` and `UNION DISTINCT` is that `UNION DISTINCT` will do a distinct transform for union result, it is equivalent to `SELECT DISTINCT` from a subquery containing `UNION ALL`. - If you use `UNION` without explicitly specifying `UNION ALL` or `UNION DISTINCT`, you can specify the union mode using the [union_default_mode](../../../operations/settings/settings.md#union-default-mode) setting. The setting values can be `ALL`, `DISTINCT` or an empty string. However, if you use `UNION` with `union_default_mode` setting to empty string, it will throw an exception. The following examples demonstrate the results of queries with different values setting. Query: diff --git a/docs/ru/sql-reference/statements/select/union.md b/docs/ru/sql-reference/statements/select/union.md index 4d7d96a23db..2bde7ce9da0 100644 --- a/docs/ru/sql-reference/statements/select/union.md +++ b/docs/ru/sql-reference/statements/select/union.md @@ -4,6 +4,10 @@ toc_title: UNION # Секция UNION {#union-clause} +Вы можете использовать `UNION` в двух режимах: `UNION ALL` или `UNION DISTINCT`. + +По умолчанию, `UNION` ведет себя так же, как и `UNION DISTINCT`. Разница между `UNION ALL` и `UNION DISTINCT` в том, что `UNION DISTINCT` выполняет явное преобразование для результата объединения. Это равнозначно выражению `SELECT DISTINCT` из подзапроса, содержащего `UNION ALL`. + Вы можете использовать `UNION ALL` чтобы объединить любое количество `SELECT` запросы путем расширения их результатов. Пример: ``` sql @@ -25,8 +29,6 @@ SELECT CounterID, 2 AS table, sum(Sign) AS c Запросы, которые являются частью `UNION ALL` не могут быть заключен в круглые скобки. [ORDER BY](order-by.md) и [LIMIT](limit.md) применяются к отдельным запросам, а не к конечному результату. Если вам нужно применить преобразование к конечному результату, вы можете разместить все объединенные с помощью `UNION ALL` запросы в подзапрос в секции [FROM](from.md). -По умолчанию, `UNION` ведет себя так же, как и `UNION DISTINCT`. Разница между `UNION ALL` и `UNION DISTINCT` в том, что `UNION DISTINCT` выполняет явное преобразование для результата объединения. Это равнозначно выражению `SELECT DISTINCT` из подзапроса, содержащего `UNION ALL`. - Если используете `UNION` без явного указания `UNION ALL` или `UNION DISTINCT`, то вы можете указать режим объединения с помощью настройки [union_default_mode](../../../operations/settings/settings.md#union-default-mode), значениями которой могут быть `ALL`, `DISTINCT` или пустая строка. Однако если вы используете `UNION` с настройкой `union_default_mode`, значением которой является пустая строка, то будет сгенерировано исключение. В следующих примерах продемонстрированы результаты запросов при разных значениях настройки. Запрос: @@ -74,6 +76,6 @@ SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; └───┘ ``` -Запросы, которые являются частью `UNION ALL`, выполняются параллельно, и их результаты могут быть смешаны вместе. +Запросы, которые являются частью `UNION/UNION ALL/UNION DISTINCT`, выполняются параллельно, и их результаты могут быть смешаны вместе. [Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/select/union/) From e8f4a19a100eb3321172cb9bc8a72fb99c1f943f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 24 Dec 2020 21:57:31 +0300 Subject: [PATCH 122/284] Feractoring StringHash --- src/Functions/ExtractString.h | 68 ++++++++++------------------------- 1 file changed, 18 insertions(+), 50 deletions(-) diff --git a/src/Functions/ExtractString.h b/src/Functions/ExtractString.h index 8313f583025..7406a813143 100644 --- a/src/Functions/ExtractString.h +++ b/src/Functions/ExtractString.h @@ -15,47 +15,33 @@ namespace DB { + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + // used by FunctionsStringSimilarity and FunctionsStringHash // includes extracting ASCII ngram, UTF8 ngram, ASCII word and UTF8 word -template +template struct ExtractStringImpl { /// Padding form ColumnsString. It is a number of bytes we can always read starting from pos if pos < end. static constexpr size_t default_padding = 16; + const size_t shingle_size; + const size_t tail_size; + /// Functions are read `default_padding - (N - 1)` bytes into the buffer. Window of size N is used. /// Read copies `N - 1` last bytes from buffer into beginning, and then reads new bytes. - static constexpr size_t buffer_size = default_padding + N - 1; + const size_t buffer_size = default_padding + tail_size; - // the length of code_points = buffer_size - // pos: the current beginning location that we want to copy data - // end: the end location of the string - static ALWAYS_INLINE size_t readASCIICodePoints(UInt8 * code_points, const char *& pos, const char * end) + explicit ExtractStringImpl(size_t shingle_size_) + : shingle_size(shingle_size_) + , tail_size(shingle_size > default_padding ? shingle_size : roundUpToPowerOfTwoOrZero(shingle_size - 1)) { - /// Offset before which we copy some data. - constexpr size_t padding_offset = default_padding - N + 1; - /// We have an array like this for ASCII (N == 4, other cases are similar) - /// |a0|a1|a2|a3|a4|a5|a6|a7|a8|a9|a10|a11|a12|a13|a14|a15|a16|a17|a18| - /// And we copy ^^^^^^^^^^^^^^^ these bytes to the start - /// Actually it is enough to copy 3 bytes, but memcpy for 4 bytes translates into 1 instruction - memcpy(code_points, code_points + padding_offset, roundUpToPowerOfTwoOrZero(N - 1) * sizeof(UInt8)); - /// Now we have an array - /// |a13|a14|a15|a16|a4|a5|a6|a7|a8|a9|a10|a11|a12|a13|a14|a15|a16|a17|a18| - /// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - /// Doing unaligned read of 16 bytes and copy them like above - /// 16 is also chosen to do two `movups`. - /// Such copying allow us to have 3 codepoints from the previous read to produce the 4-grams with them. - memcpy(code_points + (N - 1), pos, default_padding * sizeof(UInt8)); - - if constexpr (CaseInsensitive) - { - /// We really need template lambdas with C++20 to do it inline - unrollLowering(code_points, std::make_index_sequence()); - } - pos += padding_offset; - if (pos > end) - return default_padding - (pos - end); - return default_padding; + if (shingle_size == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Shingle size can't be zero"); } // read a ASCII word @@ -73,23 +59,12 @@ struct ExtractStringImpl word_buf.assign(word_start, pos); if (CaseInsensitive) { - std::transform(word_buf.begin(), word_buf.end(), word_buf.begin(), [](UInt8 c) { return std::tolower(c); }); + for (auto & symbol : word_buf) + symbol = toLowerIfAlphaASCII(symbol); } return word_buf.size(); } - static ALWAYS_INLINE inline size_t readUTF8CodePoints(UInt32 * code_points, const char *& pos, const char * end) - { - memcpy(code_points, code_points + default_padding - N + 1, roundUpToPowerOfTwoOrZero(N - 1) * sizeof(UInt32)); - - size_t num = N - 1; - while (num < default_padding && pos < end) - { - code_points[num++] = readOneUTF8Code(pos, end); - } - return num; - } - // read one UTF8 word from pos to word static ALWAYS_INLINE inline size_t readOneUTF8Word(PaddedPODArray & word_buf, const char *& pos, const char * end) { @@ -105,13 +80,6 @@ struct ExtractStringImpl return word_buf.size(); } -private: - template - static ALWAYS_INLINE inline void unrollLowering(Container & cont, const std::index_sequence &) - { - ((cont[Offset + I] = std::tolower(cont[Offset + I])), ...); - } - // we use ASCII non-alphanum character as UTF8 separator static ALWAYS_INLINE inline bool isUTF8Sep(const UInt8 c) { return c < 128 && !isAlphaNumericASCII(c); } From 1e3bd37380a1331727fa93443d20835c5cbc2a6f Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 24 Dec 2020 07:31:09 +0000 Subject: [PATCH 123/284] Add option access_to_key_from_attributes to ip dictionary --- src/Dictionaries/IPAddressDictionary.cpp | 84 +++++++++++++++---- src/Dictionaries/IPAddressDictionary.h | 9 +- .../0_stateless/01018_ip_dictionary.reference | 15 ++++ .../0_stateless/01018_ip_dictionary.sql | 21 ++++- 4 files changed, 108 insertions(+), 21 deletions(-) diff --git a/src/Dictionaries/IPAddressDictionary.cpp b/src/Dictionaries/IPAddressDictionary.cpp index e04b9111ff1..a69e038c1e4 100644 --- a/src/Dictionaries/IPAddressDictionary.cpp +++ b/src/Dictionaries/IPAddressDictionary.cpp @@ -107,6 +107,18 @@ static std::pair parseIPFromString(const std::strin } } +static size_t formatIPWithPrefix(const unsigned char * src, UInt8 prefix_len, bool isv4, char * dst) +{ + char * ptr = dst; + if (isv4) + formatIPv4(src, ptr); + else + formatIPv6(src, ptr); + *(ptr - 1) = '/'; + ptr = itoa(prefix_len, ptr); + return ptr - dst; +} + static void validateKeyTypes(const DataTypes & key_types) { if (key_types.empty() || key_types.size() > 2) @@ -231,14 +243,21 @@ IPAddressDictionary::IPAddressDictionary( const DictionaryStructure & dict_struct_, DictionarySourcePtr source_ptr_, const DictionaryLifetime dict_lifetime_, - bool require_nonempty_) + bool require_nonempty_, + bool access_to_key_from_attributes_) : IDictionaryBase(dict_id_) , dict_struct(dict_struct_) , source_ptr{std::move(source_ptr_)} , dict_lifetime(dict_lifetime_) , require_nonempty(require_nonempty_) + , access_to_key_from_attributes(access_to_key_from_attributes_) , logger(&Poco::Logger::get("IPAddressDictionary")) { + if (access_to_key_from_attributes) + { + dict_struct.attributes.emplace_back(dict_struct.key->front()); + } + createAttributes(); loadData(); @@ -453,8 +472,6 @@ void IPAddressDictionary::loadData() auto stream = source_ptr->loadAll(); stream->readPrefix(); - const auto attributes_size = attributes.size(); - std::vector ip_records; bool has_ipv6 = false; @@ -465,14 +482,19 @@ void IPAddressDictionary::loadData() element_count += rows; const ColumnPtr key_column_ptr = block.safeGetByPosition(0).column; - const auto attribute_column_ptrs = ext::map(ext::range(0, attributes_size), [&](const size_t attribute_idx) + + size_t attributes_size = dict_struct.attributes.size(); + if (access_to_key_from_attributes) { - return block.safeGetByPosition(attribute_idx + 1).column; - }); + /// last attribute contains key and will be filled in code below + attributes_size--; + } + const auto attribute_column_ptrs = ext::map(ext::range(0, attributes_size), + [&](const size_t attribute_idx) { return block.safeGetByPosition(attribute_idx + 1).column; }); for (const auto row : ext::range(0, rows)) { - for (const auto attribute_idx : ext::range(0, attributes_size)) + for (const auto attribute_idx : ext::range(0, attribute_column_ptrs.size())) { const auto & attribute_column = *attribute_column_ptrs[attribute_idx]; auto & attribute = attributes[attribute_idx]; @@ -490,6 +512,33 @@ void IPAddressDictionary::loadData() stream->readSuffix(); + if (access_to_key_from_attributes) + { + /// We format key attribute values here instead of filling with data from key_column + /// because string representation can be normalized if bits beyond mask are set. + /// Also all IPv4 will be displayed as mapped IPv6 if threre are any IPv6. + /// It's consistent with representation in table created with `ENGINE = Dictionary` from this dictionary. + char str_buffer[48]; + if (has_ipv6) + { + uint8_t ip_buffer[IPV6_BINARY_LENGTH]; + for (const auto & record : ip_records) + { + size_t str_len = formatIPWithPrefix(record.asIPv6Binary(ip_buffer), record.prefixIPv6(), false, str_buffer); + setAttributeValue(attributes.back(), String(str_buffer, str_len)); + } + } + else + { + for (const auto & record : ip_records) + { + UInt32 addr = IPv4AsUInt32(record.addr.addr()); + size_t str_len = formatIPWithPrefix(reinterpret_cast(&addr), record.prefix, true, str_buffer); + setAttributeValue(attributes.back(), String(str_buffer, str_len)); + } + } + } + row_idx.reserve(ip_records.size()); mask_column.reserve(ip_records.size()); @@ -679,7 +728,7 @@ void IPAddressDictionary::calculateBytesAllocated() template void IPAddressDictionary::createAttributeImpl(Attribute & attribute, const Field & null_value) { - attribute.null_values = T(null_value.get>()); + attribute.null_values = null_value.isNull() ? T{} : T(null_value.get>()); attribute.maps.emplace>(); } @@ -735,7 +784,8 @@ IPAddressDictionary::Attribute IPAddressDictionary::createAttributeWithType(cons case AttributeUnderlyingType::utString: { - attr.null_values = null_value.get(); + + attr.null_values = null_value.isNull() ? String() : null_value.get(); attr.maps.emplace>(); attr.string_arena = std::make_unique(); break; @@ -979,14 +1029,12 @@ static auto keyViewGetter() for (size_t row : ext::range(0, key_ip_column.size())) { UInt8 mask = key_mask_column.getElement(row); - char * ptr = buffer; + size_t str_len; if constexpr (IsIPv4) - formatIPv4(reinterpret_cast(&key_ip_column.getElement(row)), ptr); + str_len = formatIPWithPrefix(reinterpret_cast(&key_ip_column.getElement(row)), mask, true, buffer); else - formatIPv6(reinterpret_cast(key_ip_column.getDataAt(row).data), ptr); - *(ptr - 1) = '/'; - ptr = itoa(mask, ptr); - column->insertData(buffer, ptr - buffer); + str_len = formatIPWithPrefix(reinterpret_cast(key_ip_column.getDataAt(row).data), mask, false, buffer); + column->insertData(buffer, str_len); } return ColumnsWithTypeAndName{ ColumnWithTypeAndName(std::move(column), std::make_shared(), dict_attributes.front().name)}; @@ -1120,8 +1168,12 @@ void registerDictionaryTrie(DictionaryFactory & factory) const auto dict_id = StorageID::fromDictionaryConfig(config, config_prefix); const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false); + + const auto & layout_prefix = config_prefix + ".layout.ip_trie"; + const bool access_to_key_from_attributes = config.getBool(layout_prefix + ".access_to_key_from_attributes", false); // This is specialised dictionary for storing IPv4 and IPv6 prefixes. - return std::make_unique(dict_id, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty); + return std::make_unique(dict_id, dict_struct, std::move(source_ptr), dict_lifetime, + require_nonempty, access_to_key_from_attributes); }; factory.registerLayout("ip_trie", create_layout, true); } diff --git a/src/Dictionaries/IPAddressDictionary.h b/src/Dictionaries/IPAddressDictionary.h index c63e6b01adf..2009141ebcc 100644 --- a/src/Dictionaries/IPAddressDictionary.h +++ b/src/Dictionaries/IPAddressDictionary.h @@ -27,7 +27,8 @@ public: const DictionaryStructure & dict_struct_, DictionarySourcePtr source_ptr_, const DictionaryLifetime dict_lifetime_, - bool require_nonempty_); + bool require_nonempty_, + bool access_to_key_from_attributes_); std::string getKeyDescription() const { return key_description; } @@ -45,7 +46,8 @@ public: std::shared_ptr clone() const override { - return std::make_shared(getDictionaryID(), dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty); + return std::make_shared(getDictionaryID(), dict_struct, source_ptr->clone(), dict_lifetime, + require_nonempty, access_to_key_from_attributes); } const IDictionarySource * getSource() const override { return source_ptr.get(); } @@ -238,10 +240,11 @@ private: static const uint8_t * getIPv6FromOffset(const IPv6Container & ipv6_col, size_t i); - const DictionaryStructure dict_struct; + DictionaryStructure dict_struct; const DictionarySourcePtr source_ptr; const DictionaryLifetime dict_lifetime; const bool require_nonempty; + const bool access_to_key_from_attributes; const std::string key_description{dict_struct.getKeyDescription()}; /// Contains sorted IP subnetworks. If some addresses equals, subnet with lower mask is placed first. diff --git a/tests/queries/0_stateless/01018_ip_dictionary.reference b/tests/queries/0_stateless/01018_ip_dictionary.reference index becd2951be7..f11628001dd 100644 --- a/tests/queries/0_stateless/01018_ip_dictionary.reference +++ b/tests/queries/0_stateless/01018_ip_dictionary.reference @@ -123,6 +123,16 @@ 1 1 1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 ***ipv6 trie dict*** 1 1 @@ -273,6 +283,11 @@ 1 1 1 +1 +1 +1 +1 +1 ***ipv6 trie dict mask*** 1 1 diff --git a/tests/queries/0_stateless/01018_ip_dictionary.sql b/tests/queries/0_stateless/01018_ip_dictionary.sql index 4cb56b77180..5df1afcd559 100644 --- a/tests/queries/0_stateless/01018_ip_dictionary.sql +++ b/tests/queries/0_stateless/01018_ip_dictionary.sql @@ -207,9 +207,20 @@ INSERT INTO database_for_dict.table_ipv4_trie VALUES ('127.255.255.255/32', 21); CREATE DICTIONARY database_for_dict.dict_ipv4_trie ( prefix String, val UInt32 ) PRIMARY KEY prefix SOURCE(CLICKHOUSE(host 'localhost' port 9000 user 'default' db 'database_for_dict' table 'table_ipv4_trie')) -LAYOUT(IP_TRIE()) +LAYOUT(IP_TRIE(ACCESS_TO_KEY_FROM_ATTRIBUTES 1)) LIFETIME(MIN 10 MAX 100); +SELECT '127.0.0.0/24' == dictGetString('database_for_dict.dict_ipv4_trie', 'prefix', tuple(IPv4StringToNum('127.0.0.0'))); +SELECT '127.0.0.1/32' == dictGetString('database_for_dict.dict_ipv4_trie', 'prefix', tuple(IPv4StringToNum('127.0.0.1'))); +SELECT '127.0.0.0/24' == dictGetString('database_for_dict.dict_ipv4_trie', 'prefix', tuple(IPv4StringToNum('127.0.0.127'))); +SELECT '127.0.0.0/16' == dictGetString('database_for_dict.dict_ipv4_trie', 'prefix', tuple(IPv4StringToNum('127.0.255.127'))); +SELECT '127.255.0.0/16' == dictGetString('database_for_dict.dict_ipv4_trie', 'prefix', tuple(IPv4StringToNum('127.255.127.127'))); +SELECT '127.255.128.0/24' == dictGetString('database_for_dict.dict_ipv4_trie', 'prefix', tuple(IPv4StringToNum('127.255.128.9'))); +SELECT '127.255.128.0/24' == dictGetString('database_for_dict.dict_ipv4_trie', 'prefix', tuple(IPv4StringToNum('127.255.128.127'))); +SELECT '127.255.128.10/32' == dictGetString('database_for_dict.dict_ipv4_trie', 'prefix', tuple(IPv4StringToNum('127.255.128.10'))); +SELECT '127.255.128.128/25' == dictGetString('database_for_dict.dict_ipv4_trie', 'prefix', tuple(IPv4StringToNum('127.255.128.255'))); +SELECT '127.255.255.128/32' == dictGetString('database_for_dict.dict_ipv4_trie', 'prefix', tuple(IPv4StringToNum('127.255.255.128'))); + SELECT 3 == dictGetUInt32('database_for_dict.dict_ipv4_trie', 'val', tuple(IPv4StringToNum('127.0.0.0'))); SELECT 4 == dictGetUInt32('database_for_dict.dict_ipv4_trie', 'val', tuple(IPv4StringToNum('127.0.0.1'))); SELECT 3 == dictGetUInt32('database_for_dict.dict_ipv4_trie', 'val', tuple(IPv4StringToNum('127.0.0.127'))); @@ -274,7 +285,7 @@ CREATE DICTIONARY database_for_dict.dict_ip_trie ) PRIMARY KEY prefix SOURCE(CLICKHOUSE(host 'localhost' port 9000 user 'default' db 'database_for_dict' table 'table_ip_trie')) -LAYOUT(IP_TRIE()) +LAYOUT(IP_TRIE(ACCESS_TO_KEY_FROM_ATTRIBUTES 1)) LIFETIME(MIN 10 MAX 100); SELECT 'US' == dictGetString('database_for_dict.dict_ip_trie', 'val', tuple(IPv6StringToNum('2620:0:870::'))); @@ -294,6 +305,12 @@ SELECT 'JA' == dictGetString('database_for_dict.dict_ip_trie', 'val', tuple(IPv4 SELECT 1 == dictHas('database_for_dict.dict_ip_trie', tuple(IPv4StringToNum('127.0.0.1'))); SELECT 1 == dictHas('database_for_dict.dict_ip_trie', tuple(IPv6StringToNum('::ffff:127.0.0.1'))); +SELECT '2620:0:870::/48' == dictGetString('database_for_dict.dict_ip_trie', 'prefix', tuple(IPv6StringToNum('2620:0:870::'))); +SELECT '2a02:6b8:1::/48' == dictGetString('database_for_dict.dict_ip_trie', 'prefix', tuple(IPv6StringToNum('2a02:6b8:1::1'))); +SELECT '2001:db8::/32' == dictGetString('database_for_dict.dict_ip_trie', 'prefix', tuple(IPv6StringToNum('2001:db8::1'))); +SELECT '::ffff:101.79.55.22/128' == dictGetString('database_for_dict.dict_ip_trie', 'prefix', tuple(IPv6StringToNum('::ffff:654f:3716'))); +SELECT '::ffff:101.79.55.22/128' == dictGetString('database_for_dict.dict_ip_trie', 'prefix', tuple(IPv6StringToNum('::ffff:101.79.55.22'))); + SELECT '0' == dictGetString('database_for_dict.dict_ip_trie', 'val', tuple(IPv6StringToNum('::0'))); SELECT '1' == dictGetString('database_for_dict.dict_ip_trie', 'val', tuple(IPv6StringToNum('8000::'))); SELECT '2' == dictGetString('database_for_dict.dict_ip_trie', 'val', tuple(IPv6StringToNum('c000::'))); From fcfe7f2c5d6481b44bc45a019fb4224cb4c47253 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 24 Dec 2020 15:46:45 +0000 Subject: [PATCH 124/284] Add option access_to_key_from_attributes to documentation for ip_trie --- .../external-dicts-dict-layout.md | 10 +++++++++- .../external-dicts-dict-layout.md | 8 +++++++- .../external-dicts-dict-layout.md | 6 ++++++ .../external-dicts-dict-layout.md | 10 +++++++++- .../external-dicts-dict-layout.md | 6 ++++++ 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index a49ddf62eb9..05c418b1f15 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -406,6 +406,14 @@ Example: ?? ... + + + + + + true + + ``` or @@ -435,6 +443,6 @@ dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1'))) Other types are not supported yet. The function returns the attribute for the prefix that corresponds to this IP address. If there are overlapping prefixes, the most specific one is returned. -Data is stored in a `trie`. It must completely fit into RAM. +Data must completely fit into RAM. [Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_layout/) diff --git a/docs/fr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/fr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index 3bdc8d37d60..2569329fefd 100644 --- a/docs/fr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/fr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -367,6 +367,12 @@ Exemple: ?? ... + + + + true + + ``` ou @@ -396,6 +402,6 @@ dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1'))) Les autres types ne sont pas encore pris en charge. La fonction renvoie l'attribut du préfixe correspondant à cette adresse IP. S'il y a chevauchement des préfixes, le plus spécifique est retourné. -Les données sont stockées dans une `trie`. Il doit complètement s'intégrer dans la RAM. +Les données doit complètement s'intégrer dans la RAM. [Article Original](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_layout/) diff --git a/docs/ja/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/ja/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index 167e0f1f5d1..ac632b99332 100644 --- a/docs/ja/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/ja/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -362,6 +362,12 @@ LAYOUT(DIRECT()) ?? ... + + + + true + + ``` または diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index e97b1e421a4..fc4a3ac7285 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -404,6 +404,14 @@ LAYOUT(DIRECT()) ?? ... + + + + + + true + + ``` или @@ -433,6 +441,6 @@ dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1'))) Никакие другие типы не поддерживаются. Функция возвращает атрибут для префикса, соответствующего данному IP-адресу. Если есть перекрывающиеся префиксы, возвращается наиболее специфический. -Данные хранятся в побитовом дереве (`trie`), он должен полностью помещаться в оперативной памяти. +Данные должны полностью помещаться в оперативной памяти. [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_layout/) diff --git a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index 176859d3633..190fd74d705 100644 --- a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -362,6 +362,12 @@ LAYOUT(DIRECT()) ?? ... + + + + true + + ``` 或 From 3ce073163053abc99b8cf6180139e4a4a63290c5 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 25 Dec 2020 01:49:19 +0400 Subject: [PATCH 125/284] Move caching and LDAP credential verification code to ExternalAuthenticators --- src/Access/Authentication.cpp | 57 +---------- src/Access/Authentication.h | 5 +- src/Access/ExternalAuthenticators.cpp | 137 ++++++++++++++++++++------ src/Access/ExternalAuthenticators.h | 19 +++- src/Access/IAccessStorage.cpp | 2 +- src/Access/User.cpp | 16 --- src/Access/User.h | 19 ---- 7 files changed, 129 insertions(+), 126 deletions(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 2455869f783..5fab2c92624 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -1,7 +1,5 @@ #include -#include #include -#include #include #include @@ -50,7 +48,7 @@ Authentication::Digest Authentication::getPasswordDoubleSHA1() const } -bool Authentication::isCorrectPassword(const User & user_, const String & password_, const ExternalAuthenticators & external_authenticators) const +bool Authentication::isCorrectPassword(const String & user_, const String & password_, const ExternalAuthenticators & external_authenticators) const { switch (type) { @@ -82,58 +80,7 @@ bool Authentication::isCorrectPassword(const User & user_, const String & passwo } case LDAP_SERVER: - { - auto ldap_server_params = external_authenticators.getLDAPServerParams(server_name); - ldap_server_params.user = user_.getName(); - ldap_server_params.password = password_; - - const auto current_params_hash = ldap_server_params.getCoreHash(); - auto & ldap_last_successful_password_check_params_hash = user_.cache.ldap_last_successful_password_check_params_hash; - auto & ldap_last_successful_password_check_timestamp = user_.cache.ldap_last_successful_password_check_timestamp; - - { - std::scoped_lock lock(user_.cache.mutex); - const auto last_check_period = std::chrono::steady_clock::now() - ldap_last_successful_password_check_timestamp; - - if ( - // Check if the caching is enabled at all. - ldap_server_params.verification_cooldown > std::chrono::seconds{0} && - - // Forbid the initial values explicitly. - ldap_last_successful_password_check_params_hash != 0 && - ldap_last_successful_password_check_timestamp != std::chrono::steady_clock::time_point{} && - - // Check if we can "reuse" the result of the previous successful password verification. - current_params_hash == ldap_last_successful_password_check_params_hash && - last_check_period >= std::chrono::seconds{0} && - last_check_period <= ldap_server_params.verification_cooldown - ) - { - return true; - } - } - - LDAPSimpleAuthClient ldap_client(ldap_server_params); - const auto result = ldap_client.check(); - const auto current_check_timestamp = std::chrono::steady_clock::now(); - - { - std::scoped_lock lock(user_.cache.mutex); - - if (result) - { - ldap_last_successful_password_check_params_hash = current_params_hash; - ldap_last_successful_password_check_timestamp = current_check_timestamp; - } - else - { - ldap_last_successful_password_check_params_hash = 0; - ldap_last_successful_password_check_timestamp = std::chrono::steady_clock::time_point{}; - } - } - - return result; - } + return external_authenticators.checkLDAPCredentials(server_name, user_, password_); case MAX_TYPE: break; diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h index abbd43555b1..e31e36531c3 100644 --- a/src/Access/Authentication.h +++ b/src/Access/Authentication.h @@ -19,7 +19,6 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; } -struct User; class ExternalAuthenticators; /// Authentication type and encrypted password for checking when an user logins. @@ -90,8 +89,8 @@ public: void setServerName(const String & server_name_); /// Checks if the provided password is correct. Returns false if not. - /// User instance and external authenticators' info are used only by some specific authentication type (e.g., LDAP_SERVER). - bool isCorrectPassword(const User & user_, const String & password_, const ExternalAuthenticators & external_authenticators) const; + /// User name and external authenticators are used by specific authentication types only (e.g., LDAP_SERVER). + bool isCorrectPassword(const String & user_, const String & password_, const ExternalAuthenticators & external_authenticators) const; friend bool operator ==(const Authentication & lhs, const Authentication & rhs) { return (lhs.type == rhs.type) && (lhs.password_hash == rhs.password_hash); } friend bool operator !=(const Authentication & lhs, const Authentication & rhs) { return !(lhs == rhs); } diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index 9b908841576..71cc1c11020 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -1,9 +1,13 @@ #include +#include #include #include #include #include +#include +#include + namespace DB { @@ -134,16 +138,28 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str return params; } -void parseAndAddLDAPServers(ExternalAuthenticators & external_authenticators, const Poco::Util::AbstractConfiguration & config, Poco::Logger * log) +} + +void ExternalAuthenticators::reset() { + std::scoped_lock lock(mutex); + ldap_server_params.clear(); + ldap_server_caches.clear(); +} + +void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log) +{ + std::scoped_lock lock(mutex); + + reset(); + Poco::Util::AbstractConfiguration::Keys ldap_server_names; config.keys("ldap_servers", ldap_server_names); - for (const auto & ldap_server_name : ldap_server_names) { try { - external_authenticators.setLDAPServerParams(ldap_server_name, parseLDAPServer(config, ldap_server_name)); + ldap_server_params.insert_or_assign(ldap_server_name, parseLDAPServer(config, ldap_server_name)); } catch (...) { @@ -152,35 +168,100 @@ void parseAndAddLDAPServers(ExternalAuthenticators & external_authenticators, co } } -} - -void ExternalAuthenticators::reset() +bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const String & user_name, const String & password) const { - std::scoped_lock lock(mutex); - ldap_server_params.clear(); -} + std::optional params; + const auto password_hash = std::hash{}(password); -void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log) -{ - std::scoped_lock lock(mutex); - reset(); - parseAndAddLDAPServers(*this, config, log); -} + { + std::scoped_lock lock(mutex); -void ExternalAuthenticators::setLDAPServerParams(const String & server, const LDAPServerParams & params) -{ - std::scoped_lock lock(mutex); - ldap_server_params.erase(server); - ldap_server_params[server] = params; -} + // Retrieve the server parameters. + const auto pit = ldap_server_params.find(server); + if (pit == ldap_server_params.end()) + throw Exception("LDAP server '" + server + "' is not configured", ErrorCodes::BAD_ARGUMENTS); -LDAPServerParams ExternalAuthenticators::getLDAPServerParams(const String & server) const -{ - std::scoped_lock lock(mutex); - auto it = ldap_server_params.find(server); - if (it == ldap_server_params.end()) - throw Exception("LDAP server '" + server + "' is not configured", ErrorCodes::BAD_ARGUMENTS); - return it->second; + params = pit->second; + params->user = user_name; + params->password = password; + + // Check the cache, but only if the caching is enabled at all. + if (params->verification_cooldown > std::chrono::seconds{0}) + { + const auto cit = ldap_server_caches.find(server); + if (cit != ldap_server_caches.end()) + { + auto & cache = cit->second; + + const auto eit = cache.find(user_name); + if (eit != cache.end()) + { + const auto & entry = eit->second; + const auto last_check_period = std::chrono::steady_clock::now() - entry.last_successful_password_check_timestamp; + + if ( + // Forbid the initial values explicitly. + entry.last_successful_password_hash != 0 && + entry.last_successful_password_check_timestamp != std::chrono::steady_clock::time_point{} && + + // Check if we can safely "reuse" the result of the previous successful password verification. + password_hash == entry.last_successful_password_hash && + last_check_period >= std::chrono::seconds{0} && + last_check_period <= params->verification_cooldown + ) + { + return true; + } + + // Erase the entry, if expired. + if (last_check_period > params->verification_cooldown) + cache.erase(eit); + } + + // Erase the cache, if empty. + if (cache.empty()) + ldap_server_caches.erase(cit); + } + } + } + + LDAPSimpleAuthClient client(params.value()); + const auto result = client.check(); + const auto current_check_timestamp = std::chrono::steady_clock::now(); + + // Update the cache, but only if this is the latest check and the server is still configured in the same way. + if (result) + { + std::scoped_lock lock(mutex); + + // If the server was removed from the config while we were checking the password, we discard this result. + const auto pit = ldap_server_params.find(server); + if (pit == ldap_server_params.end()) + return false; + + auto new_params = pit->second; + new_params.user = user_name; + new_params.password = password; + + // If the critical server params have changed while we were checking the password, we discard the result. + if (params->getCoreHash() != new_params.getCoreHash()) + return false; + + auto & entry = ldap_server_caches[server][user_name]; + + if (entry.last_successful_password_check_timestamp < current_check_timestamp) + { + entry.last_successful_password_hash = password_hash; + entry.last_successful_password_check_timestamp = current_check_timestamp; + } + else if (entry.last_successful_password_hash != password_hash) + { + // Somehow a newer check succeeded but with other password, so this one is obsolete and we discard it. + return false; + } + } + + return result; } } diff --git a/src/Access/ExternalAuthenticators.h b/src/Access/ExternalAuthenticators.h index 7502409d817..16a69ec7709 100644 --- a/src/Access/ExternalAuthenticators.h +++ b/src/Access/ExternalAuthenticators.h @@ -3,9 +3,10 @@ #include #include +#include #include -#include #include +#include namespace Poco @@ -27,13 +28,23 @@ class ExternalAuthenticators public: void reset(); void setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log); + bool checkLDAPCredentials(const String & server, const String & user_name, const String & password) const; - void setLDAPServerParams(const String & server, const LDAPServerParams & params); - LDAPServerParams getLDAPServerParams(const String & server) const; +private: + struct LDAPCacheEntry + { + std::size_t last_successful_password_hash = 0; + std::chrono::steady_clock::time_point last_successful_password_check_timestamp; + }; + + using LDAPServerCache = std::unordered_map; // user name -> cache entry + using LDAPServerCaches = std::map; // server name -> cache + using LDAPServersParams = std::map; // server name -> params private: mutable std::recursive_mutex mutex; - std::map ldap_server_params; + LDAPServersParams ldap_server_params; + mutable LDAPServerCaches ldap_server_caches; }; } diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 01516017d89..c68f5f55ef5 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -463,7 +463,7 @@ UUID IAccessStorage::loginImpl( bool IAccessStorage::isPasswordCorrectImpl(const User & user, const String & password, const ExternalAuthenticators & external_authenticators) const { - return user.authentication.isCorrectPassword(user, password, external_authenticators); + return user.authentication.isCorrectPassword(user.getName(), password, external_authenticators); } diff --git a/src/Access/User.cpp b/src/Access/User.cpp index 9b705fbdb5e..f57ec7c1359 100644 --- a/src/Access/User.cpp +++ b/src/Access/User.cpp @@ -14,20 +14,4 @@ bool User::equal(const IAccessEntity & other) const && (settings == other_user.settings); } -UserEtcCache & UserEtcCache::operator= (const UserEtcCache & other) -{ - std::scoped_lock lock(mutex, other.mutex); - ldap_last_successful_password_check_params_hash = other.ldap_last_successful_password_check_params_hash; - ldap_last_successful_password_check_timestamp = other.ldap_last_successful_password_check_timestamp; - return *this; -} - -UserEtcCache & UserEtcCache::operator= (UserEtcCache && other) -{ - std::scoped_lock lock(mutex, other.mutex); - ldap_last_successful_password_check_params_hash = std::move(other.ldap_last_successful_password_check_params_hash); - ldap_last_successful_password_check_timestamp = std::move(other.ldap_last_successful_password_check_timestamp); - return *this; -} - } diff --git a/src/Access/User.h b/src/Access/User.h index ed3366b4712..13f1e532015 100644 --- a/src/Access/User.h +++ b/src/Access/User.h @@ -8,27 +8,9 @@ #include #include -#include -#include namespace DB { - -/** Various cached data bound to a User instance. Access to any member must be synchronized via 'mutex' member. - */ -struct UserEtcCache -{ - mutable std::recursive_mutex mutex; - std::size_t ldap_last_successful_password_check_params_hash = 0; - std::chrono::steady_clock::time_point ldap_last_successful_password_check_timestamp; - - explicit UserEtcCache() = default; - explicit UserEtcCache(const UserEtcCache & other) { (*this) = other; } - explicit UserEtcCache(UserEtcCache && other) { (*this) = std::move(other); } - UserEtcCache & operator= (const UserEtcCache & other); - UserEtcCache & operator= (UserEtcCache && other); -}; - /** User and ACL. */ struct User : public IAccessEntity @@ -39,7 +21,6 @@ struct User : public IAccessEntity GrantedRoles granted_roles; RolesOrUsersSet default_roles = RolesOrUsersSet::AllTag{}; SettingsProfileElements settings; - mutable UserEtcCache cache; bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } From b1e46ccef82d8bffe5eead8a10844953f9211264 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 25 Dec 2020 01:57:13 +0400 Subject: [PATCH 126/284] Remove unneeded include --- src/Access/Authentication.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h index e31e36531c3..f98d2ed4679 100644 --- a/src/Access/Authentication.h +++ b/src/Access/Authentication.h @@ -6,7 +6,6 @@ #include #include #include -#include namespace DB @@ -89,7 +88,7 @@ public: void setServerName(const String & server_name_); /// Checks if the provided password is correct. Returns false if not. - /// User name and external authenticators are used by specific authentication types only (e.g., LDAP_SERVER). + /// User name and external authenticators are used by the specific authentication types only (e.g., LDAP_SERVER). bool isCorrectPassword(const String & user_, const String & password_, const ExternalAuthenticators & external_authenticators) const; friend bool operator ==(const Authentication & lhs, const Authentication & rhs) { return (lhs.type == rhs.type) && (lhs.password_hash == rhs.password_hash); } From 07aa3fe30d952207c4e769975629c1334afc85ca Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 25 Dec 2020 03:46:08 +0400 Subject: [PATCH 127/284] Refine the caching --- src/Access/ExternalAuthenticators.cpp | 30 +++++++++++++-------------- src/Access/ExternalAuthenticators.h | 4 ++-- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index 71cc1c11020..81ab1af3b29 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -171,7 +171,7 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const String & user_name, const String & password) const { std::optional params; - const auto password_hash = std::hash{}(password); + std::size_t params_hash = 0; { std::scoped_lock lock(mutex); @@ -184,6 +184,7 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const S params = pit->second; params->user = user_name; params->password = password; + params_hash = params->getCoreHash(); // Check the cache, but only if the caching is enabled at all. if (params->verification_cooldown > std::chrono::seconds{0}) @@ -197,15 +198,15 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const S if (eit != cache.end()) { const auto & entry = eit->second; - const auto last_check_period = std::chrono::steady_clock::now() - entry.last_successful_password_check_timestamp; + const auto last_check_period = std::chrono::steady_clock::now() - entry.last_successful_authentication_timestamp; if ( // Forbid the initial values explicitly. - entry.last_successful_password_hash != 0 && - entry.last_successful_password_check_timestamp != std::chrono::steady_clock::time_point{} && + entry.last_successful_params_hash != 0 && + entry.last_successful_authentication_timestamp != std::chrono::steady_clock::time_point{} && // Check if we can safely "reuse" the result of the previous successful password verification. - password_hash == entry.last_successful_password_hash && + entry.last_successful_params_hash == params_hash && last_check_period >= std::chrono::seconds{0} && last_check_period <= params->verification_cooldown ) @@ -229,12 +230,12 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const S const auto result = client.check(); const auto current_check_timestamp = std::chrono::steady_clock::now(); - // Update the cache, but only if this is the latest check and the server is still configured in the same way. + // Update the cache, but only if this is the latest check and the server is still configured in a compatible way. if (result) { std::scoped_lock lock(mutex); - // If the server was removed from the config while we were checking the password, we discard this result. + // If the server was removed from the config while we were checking the password, we discard the current result. const auto pit = ldap_server_params.find(server); if (pit == ldap_server_params.end()) return false; @@ -243,20 +244,19 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const S new_params.user = user_name; new_params.password = password; - // If the critical server params have changed while we were checking the password, we discard the result. - if (params->getCoreHash() != new_params.getCoreHash()) + // If the critical server params have changed while we were checking the password, we discard the current result. + if (params_hash != new_params.getCoreHash()) return false; auto & entry = ldap_server_caches[server][user_name]; - - if (entry.last_successful_password_check_timestamp < current_check_timestamp) + if (entry.last_successful_authentication_timestamp < current_check_timestamp) { - entry.last_successful_password_hash = password_hash; - entry.last_successful_password_check_timestamp = current_check_timestamp; + entry.last_successful_params_hash = params_hash; + entry.last_successful_authentication_timestamp = current_check_timestamp; } - else if (entry.last_successful_password_hash != password_hash) + else if (entry.last_successful_params_hash != params_hash) { - // Somehow a newer check succeeded but with other password, so this one is obsolete and we discard it. + // Somehow a newer check with different params/password succeeded, so the current result is obsolete and we discard it. return false; } } diff --git a/src/Access/ExternalAuthenticators.h b/src/Access/ExternalAuthenticators.h index 16a69ec7709..fa618c92b3f 100644 --- a/src/Access/ExternalAuthenticators.h +++ b/src/Access/ExternalAuthenticators.h @@ -33,8 +33,8 @@ public: private: struct LDAPCacheEntry { - std::size_t last_successful_password_hash = 0; - std::chrono::steady_clock::time_point last_successful_password_check_timestamp; + std::size_t last_successful_params_hash = 0; + std::chrono::steady_clock::time_point last_successful_authentication_timestamp; }; using LDAPServerCache = std::unordered_map; // user name -> cache entry From 5bd025a180520f461e791486537561996e27d0b9 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 25 Dec 2020 06:13:30 +0300 Subject: [PATCH 128/284] fix DISTINCT and ORDER BY --- src/Interpreters/ExpressionAnalyzer.cpp | 65 +++++++++++-------- src/Interpreters/ExpressionAnalyzer.h | 5 +- src/Interpreters/InterpreterSelectQuery.cpp | 20 ++++-- .../01591_window_functions.reference | 26 ++++++-- .../0_stateless/01591_window_functions.sql | 11 ++-- 5 files changed, 87 insertions(+), 40 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 4af2a17e164..c10a8313bee 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -970,7 +970,9 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments( ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns); // 1) Add actions for window functions and their arguments; - // 2) Mark the columns that are really required. + // 2) Mark the columns that are really required. We have to mark them as + // required because we finish the expression chain before processing the + // window functions. for (const auto & [_, w] : window_descriptions) { for (const auto & f : w.window_functions) @@ -1425,49 +1427,47 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( /// If there is aggregation, we execute expressions in SELECT and ORDER BY on the initiating server, otherwise on the source servers. query_analyzer.appendSelect(chain, only_types || (need_aggregate ? !second_stage : !first_stage)); - has_order_by = query.orderBy() != nullptr; - // FIXME selected columns was set here. Should split order by and select - // and insert window functions in between. - // Will fix: - // 1) window function in DISTINCT - // 2) totally broken DISTINCT - // 3) window functions in order by - before_order_and_select = query_analyzer.appendOrderBy( - chain, - only_types || (need_aggregate ? !second_stage : !first_stage), - optimize_read_in_order, - order_by_elements_actions); - + // Window functions are processed in a separate expression chain after + // the main SELECT, similar to what we do for aggregate functions. if (has_window) { query_analyzer.appendWindowFunctionsArguments(chain, only_types || !first_stage); - auto select_required_columns = chain.getLastStep().required_output; for (const auto & x : chain.getLastActions()->getNamesAndTypesList()) { query_analyzer.columns_after_window.push_back(x); } + before_window = chain.getLastActions(); finalize_chain(chain); auto & step = chain.lastStep(query_analyzer.columns_after_window); - step.required_output = select_required_columns; + + // The output of this expression chain is the result of + // SELECT (before "final projection" i.e. renaming the columns), so + // we have to mark the expressions that are required in the output, + // again. We did it for the previous expression chain ("select w/o + // window functions") earlier, in appendSelect(). But that chain also + // produced the expressions required to calculate window functions. + // They are not needed in the final SELECT result. Knowing the correct + // list of columns is important when we apply SELECT DISTINCT later. const auto * select_query = query_analyzer.getSelectQuery(); for (const auto & child : select_query->select()->children) { - if (const auto * function - = typeid_cast(child.get()); - function && function->is_window_function) - { - // See its counterpart in appendSelect() - step.required_output.push_back(child->getColumnName()); - } + step.required_output.push_back(child->getColumnName()); } } selected_columns = chain.getLastStep().required_output; + has_order_by = query.orderBy() != nullptr; + before_order_by = query_analyzer.appendOrderBy( + chain, + only_types || (need_aggregate ? !second_stage : !first_stage), + optimize_read_in_order, + order_by_elements_actions); + if (query_analyzer.appendLimitBy(chain, only_types || !second_stage)) { before_limit_by = chain.getLastActions(); @@ -1479,7 +1479,6 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( finalize_chain(chain); } - /// Before executing WHERE and HAVING, remove the extra columns from the block (mostly the aggregation keys). removeExtraColumns(); @@ -1610,9 +1609,9 @@ std::string ExpressionAnalysisResult::dump() const ss << "before_window " << before_window->dumpDAG() << "\n"; } - if (before_order_and_select) + if (before_order_by) { - ss << "before_order_and_select " << before_order_and_select->dumpDAG() << "\n"; + ss << "before_order_by " << before_order_by->dumpDAG() << "\n"; } if (before_limit_by) @@ -1625,6 +1624,20 @@ std::string ExpressionAnalysisResult::dump() const ss << "final_projection " << final_projection->dumpDAG() << "\n"; } + if (!selected_columns.empty()) + { + ss << "selected_columns "; + for (size_t i = 0; i < selected_columns.size(); i++) + { + if (i > 0) + { + ss << ", "; + } + ss << backQuote(selected_columns[i]); + } + ss << "\n"; + } + return ss.str(); } diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 1095689a775..ea43efa6036 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -204,11 +204,12 @@ struct ExpressionAnalysisResult ActionsDAGPtr before_aggregation; ActionsDAGPtr before_having; ActionsDAGPtr before_window; - ActionsDAGPtr before_order_and_select; + ActionsDAGPtr before_order_by; ActionsDAGPtr before_limit_by; ActionsDAGPtr final_projection; - /// Columns from the SELECT list, before renaming them to aliases. + /// Columns from the SELECT list, before renaming them to aliases. Used to + /// perform SELECT DISTINCT. Names selected_columns; /// Columns will be removed after prewhere actions execution. diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 38cc19a00d6..eb7b78c35b4 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -538,7 +538,10 @@ Block InterpreterSelectQuery::getSampleBlockImpl() if (options.to_stage == QueryProcessingStage::Enum::WithMergeableState) { if (!analysis_result.need_aggregate) - return analysis_result.before_order_and_select->getResultColumns(); + { + // What's the difference with selected_columns? + return analysis_result.before_order_by->getResultColumns(); + } Block header = analysis_result.before_aggregation->getResultColumns(); @@ -564,7 +567,8 @@ Block InterpreterSelectQuery::getSampleBlockImpl() if (options.to_stage == QueryProcessingStage::Enum::WithMergeableStateAfterAggregation) { - return analysis_result.before_order_and_select->getResultColumns(); + // What's the difference with selected_columns? + return analysis_result.before_order_by->getResultColumns(); } return analysis_result.final_projection->getResultColumns(); @@ -958,8 +962,9 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu } else { - executeExpression(query_plan, expressions.before_order_and_select, "Before ORDER BY and SELECT"); + executeExpression(query_plan, expressions.before_window, "Before window functions"); executeWindow(query_plan); + executeExpression(query_plan, expressions.before_order_by, "Before ORDER BY"); executeDistinct(query_plan, true, expressions.selected_columns, true); } @@ -1005,8 +1010,10 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu else if (expressions.hasHaving()) executeHaving(query_plan, expressions.before_having); - executeExpression(query_plan, expressions.before_order_and_select, "Before ORDER BY and SELECT"); + executeExpression(query_plan, expressions.before_window, + "Before window functions"); executeWindow(query_plan); + executeExpression(query_plan, expressions.before_order_by, "Before ORDER BY"); executeDistinct(query_plan, true, expressions.selected_columns, true); } @@ -1745,6 +1752,11 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modific void InterpreterSelectQuery::executeExpression(QueryPlan & query_plan, const ActionsDAGPtr & expression, const std::string & description) { + if (!expression) + { + return; + } + auto expression_step = std::make_unique(query_plan.getCurrentDataStream(), expression); expression_step->setStepDescription(description); diff --git a/tests/queries/0_stateless/01591_window_functions.reference b/tests/queries/0_stateless/01591_window_functions.reference index 704c5a98811..e56c6bdeb3a 100644 --- a/tests/queries/0_stateless/01591_window_functions.reference +++ b/tests/queries/0_stateless/01591_window_functions.reference @@ -79,12 +79,22 @@ select number, quantileExact(number) over (partition by intDiv(number, 3)) q fro 9 9 select q * 10, quantileExact(number) over (partition by intDiv(number, 3)) q from numbers(10); -- { serverError 47 } --- should work in ORDER BY though --- doesn't work now --- select number, max(number) over (partition by intDiv(number, 3) order by number desc) m from numbers(10) order by m desc, number; +-- should work in ORDER BY --- at least it works in ORDER BY if you wrap it in a subquery +select number, max(number) over (partition by intDiv(number, 3) order by number desc) m from numbers(10) order by m desc, number; +-- also works in ORDER BY if you wrap it in a subquery + +9 9 +6 8 +7 8 +8 8 +3 5 +4 5 +5 5 +0 2 +1 2 +2 2 select * from (select count(*) over () c from numbers(3)) order by c; -- must work in WHERE if you wrap it in a subquery @@ -168,4 +178,12 @@ select groupArray(number) over () from numbers(3); [0,1,2] select count(1) over (), max(number + 1) over () from numbers(3); +-- Should work in DISTINCT + 1 3 +select distinct sum(0) over () from numbers(2); + +0 +select distinct any(number) over () from numbers(2); + +0 diff --git a/tests/queries/0_stateless/01591_window_functions.sql b/tests/queries/0_stateless/01591_window_functions.sql index fd2831ea2ac..8b82b8f95fc 100644 --- a/tests/queries/0_stateless/01591_window_functions.sql +++ b/tests/queries/0_stateless/01591_window_functions.sql @@ -24,11 +24,10 @@ select number, quantileExact(number) over (partition by intDiv(number, 3)) q fro -- last stage of select, after all other functions. select q * 10, quantileExact(number) over (partition by intDiv(number, 3)) q from numbers(10); -- { serverError 47 } --- should work in ORDER BY though --- doesn't work now --- select number, max(number) over (partition by intDiv(number, 3) order by number desc) m from numbers(10) order by m desc, number; +-- should work in ORDER BY +select number, max(number) over (partition by intDiv(number, 3) order by number desc) m from numbers(10) order by m desc, number; --- at least it works in ORDER BY if you wrap it in a subquery +-- also works in ORDER BY if you wrap it in a subquery select * from (select count(*) over () c from numbers(3)) order by c; -- must work in WHERE if you wrap it in a subquery @@ -57,3 +56,7 @@ select groupArray(number) over () from numbers(3); -- This one tests we properly process the window function arguments. -- Seen errors like 'column `1` not found' from count(1). select count(1) over (), max(number + 1) over () from numbers(3); + +-- Should work in DISTINCT +select distinct sum(0) over () from numbers(2); +select distinct any(number) over () from numbers(2); From a38787553ce363c41df430d1642ac1ba922261c0 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 25 Dec 2020 06:15:36 +0300 Subject: [PATCH 129/284] perf test fix --- tests/performance/window_functions.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/performance/window_functions.xml b/tests/performance/window_functions.xml index e4578a575a0..dcc873b5029 100644 --- a/tests/performance/window_functions.xml +++ b/tests/performance/window_functions.xml @@ -3,6 +3,10 @@ hits_100m_single + + 1 + + diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index e9952fc76fd..55e1ba66edc 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -99,6 +99,14 @@ If you add a new column to a table but later change its default expression, the It is not possible to set default values for elements in nested data structures. +## NULL Or NOT NULL Modifiers + +`NULL` and `NOT NULL` modifiers after data type in table definition allow or do not allow it to be [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable). + +If the type is not `Nullable` and if `NULL` is specified, it will be treated as `Nullable`; if `NOT NULL` is specified, then no. For example, `INT NULL` is the same as `Nullable(INT)`. If the type is `Nullable` and `NULL` or `NOT NULL` modifiers are specified, the exception will be thrown. + +See also [data_type_default_nullable](../../operations/settings/settings.md#data_type_default_nullable) setting. + ## Primary Key {#primary-key} You can define a [primary key](../../../engines/table-engines/mergetree-family/mergetree.md#primary-keys-and-indexes-in-queries) when creating a table. Primary key can be specified in two ways: From 99b50dbba3c452cfa93bc5b77b915241ab452c76 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Sat, 26 Dec 2020 16:10:37 +0300 Subject: [PATCH 156/284] DOCSUP-5043: Fix PR comments. --- .../operations/server-configuration-parameters/settings.md | 6 +++--- docs/ru/commercial/cloud.md | 5 ++--- .../operations/server-configuration-parameters/settings.md | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 1877e474555..4095e84627a 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -80,7 +80,7 @@ List of prefixes for [custom settings](../../operations/settings/index.md#custom - [Custom settings](../../operations/settings/index.md#custom_settings) -## core_dump +## core_dump {#server_configuration_parameters-core_dump} Configures soft limit for core dump file size. @@ -88,7 +88,7 @@ Possible values: - Positive integer. -Default value: `1073741824`. +Default value: `1073741824` (1 GB). !!! info "Note" Hard limit is configured via system tools @@ -440,7 +440,7 @@ Limits total RAM usage by the ClickHouse server. Possible values: - Positive integer. -- 0 (auto). +- 0 — Auto. Default value: `0`. diff --git a/docs/ru/commercial/cloud.md b/docs/ru/commercial/cloud.md index 8d02a6332a7..885aac97312 100644 --- a/docs/ru/commercial/cloud.md +++ b/docs/ru/commercial/cloud.md @@ -21,13 +21,12 @@ toc_title: "\u041f\u043e\u0441\u0442\u0430\u0432\u0449\u0438\u043a\u0438\u0020\u ## Altinity.Cloud {#altinity.cloud} [Altinity.Cloud](https://altinity.com/cloud-database/) — это полностью управляемый ClickHouse-as-a-Service для публичного облака Amazon. + - Быстрое развертывание кластеров ClickHouse на ресурсах Amazon -- Easy scale-out/scale-in as well as vertical scaling of nodes - Легкое горизонтальное масштабирование также, как и вертикальное масштабирование узлов - Изолированные виртуальные сети для каждого клиента с общедоступным эндпоинтом или пирингом VPC - Настраиваемые типы и объемы хранилищ -- Cross-AZ scaling for performance and high availability -- Cross-AZ масштабирование для повышения производительности и высокой доступности +- Cross-AZ масштабирование для повышения производительности и обеспечения высокой доступности - Встроенный мониторинг и редактор SQL-запросов {## [Оригинальная статья](https://clickhouse.tech/docs/ru/commercial/cloud/) ##} diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index affe09c7703..b75f0371223 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -80,7 +80,7 @@ ClickHouse проверяет условия для `min_part_size` и `min_part - [Пользовательские настройки](../../operations/settings/index.md#custom_settings) -## core_dump +## core_dump {#server_configuration_parameters-core_dump} Задает мягкое ограничение для размера файла дампа ядра. @@ -88,7 +88,7 @@ ClickHouse проверяет условия для `min_part_size` и `min_part - Положительное целое число. -Значение по умолчанию: `1073741824`. +Значение по умолчанию: `1073741824` (1 ГБ). !!! info "Примечание" Жесткое ограничение настраивается с помощью системных инструментов. From d2818278c5ed7bcef325dc2924458395a9ccc492 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 26 Dec 2020 17:26:50 +0300 Subject: [PATCH 157/284] Update ENABLE_CHECK_HEAVY_BUILDS limits for gcc and MSan --- CMakeLists.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index af15bc94018..0209eaee249 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,8 +69,14 @@ include (cmake/find/ccache.cmake) option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling" OFF) if (ENABLE_CHECK_HEAVY_BUILDS) # set DATA (since RSS does not work since 2.6.x+) to 2G + set (RLIMIT_DATA 5000000000) # set VIRT (RLIMIT_AS) to 10G (DATA*10) - set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=10000000000 --data=5000000000 --cpu=600) + set (RLIMIT_AS 10000000000) + # gcc10/gcc10/clang -fsanitize=memory is too heavy + if (SANITIZE STREQUAL "memory" OR COMPILER_GCC) + set (RLIMIT_DATA 10000000000) + endif() + set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS}--data=${RLIMIT_DATA}--cpu=600) endif () if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None") From 9271b3d6273f4d2516d14b98249ffaa74535f677 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 22 Dec 2020 02:58:54 +0300 Subject: [PATCH 158/284] SELECT JOIN now requires the SELECT privilege on each of the joined tables. --- src/Interpreters/InterpreterSelectQuery.cpp | 37 +++-- src/Interpreters/JoinedTables.cpp | 1 + src/Interpreters/JoinedTables.h | 5 - .../test_select_access_rights/__init__.py | 0 .../test_select_access_rights/test.py | 157 ++++++++++++++++++ 5 files changed, 184 insertions(+), 16 deletions(-) create mode 100644 tests/integration/test_select_access_rights/__init__.py create mode 100644 tests/integration/test_select_access_rights/test.py diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index aee4eae8782..061a205731c 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -210,6 +210,18 @@ static void rewriteMultipleJoins(ASTPtr & query, const TablesWithColumns & table JoinToSubqueryTransformVisitor(join_to_subs_data).visit(query); } +/// Returns true if we should ignore quotas and limits for a specified table in the system database. +static bool shouldIgnoreQuotaAndLimits(const StorageID & table_id) +{ + if (table_id.database_name == DatabaseCatalog::SYSTEM_DATABASE) + { + static const boost::container::flat_set tables_ignoring_quota{"quotas", "quota_limits", "quota_usage", "quotas_usage", "one"}; + if (tables_ignoring_quota.count(table_id.table_name)) + return true; + } + return false; +} + InterpreterSelectQuery::InterpreterSelectQuery( const ASTPtr & query_ptr_, const Context & context_, @@ -254,14 +266,18 @@ InterpreterSelectQuery::InterpreterSelectQuery( JoinedTables joined_tables(getSubqueryContext(*context), getSelectQuery()); + bool got_storage_from_query = false; if (!has_input && !storage) + { storage = joined_tables.getLeftTableStorage(); + got_storage_from_query = true; + } if (storage) { table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); table_id = storage->getStorageID(); - if (metadata_snapshot == nullptr) + if (!metadata_snapshot) metadata_snapshot = storage->getInMemoryMetadataPtr(); } @@ -279,9 +295,10 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (storage && joined_tables.isLeftTableSubquery()) { /// Rewritten with subquery. Free storage locks here. - storage = {}; + storage = nullptr; table_lock.reset(); table_id = StorageID::createEmpty(); + metadata_snapshot = nullptr; } } @@ -444,16 +461,14 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (query.prewhere() && !query.where()) analysis_result.prewhere_info->need_filter = true; - const StorageID & left_table_id = joined_tables.leftTableID(); - - if (left_table_id) - context->checkAccess(AccessType::SELECT, left_table_id, required_columns); - - /// Remove limits for some tables in the `system` database. - if (left_table_id.database_name == "system") + if (table_id && got_storage_from_query && !joined_tables.isLeftTableFunction()) { - static const boost::container::flat_set system_tables_ignoring_quota{"quotas", "quota_limits", "quota_usage", "quotas_usage", "one"}; - if (system_tables_ignoring_quota.count(left_table_id.table_name)) + /// The current user should have the SELECT privilege. + /// If this table_id is for a table function we don't check access rights here because in this case they have been already checked in ITableFunction::execute(). + context->checkAccess(AccessType::SELECT, table_id, required_columns); + + /// Remove limits for some tables in the `system` database. + if (shouldIgnoreQuotaAndLimits(table_id) && (joined_tables.tablesCount() <= 1)) { options.ignore_quota = true; options.ignore_limits = true; diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index c0511122c1e..17d7949e478 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -161,6 +161,7 @@ StoragePtr JoinedTables::getLeftTableStorage() if (isLeftTableFunction()) return context.getQueryContext().executeTableFunction(left_table_expression); + StorageID table_id = StorageID::createEmpty(); if (left_db_and_table) { table_id = context.resolveStorageID(StorageID(left_db_and_table->database, left_db_and_table->table, left_db_and_table->uuid)); diff --git a/src/Interpreters/JoinedTables.h b/src/Interpreters/JoinedTables.h index 1e787ee4a65..812808fed61 100644 --- a/src/Interpreters/JoinedTables.h +++ b/src/Interpreters/JoinedTables.h @@ -43,8 +43,6 @@ public: bool isLeftTableFunction() const; size_t tablesCount() const { return table_expressions.size(); } - const StorageID & leftTableID() const { return table_id; } - void rewriteDistributedInAndJoins(ASTPtr & query); std::unique_ptr makeLeftTableSubquery(const SelectQueryOptions & select_options); @@ -57,9 +55,6 @@ private: /// Legacy (duplicated left table values) ASTPtr left_table_expression; std::optional left_db_and_table; - - /// left_db_and_table or 'system.one' - StorageID table_id = StorageID::createEmpty(); }; } diff --git a/tests/integration/test_select_access_rights/__init__.py b/tests/integration/test_select_access_rights/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_select_access_rights/test.py b/tests/integration/test_select_access_rights/test.py new file mode 100644 index 00000000000..ccea77d6fb7 --- /dev/null +++ b/tests/integration/test_select_access_rights/test.py @@ -0,0 +1,157 @@ +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('instance') + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def cleanup_after_test(): + instance.query("CREATE USER OR REPLACE A") + yield + instance.query("DROP TABLE IF EXISTS table1") + instance.query("DROP TABLE IF EXISTS table2") + + +def test_select_single_column(): + instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d") + + select_query = "SELECT a FROM table1" + assert "it's necessary to have grant SELECT(a) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT(a) ON default.table1 TO A") + assert instance.query(select_query, user = 'A') == "" + + instance.query("REVOKE SELECT(a) ON default.table1 FROM A") + assert "it's necessary to have grant SELECT(a) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + +def test_select_single_column_with_table_grant(): + instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d") + + select_query = "SELECT a FROM table1" + assert "it's necessary to have grant SELECT(a) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT ON default.table1 TO A") + assert instance.query(select_query, user = 'A') == "" + + instance.query("REVOKE SELECT(a) ON default.table1 FROM A") + assert "it's necessary to have grant SELECT(a) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + +def test_select_all_columns(): + instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d") + + select_query = "SELECT * FROM table1" + assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT(d) ON default.table1 TO A") + assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT(a) ON default.table1 TO A") + assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT(b) ON default.table1 TO A") + assert instance.query(select_query, user = 'A') == "" + + +def test_select_all_columns_with_table_grant(): + instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d") + + select_query = "SELECT * FROM table1" + assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT ON default.table1 TO A") + assert instance.query(select_query, user = 'A') == "" + + +def test_alias(): + instance.query("CREATE TABLE table1(x Int32, y Int32) ENGINE = MergeTree ORDER BY tuple()") + + select_query = "SELECT x, y, x + y AS s FROM table1" + assert "it's necessary to have grant SELECT(x, y) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT(x, y) ON default.table1 TO A") + assert instance.query(select_query, user = 'A') == "" + + +def test_alias_columns(): + instance.query("CREATE TABLE table1(x Int32, y Int32, s Int32 ALIAS x + y) ENGINE = MergeTree ORDER BY tuple()") + + select_query = "SELECT * FROM table1" + assert "it's necessary to have grant SELECT(x, y) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT(x,y) ON default.table1 TO A") + assert instance.query(select_query, user = 'A') == "" + + select_query = "SELECT s FROM table1" + assert "it's necessary to have grant SELECT(s) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT(s) ON default.table1 TO A") + assert instance.query(select_query, user = 'A') == "" + + instance.query("REVOKE SELECT(x,y) ON default.table1 FROM A") + assert instance.query(select_query, user = 'A') == "" + + +def test_materialized_columns(): + instance.query("CREATE TABLE table1(x Int32, y Int32, p Int32 MATERIALIZED x * y) ENGINE = MergeTree ORDER BY tuple()") + + select_query = "SELECT * FROM table1" + assert "it's necessary to have grant SELECT(x, y) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT(x,y) ON default.table1 TO A") + assert instance.query(select_query, user = 'A') == "" + + select_query = "SELECT p FROM table1" + assert "it's necessary to have grant SELECT(p) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT(p) ON default.table1 TO A") + assert instance.query(select_query, user = 'A') == "" + + instance.query("REVOKE SELECT(x,y) ON default.table1 FROM A") + assert instance.query(select_query, user = 'A') == "" + + +def test_select_join(): + instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d") + instance.query("CREATE TABLE table2(d DATE, x UInt32, y UInt8) ENGINE = MergeTree ORDER BY d") + + select_query = "SELECT * FROM table1 JOIN table2 USING(d)" + assert "it's necessary to have grant SELECT(d, x, y) ON default.table2" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT(d, x, y) ON default.table2 TO A") + assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT(d, a, b) ON default.table1 TO A") + assert instance.query(select_query, user = 'A') == "" + + instance.query("REVOKE SELECT ON default.table2 FROM A") + assert "it's necessary to have grant SELECT(d, x, y) ON default.table2" in instance.query_and_get_error(select_query, user = 'A') + + +def test_select_union(): + instance.query("CREATE TABLE table1(a String, b UInt8) ENGINE = MergeTree ORDER BY tuple()") + instance.query("CREATE TABLE table2(a String, b UInt8) ENGINE = MergeTree ORDER BY tuple()") + + select_query = "SELECT * FROM table1 UNION ALL SELECT * FROM table2" + assert "it's necessary to have grant SELECT(a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT(a, b) ON default.table1 TO A") + assert "it's necessary to have grant SELECT(a, b) ON default.table2" in instance.query_and_get_error(select_query, user = 'A') + + instance.query("GRANT SELECT(a, b) ON default.table2 TO A") + assert instance.query(select_query, user = 'A') == "" + + instance.query("REVOKE SELECT ON default.table1 FROM A") + assert "it's necessary to have grant SELECT(a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') From cc034b337310df83093d84de5abf9971ef32f4cc Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sun, 27 Dec 2020 11:36:18 +0300 Subject: [PATCH 159/284] Update union.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Подкорректировал английскую и русскую версии. --- docs/en/sql-reference/statements/select/union.md | 6 +++--- docs/ru/sql-reference/statements/select/union.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/statements/select/union.md b/docs/en/sql-reference/statements/select/union.md index 18b132bfac7..ade4a19ab3b 100644 --- a/docs/en/sql-reference/statements/select/union.md +++ b/docs/en/sql-reference/statements/select/union.md @@ -8,7 +8,7 @@ You can use `UNION` with explicitly specifying `UNION ALL` or `UNION DISTINCT`. By default, `UNION` has the same behavior as `UNION DISTINCT`. The difference between `UNION ALL` and `UNION DISTINCT` is that `UNION DISTINCT` will do a distinct transform for union result, it is equivalent to `SELECT DISTINCT` from a subquery containing `UNION ALL`. -You can use `UNION ALL` to combine any number of `SELECT` queries by extending their results. Example: +You can use `UNION` to combine any number of `SELECT` queries by extending their results. Example: ``` sql SELECT CounterID, 1 AS table, toInt64(count()) AS c @@ -25,9 +25,9 @@ SELECT CounterID, 2 AS table, sum(Sign) AS c Result columns are matched by their index (order inside `SELECT`). If column names do not match, names for the final result are taken from the first query. -Type casting is performed for unions. For example, if two queries being combined have the same field with non-`Nullable` and `Nullable` types from a compatible type, the resulting `UNION ALL` has a `Nullable` type field. +Type casting is performed for unions. For example, if two queries being combined have the same field with non-`Nullable` and `Nullable` types from a compatible type, the resulting `UNION` has a `Nullable` type field. -Queries that are parts of `UNION ALL` can’t be enclosed in round brackets. [ORDER BY](../../../sql-reference/statements/select/order-by.md) and [LIMIT](../../../sql-reference/statements/select/limit.md) are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with `UNION ALL` in a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause. +Queries that are parts of `UNION` can’t be enclosed in round brackets. [ORDER BY](../../../sql-reference/statements/select/order-by.md) and [LIMIT](../../../sql-reference/statements/select/limit.md) are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with `UNION` in a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause. If you use `UNION` without explicitly specifying `UNION ALL` or `UNION DISTINCT`, you can specify the union mode using the [union_default_mode](../../../operations/settings/settings.md#union-default-mode) setting. The setting values can be `ALL`, `DISTINCT` or an empty string. However, if you use `UNION` with `union_default_mode` setting to empty string, it will throw an exception. The following examples demonstrate the results of queries with different values setting. diff --git a/docs/ru/sql-reference/statements/select/union.md b/docs/ru/sql-reference/statements/select/union.md index 2bde7ce9da0..ca9616cfd25 100644 --- a/docs/ru/sql-reference/statements/select/union.md +++ b/docs/ru/sql-reference/statements/select/union.md @@ -8,7 +8,7 @@ toc_title: UNION По умолчанию, `UNION` ведет себя так же, как и `UNION DISTINCT`. Разница между `UNION ALL` и `UNION DISTINCT` в том, что `UNION DISTINCT` выполняет явное преобразование для результата объединения. Это равнозначно выражению `SELECT DISTINCT` из подзапроса, содержащего `UNION ALL`. -Вы можете использовать `UNION ALL` чтобы объединить любое количество `SELECT` запросы путем расширения их результатов. Пример: +Чтобы объединить любое количество `SELECT` запросов путем расширения их результатов, вы можете использовать `UNION`. Пример: ``` sql SELECT CounterID, 1 AS table, toInt64(count()) AS c @@ -25,9 +25,9 @@ SELECT CounterID, 2 AS table, sum(Sign) AS c Результирующие столбцы сопоставляются по их индексу (порядку внутри `SELECT`). Если имена столбцов не совпадают, то имена для конечного результата берутся из первого запроса. -При объединении выполняет приведение типов. Например, если два запроса имеют одно и то же поле с не-`Nullable` и `Nullable` совместимыми типами, полученные в результате `UNION ALL` данные будут иметь `Nullable` тип. +При объединении выполняет приведение типов. Например, если два запроса имеют одно и то же поле с не-`Nullable` и `Nullable` совместимыми типами, полученные в результате `UNION` данные будут иметь `Nullable` тип. -Запросы, которые являются частью `UNION ALL` не могут быть заключен в круглые скобки. [ORDER BY](order-by.md) и [LIMIT](limit.md) применяются к отдельным запросам, а не к конечному результату. Если вам нужно применить преобразование к конечному результату, вы можете разместить все объединенные с помощью `UNION ALL` запросы в подзапрос в секции [FROM](from.md). +Запросы, которые являются частью `UNION`, не могут быть заключены в круглые скобки. [ORDER BY](order-by.md) и [LIMIT](limit.md) применяются к отдельным запросам, а не к конечному результату. Если вам нужно применить преобразование к конечному результату, вы можете разместить все объединенные с помощью `UNION` запросы в подзапрос в секции [FROM](from.md). Если используете `UNION` без явного указания `UNION ALL` или `UNION DISTINCT`, то вы можете указать режим объединения с помощью настройки [union_default_mode](../../../operations/settings/settings.md#union-default-mode), значениями которой могут быть `ALL`, `DISTINCT` или пустая строка. Однако если вы используете `UNION` с настройкой `union_default_mode`, значением которой является пустая строка, то будет сгенерировано исключение. В следующих примерах продемонстрированы результаты запросов при разных значениях настройки. From be4832de6dc2fcbf0c9910022bcdbf4c2a96d0ad Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 27 Dec 2020 12:54:59 +0300 Subject: [PATCH 160/284] Try fix test. --- src/Functions/FunctionsStringHash.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Functions/FunctionsStringHash.cpp b/src/Functions/FunctionsStringHash.cpp index 9c0b9f79448..e389e2f7f98 100644 --- a/src/Functions/FunctionsStringHash.cpp +++ b/src/Functions/FunctionsStringHash.cpp @@ -262,6 +262,9 @@ struct SimHashImpl words.emplace_back(BytesRef{word_start, length}); } + if (words.empty()) + return 0; + UInt64 hash_value = Hash::shingleHash(words); updateFingerVector(finger_vec, hash_value); @@ -488,6 +491,9 @@ struct MinHashImpl words.emplace_back(BytesRef{word_start, length}); } + if (words.empty()) + return; + UInt64 hash_value = Hash::shingleHash(words); { const UInt8 * shingle_start = words.front().data; From d6f501f497f728ba5563ebc38d79396a535e4b45 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 27 Dec 2020 13:30:00 +0300 Subject: [PATCH 161/284] Try fix Pipeline stuck --- src/Processors/QueryPlan/AddingDelayedSourceStep.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Processors/QueryPlan/AddingDelayedSourceStep.cpp b/src/Processors/QueryPlan/AddingDelayedSourceStep.cpp index 7247dc55ce2..99db95059a0 100644 --- a/src/Processors/QueryPlan/AddingDelayedSourceStep.cpp +++ b/src/Processors/QueryPlan/AddingDelayedSourceStep.cpp @@ -32,6 +32,11 @@ void AddingDelayedSourceStep::transformPipeline(QueryPipeline & pipeline) { source->setQueryPlanStep(this); pipeline.addDelayedStream(source); + + /// Now, after adding delayed stream, it has implicit dependency on other port. + /// Here we add resize processor to remove this dependency. + /// Otherwise, if we add MergeSorting + MergingSorted transform to pipeline, we could get `Pipeline stuck` + pipeline.resize(pipeline.getNumStreams(), true); } } From a85cfa81db00469e2985f8fc6886d95ac70c6e1f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 27 Dec 2020 14:02:21 +0300 Subject: [PATCH 162/284] Try fix Pipeline stuck --- src/Processors/DelayedPortsProcessor.cpp | 27 ++++++++++++++++++------ src/Processors/DelayedPortsProcessor.h | 2 +- src/Processors/QueryPipeline.cpp | 2 +- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/Processors/DelayedPortsProcessor.cpp b/src/Processors/DelayedPortsProcessor.cpp index 5a6297413e1..a634f348771 100644 --- a/src/Processors/DelayedPortsProcessor.cpp +++ b/src/Processors/DelayedPortsProcessor.cpp @@ -3,8 +3,10 @@ namespace DB { -DelayedPortsProcessor::DelayedPortsProcessor(const Block & header, size_t num_ports, const PortNumbers & delayed_ports) - : IProcessor(InputPorts(num_ports, header), OutputPorts(num_ports, header)) +DelayedPortsProcessor::DelayedPortsProcessor( + const Block & header, size_t num_ports, const PortNumbers & delayed_ports, bool assert_delayed_ports_empty) + : IProcessor(InputPorts(num_ports, header), + OutputPorts(num_ports - (assert_delayed_ports_empty ? delayed_ports.size() : 0), header)) , num_delayed(delayed_ports.size()) { port_pairs.resize(num_ports); @@ -14,9 +16,13 @@ DelayedPortsProcessor::DelayedPortsProcessor(const Block & header, size_t num_po for (size_t i = 0; i < num_ports; ++i) { port_pairs[i].input_port = &*input_it; - port_pairs[i].output_port = &*output_it; ++input_it; - ++output_it; + + if (output_it != outputs.end()) + { + port_pairs[i].output_port = &*output_it; + ++output_it; + } } for (const auto & delayed : delayed_ports) @@ -34,7 +40,7 @@ bool DelayedPortsProcessor::processPair(PortsPair & pair) } }; - if (pair.output_port->isFinished()) + if (pair.output_port && pair.output_port->isFinished()) { pair.input_port->close(); finish(); @@ -43,17 +49,24 @@ bool DelayedPortsProcessor::processPair(PortsPair & pair) if (pair.input_port->isFinished()) { - pair.output_port->finish(); + if (pair.output_port) + pair.output_port->finish(); finish(); return false; } - if (!pair.output_port->canPush()) + if (pair.output_port && !pair.output_port->canPush()) return false; pair.input_port->setNeeded(); if (pair.input_port->hasData()) + { + if (!pair.output_port) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Input port for DelayedPortsProcessor is assumed to have no data, but it has one"); + pair.output_port->pushData(pair.input_port->pullData()); + } return true; } diff --git a/src/Processors/DelayedPortsProcessor.h b/src/Processors/DelayedPortsProcessor.h index 44dd632f8a8..77409332d8e 100644 --- a/src/Processors/DelayedPortsProcessor.h +++ b/src/Processors/DelayedPortsProcessor.h @@ -11,7 +11,7 @@ namespace DB class DelayedPortsProcessor : public IProcessor { public: - DelayedPortsProcessor(const Block & header, size_t num_ports, const PortNumbers & delayed_ports); + DelayedPortsProcessor(const Block & header, size_t num_ports, const PortNumbers & delayed_ports, bool assert_delayed_ports_empty = false); String getName() const override { return "DelayedPorts"; } diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index ffcab2e7a8d..ff086fd5b11 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -298,7 +298,7 @@ void QueryPipeline::addPipelineBefore(QueryPipeline pipeline) pipes.emplace_back(QueryPipeline::getPipe(std::move(pipeline))); pipe = Pipe::unitePipes(std::move(pipes), collected_processors); - auto processor = std::make_shared(getHeader(), pipe.numOutputPorts(), delayed_streams); + auto processor = std::make_shared(getHeader(), pipe.numOutputPorts(), delayed_streams, true); addTransform(std::move(processor)); } From b19146065f4f9ac424e1909046edf67f45feb6c8 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 27 Dec 2020 14:06:14 +0300 Subject: [PATCH 163/284] Try fix Pipeline stuck --- src/Processors/DelayedPortsProcessor.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Processors/DelayedPortsProcessor.cpp b/src/Processors/DelayedPortsProcessor.cpp index a634f348771..d6a773d2390 100644 --- a/src/Processors/DelayedPortsProcessor.cpp +++ b/src/Processors/DelayedPortsProcessor.cpp @@ -11,6 +11,9 @@ DelayedPortsProcessor::DelayedPortsProcessor( { port_pairs.resize(num_ports); + for (const auto & delayed : delayed_ports) + port_pairs[delayed].is_delayed = true; + auto input_it = inputs.begin(); auto output_it = outputs.begin(); for (size_t i = 0; i < num_ports; ++i) @@ -18,15 +21,12 @@ DelayedPortsProcessor::DelayedPortsProcessor( port_pairs[i].input_port = &*input_it; ++input_it; - if (output_it != outputs.end()) + if (!port_pairs[i].is_delayed || !assert_delayed_ports_empty) { port_pairs[i].output_port = &*output_it; ++output_it; } } - - for (const auto & delayed : delayed_ports) - port_pairs[delayed].is_delayed = true; } bool DelayedPortsProcessor::processPair(PortsPair & pair) From 30bbba6ed5d462840c3a33be5e6e4197161b7f29 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 27 Dec 2020 14:13:17 +0300 Subject: [PATCH 164/284] Try fix Pipeline stuck --- src/Processors/DelayedPortsProcessor.cpp | 7 +++++-- src/Processors/DelayedPortsProcessor.h | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Processors/DelayedPortsProcessor.cpp b/src/Processors/DelayedPortsProcessor.cpp index d6a773d2390..a2dc833b039 100644 --- a/src/Processors/DelayedPortsProcessor.cpp +++ b/src/Processors/DelayedPortsProcessor.cpp @@ -10,6 +10,7 @@ DelayedPortsProcessor::DelayedPortsProcessor( , num_delayed(delayed_ports.size()) { port_pairs.resize(num_ports); + output_to_pair.reserve(outputs.size()); for (const auto & delayed : delayed_ports) port_pairs[delayed].is_delayed = true; @@ -24,6 +25,7 @@ DelayedPortsProcessor::DelayedPortsProcessor( if (!port_pairs[i].is_delayed || !assert_delayed_ports_empty) { port_pairs[i].output_port = &*output_it; + output_to_pair.push_back(i); ++output_it; } } @@ -78,8 +80,9 @@ IProcessor::Status DelayedPortsProcessor::prepare(const PortNumbers & updated_in for (const auto & output_number : updated_outputs) { - if (!skip_delayed || !port_pairs[output_number].is_delayed) - need_data = processPair(port_pairs[output_number]) || need_data; + auto pair_num = output_to_pair[output_number]; + if (!skip_delayed || !port_pairs[pair_num].is_delayed) + need_data = processPair(port_pairs[pair_num]) || need_data; } for (const auto & input_number : updated_inputs) diff --git a/src/Processors/DelayedPortsProcessor.h b/src/Processors/DelayedPortsProcessor.h index 77409332d8e..1bcce20b5fa 100644 --- a/src/Processors/DelayedPortsProcessor.h +++ b/src/Processors/DelayedPortsProcessor.h @@ -31,6 +31,8 @@ private: size_t num_delayed; size_t num_finished = 0; + std::vector output_to_pair; + bool processPair(PortsPair & pair); }; From 7064a8500b681f516f5d7de8ae29539ada987440 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 27 Dec 2020 14:16:40 +0300 Subject: [PATCH 165/284] Try fix Pipeline stuck --- src/Processors/DelayedPortsProcessor.cpp | 6 +++--- src/Processors/DelayedPortsProcessor.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Processors/DelayedPortsProcessor.cpp b/src/Processors/DelayedPortsProcessor.cpp index a2dc833b039..7689dd85245 100644 --- a/src/Processors/DelayedPortsProcessor.cpp +++ b/src/Processors/DelayedPortsProcessor.cpp @@ -4,9 +4,9 @@ namespace DB { DelayedPortsProcessor::DelayedPortsProcessor( - const Block & header, size_t num_ports, const PortNumbers & delayed_ports, bool assert_delayed_ports_empty) + const Block & header, size_t num_ports, const PortNumbers & delayed_ports, bool assert_main_ports_empty) : IProcessor(InputPorts(num_ports, header), - OutputPorts(num_ports - (assert_delayed_ports_empty ? delayed_ports.size() : 0), header)) + OutputPorts((assert_main_ports_empty ? delayed_ports.size() : num_ports), header)) , num_delayed(delayed_ports.size()) { port_pairs.resize(num_ports); @@ -22,7 +22,7 @@ DelayedPortsProcessor::DelayedPortsProcessor( port_pairs[i].input_port = &*input_it; ++input_it; - if (!port_pairs[i].is_delayed || !assert_delayed_ports_empty) + if (port_pairs[i].is_delayed || !assert_main_ports_empty) { port_pairs[i].output_port = &*output_it; output_to_pair.push_back(i); diff --git a/src/Processors/DelayedPortsProcessor.h b/src/Processors/DelayedPortsProcessor.h index 1bcce20b5fa..cb04e84b1ad 100644 --- a/src/Processors/DelayedPortsProcessor.h +++ b/src/Processors/DelayedPortsProcessor.h @@ -11,7 +11,7 @@ namespace DB class DelayedPortsProcessor : public IProcessor { public: - DelayedPortsProcessor(const Block & header, size_t num_ports, const PortNumbers & delayed_ports, bool assert_delayed_ports_empty = false); + DelayedPortsProcessor(const Block & header, size_t num_ports, const PortNumbers & delayed_ports, bool assert_main_ports_empty = false); String getName() const override { return "DelayedPorts"; } From bc277892f4e7bc386adaaa41db47bd774ac6da38 Mon Sep 17 00:00:00 2001 From: Anna Date: Sun, 27 Dec 2020 14:21:58 +0300 Subject: [PATCH 166/284] add desc for encodeXMLComponent --- .../functions/string-functions.md | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index a846a01f11f..21e6f224b67 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -558,4 +558,36 @@ Result: └─────┘ ``` +## encodeXMLComponent {#encode-xml-component} + +Escape characters to place string into XML text node or attribute. + +**Syntax** + +``` sql +encodeXMLComponent(x) +``` + +**Parameters** + +- `x` — Sequence of characters. [String](../../sql-reference/data-types/string.md). + +**Returned value(s)** + +- Sequence of characters. + +Type: [String](../../sql-reference/data-types/string.md). + +**Example** + +Query: + +``` sql +``` + +Result: + +``` text +``` + [Original article](https://clickhouse.tech/docs/en/query_language/functions/string_functions/) From da6bfc2cfbc87da0e7c3c71554eec4acb1e4b96e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 27 Dec 2020 14:22:46 +0300 Subject: [PATCH 167/284] Try fix Pipeline stuck --- src/Processors/DelayedPortsProcessor.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Processors/DelayedPortsProcessor.cpp b/src/Processors/DelayedPortsProcessor.cpp index 7689dd85245..9215409af47 100644 --- a/src/Processors/DelayedPortsProcessor.cpp +++ b/src/Processors/DelayedPortsProcessor.cpp @@ -78,6 +78,11 @@ IProcessor::Status DelayedPortsProcessor::prepare(const PortNumbers & updated_in bool skip_delayed = (num_finished + num_delayed) < port_pairs.size(); bool need_data = false; + if (!updated_outputs.empty()) + for (const auto & pair : port_pairs) + if (!pair.output_port) + pair.input_port->setNeeded(); + for (const auto & output_number : updated_outputs) { auto pair_num = output_to_pair[output_number]; From 1ebab1dc487bd0985c170efb82f29f87b8ba0b76 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 27 Dec 2020 14:25:43 +0300 Subject: [PATCH 168/284] Try fix Pipeline stuck --- src/Processors/DelayedPortsProcessor.cpp | 7 ++++++- src/Processors/DelayedPortsProcessor.h | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Processors/DelayedPortsProcessor.cpp b/src/Processors/DelayedPortsProcessor.cpp index 9215409af47..ce21c5a018c 100644 --- a/src/Processors/DelayedPortsProcessor.cpp +++ b/src/Processors/DelayedPortsProcessor.cpp @@ -78,11 +78,16 @@ IProcessor::Status DelayedPortsProcessor::prepare(const PortNumbers & updated_in bool skip_delayed = (num_finished + num_delayed) < port_pairs.size(); bool need_data = false; - if (!updated_outputs.empty()) + if (!are_inputs_initialized && !updated_outputs.empty()) + { + /// Activate inputs with no output. for (const auto & pair : port_pairs) if (!pair.output_port) pair.input_port->setNeeded(); + are_inputs_initialized = true; + } + for (const auto & output_number : updated_outputs) { auto pair_num = output_to_pair[output_number]; diff --git a/src/Processors/DelayedPortsProcessor.h b/src/Processors/DelayedPortsProcessor.h index cb04e84b1ad..3e44c945bd4 100644 --- a/src/Processors/DelayedPortsProcessor.h +++ b/src/Processors/DelayedPortsProcessor.h @@ -32,6 +32,7 @@ private: size_t num_finished = 0; std::vector output_to_pair; + bool are_inputs_initialized = false; bool processPair(PortsPair & pair); }; From 7f1eb506ab70317b3aa701843198668f94ff4695 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 27 Dec 2020 14:28:13 +0300 Subject: [PATCH 169/284] Added test. --- .../0_stateless/01621_sort_after_join_pipeline_stuck.reference | 0 .../queries/0_stateless/01621_sort_after_join_pipeline_stuck.sql | 1 + 2 files changed, 1 insertion(+) create mode 100644 tests/queries/0_stateless/01621_sort_after_join_pipeline_stuck.reference create mode 100644 tests/queries/0_stateless/01621_sort_after_join_pipeline_stuck.sql diff --git a/tests/queries/0_stateless/01621_sort_after_join_pipeline_stuck.reference b/tests/queries/0_stateless/01621_sort_after_join_pipeline_stuck.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01621_sort_after_join_pipeline_stuck.sql b/tests/queries/0_stateless/01621_sort_after_join_pipeline_stuck.sql new file mode 100644 index 00000000000..ce9ae65e71e --- /dev/null +++ b/tests/queries/0_stateless/01621_sort_after_join_pipeline_stuck.sql @@ -0,0 +1 @@ +SELECT k FROM (SELECT NULL, nullIf(number, 3) AS k, '1048575', (65536, -9223372036854775808), toString(number) AS a FROM system.numbers LIMIT 1048577) AS js1 ANY RIGHT JOIN (SELECT 1.000100016593933, nullIf(number, NULL) AS k, toString(number) AS b FROM system.numbers LIMIT 2, 255) AS js2 USING (k) ORDER BY 257 ASC NULLS LAST FORMAT Null; From 436df8fe3ac79638fcd2badcb7f851285354e3cf Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 27 Dec 2020 14:36:08 +0300 Subject: [PATCH 170/284] Check fuzzer tests --- .../queries/0_stateless/01593_concurrent_alter_mutations_kill.sh | 1 + .../01593_concurrent_alter_mutations_kill_many_replicas.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh index dd8eb9d731e..fe243a7e929 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh @@ -24,6 +24,7 @@ function alter_thread function kill_mutation_thread { while true; do + # find any mutation and kill it mutation_id=$($CLICKHOUSE_CLIENT --query "SELECT mutation_id FROM system.mutations WHERE is_done=0 and database='${CLICKHOUSE_DATABASE}' and table='concurrent_mutate_kill' LIMIT 1") if [ ! -z "$mutation_id" ]; then $CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE mutation_id='$mutation_id'" 1> /dev/null diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh index cfbb183263d..e9513557722 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh @@ -36,6 +36,7 @@ function alter_thread function kill_mutation_thread { while true; do + # find any mutation and kill it mutation_id=$($CLICKHOUSE_CLIENT --query "SELECT mutation_id FROM system.mutations WHERE is_done = 0 and table like 'concurrent_kill_%' and database='${CLICKHOUSE_DATABASE}' LIMIT 1") if [ ! -z "$mutation_id" ]; then $CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE mutation_id='$mutation_id'" 1> /dev/null From 45b8ddd7ab1e7cedfc0a2721f874e930a12808fd Mon Sep 17 00:00:00 2001 From: Anna Date: Sun, 27 Dec 2020 14:47:15 +0300 Subject: [PATCH 171/284] fixed --- docs/en/operations/settings/settings.md | 6 +++--- .../sql-reference/statements/create/table.md | 20 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 4528f7010bf..7a2c8a487c1 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2472,12 +2472,12 @@ Default value: `0`. ## data_type_default_nullable {#data_type_default_nullable} -Allows data types without explicit modifiers in table definition will be [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable). +Allows data types without explicit modifiers in column definition will be [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable). Possible values: -- 1 — `Nullable`-type expressions are allowed in a table definition. -- 0 — `Nullable`-type expressions are not allowed in a table definition. +- 1 — The data type in a column definition is `Nullable`. +- 0 — The data type in a column definition is not `Nullable`. Default value: `0`. diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 55e1ba66edc..b1a5fdb19b5 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -16,8 +16,8 @@ By default, tables are created only on the current server. Distributed DDL queri ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [compression_codec] [TTL expr2], + name1 [type1] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], + name2 [type2] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|ALIAS expr2] [compression_codec] [TTL expr2], ... ) ENGINE = engine ``` @@ -57,6 +57,14 @@ In all cases, if `IF NOT EXISTS` is specified, the query won’t return an error There can be other clauses after the `ENGINE` clause in the query. See detailed documentation on how to create tables in the descriptions of [table engines](../../../engines/table-engines/index.md#table_engines). +## NULL Or NOT NULL Modifiers {#null-modifiers} + +`NULL` and `NOT NULL` modifiers after data type in column definition allow or do not allow it to be [Nullable](../../../sql-reference/data-types/nullable.md#data_type-nullable). + +If the type is not `Nullable` and if `NULL` is specified, it will be treated as `Nullable`; if `NOT NULL` is specified, then no. For example, `INT NULL` is the same as `Nullable(INT)`. If the type is `Nullable` and `NULL` or `NOT NULL` modifiers are specified, the exception will be thrown. + +See also [data_type_default_nullable](../../../operations/settings/settings.md#data_type_default_nullable) setting. + ## Default Values {#create-default-values} The column description can specify an expression for a default value, in one of the following ways: `DEFAULT expr`, `MATERIALIZED expr`, `ALIAS expr`. @@ -99,14 +107,6 @@ If you add a new column to a table but later change its default expression, the It is not possible to set default values for elements in nested data structures. -## NULL Or NOT NULL Modifiers - -`NULL` and `NOT NULL` modifiers after data type in table definition allow or do not allow it to be [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable). - -If the type is not `Nullable` and if `NULL` is specified, it will be treated as `Nullable`; if `NOT NULL` is specified, then no. For example, `INT NULL` is the same as `Nullable(INT)`. If the type is `Nullable` and `NULL` or `NOT NULL` modifiers are specified, the exception will be thrown. - -See also [data_type_default_nullable](../../operations/settings/settings.md#data_type_default_nullable) setting. - ## Primary Key {#primary-key} You can define a [primary key](../../../engines/table-engines/mergetree-family/mergetree.md#primary-keys-and-indexes-in-queries) when creating a table. Primary key can be specified in two ways: From 9c62adcb4b7422d277b6d7d9da268cbb421f561a Mon Sep 17 00:00:00 2001 From: Anna Date: Sun, 27 Dec 2020 15:26:31 +0300 Subject: [PATCH 172/284] fixed --- docs/en/operations/settings/settings.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 7a2c8a487c1..81c8787d442 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2472,12 +2472,12 @@ Default value: `0`. ## data_type_default_nullable {#data_type_default_nullable} -Allows data types without explicit modifiers in column definition will be [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable). +Allows data types without explicit modifiers [NULL or NOT NULL](../../sql-reference/statements/create/table.md#null-modifiers) in column definition will be [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable). Possible values: -- 1 — The data type in a column definition is `Nullable`. -- 0 — The data type in a column definition is not `Nullable`. +- 1 — The data types in column definitions are set to `Nullable` by default. +- 0 — The data types in column definitions are set to not `Nullable` by default. Default value: `0`. From 2e69757062806a7683eccf76592db50cb471259a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 27 Dec 2020 16:53:42 +0300 Subject: [PATCH 173/284] Fix style --- src/Processors/DelayedPortsProcessor.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Processors/DelayedPortsProcessor.cpp b/src/Processors/DelayedPortsProcessor.cpp index ce21c5a018c..cb181a4e4ac 100644 --- a/src/Processors/DelayedPortsProcessor.cpp +++ b/src/Processors/DelayedPortsProcessor.cpp @@ -3,6 +3,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + DelayedPortsProcessor::DelayedPortsProcessor( const Block & header, size_t num_ports, const PortNumbers & delayed_ports, bool assert_main_ports_empty) : IProcessor(InputPorts(num_ports, header), From ca6ddfafc49f3b20908234da31047c8652a3570e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 27 Dec 2020 16:55:55 +0300 Subject: [PATCH 174/284] Fix style --- src/Functions/ExtractString.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Functions/ExtractString.h b/src/Functions/ExtractString.h index 94ee01ae609..8b88a5a7c37 100644 --- a/src/Functions/ExtractString.h +++ b/src/Functions/ExtractString.h @@ -16,11 +16,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - // used by FunctionsStringSimilarity and FunctionsStringHash // includes extracting ASCII ngram, UTF8 ngram, ASCII word and UTF8 word struct ExtractStringImpl From 5337d4e2dc8f7b808b519e3f6bd3cb76bcf05d49 Mon Sep 17 00:00:00 2001 From: awesomeleo Date: Sun, 27 Dec 2020 23:49:34 +0800 Subject: [PATCH 175/284] fix 18450, DB::Exception: Unsupported type FixedString --- src/Core/ExternalResultDescription.cpp | 3 +++ src/Core/ExternalResultDescription.h | 3 ++- src/Formats/MySQLBlockInputStream.cpp | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Core/ExternalResultDescription.cpp b/src/Core/ExternalResultDescription.cpp index 7165d73b7d0..4be80a352c7 100644 --- a/src/Core/ExternalResultDescription.cpp +++ b/src/Core/ExternalResultDescription.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -76,6 +77,8 @@ void ExternalResultDescription::init(const Block & sample_block_) types.emplace_back(ValueType::vtDecimal128, is_nullable); else if (typeid_cast *>(type)) types.emplace_back(ValueType::vtDecimal256, is_nullable); + else if (typeid_cast(type)) + types.emplace_back(ValueType::vtFixedString, is_nullable); else throw Exception{"Unsupported type " + type->getName(), ErrorCodes::UNKNOWN_TYPE}; } diff --git a/src/Core/ExternalResultDescription.h b/src/Core/ExternalResultDescription.h index f8ba2a6bba2..cc3b26ad841 100644 --- a/src/Core/ExternalResultDescription.h +++ b/src/Core/ExternalResultDescription.h @@ -30,7 +30,8 @@ struct ExternalResultDescription vtDecimal32, vtDecimal64, vtDecimal128, - vtDecimal256 + vtDecimal256, + vtFixedString }; Block sample_block; diff --git a/src/Formats/MySQLBlockInputStream.cpp b/src/Formats/MySQLBlockInputStream.cpp index 2ff8e8e5fb2..026f688a67f 100644 --- a/src/Formats/MySQLBlockInputStream.cpp +++ b/src/Formats/MySQLBlockInputStream.cpp @@ -8,6 +8,7 @@ # include # include # include +# include # include # include # include @@ -110,6 +111,9 @@ namespace data_type.deserializeAsWholeText(column, buffer, FormatSettings{}); break; } + case ValueType::vtFixedString: + assert_cast(column).insertData(value.data(), value.size()); + break; } } From 8c368688a847d7e87f6d34108872abe5d2e08edd Mon Sep 17 00:00:00 2001 From: Anna Date: Sun, 27 Dec 2020 23:49:05 +0300 Subject: [PATCH 176/284] update --- .../en/sql-reference/functions/string-functions.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 21e6f224b67..ddb27756df4 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -562,6 +562,8 @@ Result: Escape characters to place string into XML text node or attribute. +The following five XML predefined entities will be replaced: `<`, `&`, `>`, `"`, `\'`. + **Syntax** ``` sql @@ -570,11 +572,11 @@ encodeXMLComponent(x) **Parameters** -- `x` — Sequence of characters. [String](../../sql-reference/data-types/string.md). +- `x` — The sequence of characters. [String](../../sql-reference/data-types/string.md). **Returned value(s)** -- Sequence of characters. +- The sequence of characters with escape characters. Type: [String](../../sql-reference/data-types/string.md). @@ -583,11 +585,19 @@ Type: [String](../../sql-reference/data-types/string.md). Query: ``` sql +SELECT encodeXMLComponent('Hello, "world"!'); +SELECT encodeXMLComponent('<123>'); +SELECT encodeXMLComponent('&clickhouse'); +SELECT encodeXMLComponent('\'foo\''); ``` Result: ``` text +Hello, "world"! +<123> +&clickhouse +'foo' ``` [Original article](https://clickhouse.tech/docs/en/query_language/functions/string_functions/) From 2905f70cce59728bb91f415aeda7d6f4506c80b6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 28 Dec 2020 12:56:38 +0300 Subject: [PATCH 177/284] fix aliases in partition by/order by --- programs/client/QueryFuzzer.cpp | 4 +- src/Interpreters/ActionsVisitor.cpp | 6 +-- src/Interpreters/ExpressionAnalyzer.cpp | 40 +++++++------------ src/Interpreters/QueryNormalizer.cpp | 40 +++++++++++-------- src/Interpreters/QueryNormalizer.h | 2 +- src/Parsers/ASTFunction.cpp | 9 +++-- src/Parsers/ASTFunction.h | 22 ++++++++-- src/Parsers/ExpressionElementParsers.cpp | 9 +++-- .../01591_window_functions.reference | 16 +++++++- .../0_stateless/01591_window_functions.sql | 6 ++- 10 files changed, 93 insertions(+), 61 deletions(-) diff --git a/programs/client/QueryFuzzer.cpp b/programs/client/QueryFuzzer.cpp index 53ede4a3d92..fe0b6a975ce 100644 --- a/programs/client/QueryFuzzer.cpp +++ b/programs/client/QueryFuzzer.cpp @@ -405,8 +405,8 @@ void QueryFuzzer::fuzz(ASTPtr & ast) if (fn->is_window_function) { - fuzzColumnLikeExpressionList(fn->window_partition_by); - fuzzOrderByList(fn->window_order_by); + fuzzColumnLikeExpressionList(fn->window_partition_by.get()); + fuzzOrderByList(fn->window_order_by.get()); } fuzz(fn->children); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index bb47129f58c..00600bebf07 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -738,15 +738,13 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & if (node.is_window_function) { // Also add columns from PARTITION BY and ORDER BY of window functions. - // Requiring a constant reference to a shared pointer to non-const AST - // doesn't really look sane, but the visitor does indeed require it. if (node.window_partition_by) { - visit(node.window_partition_by->clone(), data); + visit(node.window_partition_by, data); } if (node.window_order_by) { - visit(node.window_order_by->clone(), data); + visit(node.window_order_by, data); } // Also manually add columns for arguments of the window function itself. diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index c10a8313bee..a80b7799a98 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -983,36 +983,14 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments( getRootActionsNoMakeSet(f.function_node->clone(), true /* no_subqueries */, step.actions()); - // FIXME rewrite this comment - // 1.2) result of window function: an empty INPUT. - // It is an aggregate function, so it won't be added by getRootActions. - // This is something of a hack. Other options: - // a] do it like aggregate function -- break the chain of actions - // and manually add window functions to the starting list of - // input columns. Logically this is similar to what we're doing - // now, but would require to split the window function processing - // into a full-fledged step after plain functions. This would be - // somewhat cumbersome. With INPUT hack we can avoid a separate - // step and pretend that window functions are almost "normal" - // select functions. The limitation of both these ways is that - // we can't reference window functions in other SELECT - // expressions. - // b] add a WINDOW action type, then sort, then split the chain on - // each WINDOW action and insert the Window pipeline between the - // Expression pipelines. This is a "proper" way that would allow - // us to depend on window functions in other functions. But it's - // complicated so I avoid doing it for now. - columns_after_window.push_back({f.column_name, - f.aggregate_function->getReturnType()}); - + // 2.1) function arguments; for (const auto & a : f.function_node->arguments->children) { - // 2.1) function arguments; step.required_output.push_back(a->getColumnName()); } } - // 2.3) PARTITION BY and ORDER BY columns. + // 2.1) PARTITION BY and ORDER BY columns. for (const auto & c : w.full_sort_description) { step.required_output.push_back(c.column_name); @@ -1433,11 +1411,23 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( { query_analyzer.appendWindowFunctionsArguments(chain, only_types || !first_stage); - + // Build a list of output columns of the window step. + // 1) We need the columns that are the output of ExpressionActions. for (const auto & x : chain.getLastActions()->getNamesAndTypesList()) { query_analyzer.columns_after_window.push_back(x); } + // 2) We also have to manually add the output of the window function + // to the list of the output columns of the window step, because the + // window functions are not in the ExpressionActions. + for (const auto & [_, w] : query_analyzer.window_descriptions) + { + for (const auto & f : w.window_functions) + { + query_analyzer.columns_after_window.push_back( + {f.column_name, f.aggregate_function->getReturnType()}); + } + } before_window = chain.getLastActions(); finalize_chain(chain); diff --git a/src/Interpreters/QueryNormalizer.cpp b/src/Interpreters/QueryNormalizer.cpp index 213e354a13f..848681953e4 100644 --- a/src/Interpreters/QueryNormalizer.cpp +++ b/src/Interpreters/QueryNormalizer.cpp @@ -148,7 +148,7 @@ void QueryNormalizer::visit(ASTSelectQuery & select, const ASTPtr &, Data & data /// Don't go into select query. It processes children itself. /// Do not go to the left argument of lambda expressions, so as not to replace the formal parameters /// on aliases in expressions of the form 123 AS x, arrayMap(x -> 1, [2]). -void QueryNormalizer::visitChildren(ASTPtr & node, Data & data) +void QueryNormalizer::visitChildren(IAST * node, Data & data) { if (auto * func_node = node->as()) { @@ -159,27 +159,33 @@ void QueryNormalizer::visitChildren(ASTPtr & node, Data & data) /// Don't go into query argument. return; } + /// We skip the first argument. We also assume that the lambda function can not have parameters. + size_t first_pos = 0; + if (func_node->name == "lambda") + first_pos = 1; - // Process all function node children (arguments + window definition for - // window functions). - // We have to skip the first argument if it's the "lambda" function, - // because it contains formal parameters like x->x + 1; - const IAST * argument_to_skip = nullptr; - if (func_node->name == "lambda" - && func_node->arguments - && !func_node->arguments->children.empty()) + if (func_node->arguments) { - argument_to_skip = func_node->arguments->children[0].get(); - } + auto & func_children = func_node->arguments->children; - for (auto & child : func_node->children) - { - if (child.get() != argument_to_skip - && needVisitChild(child)) + for (size_t i = first_pos; i < func_children.size(); ++i) { - visit(child, data); + auto & child = func_children[i]; + + if (needVisitChild(child)) + visit(child, data); } } + + if (func_node->window_partition_by) + { + visitChildren(func_node->window_partition_by.get(), data); + } + + if (func_node->window_order_by) + { + visitChildren(func_node->window_order_by.get(), data); + } } else if (!node->as()) { @@ -225,7 +231,7 @@ void QueryNormalizer::visit(ASTPtr & ast, Data & data) if (ast.get() != initial_ast.get()) visit(ast, data); else - visitChildren(ast, data); + visitChildren(ast.get(), data); current_asts.erase(initial_ast.get()); current_asts.erase(ast.get()); diff --git a/src/Interpreters/QueryNormalizer.h b/src/Interpreters/QueryNormalizer.h index 951628d0738..e481f76ca8e 100644 --- a/src/Interpreters/QueryNormalizer.h +++ b/src/Interpreters/QueryNormalizer.h @@ -69,7 +69,7 @@ private: static void visit(ASTTablesInSelectQueryElement &, const ASTPtr &, Data &); static void visit(ASTSelectQuery &, const ASTPtr &, Data &); - static void visitChildren(ASTPtr & node, Data & data); + static void visitChildren(IAST * node, Data & data); }; } diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 980c395c7aa..d9159117b2d 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -67,17 +67,20 @@ ASTPtr ASTFunction::clone() const if (window_name) { - res->set(res->window_name, window_name->clone()); + res->window_name = window_name->clone(); + res->children.push_back(res->window_name); } if (window_partition_by) { - res->set(res->window_partition_by, window_partition_by->clone()); + res->window_partition_by = window_partition_by->clone(); + res->children.push_back(res->window_partition_by); } if (window_order_by) { - res->set(res->window_order_by, window_order_by->clone()); + res->window_order_by = window_order_by->clone(); + res->children.push_back(res->window_order_by); } return res; diff --git a/src/Parsers/ASTFunction.h b/src/Parsers/ASTFunction.h index 38e5f3f095c..4c20309fcb9 100644 --- a/src/Parsers/ASTFunction.h +++ b/src/Parsers/ASTFunction.h @@ -21,9 +21,25 @@ public: ASTPtr parameters; bool is_window_function = false; - ASTIdentifier * window_name; - ASTExpressionList * window_partition_by; - ASTExpressionList * window_order_by; + + // We have to make these fields ASTPtr because this is what the visitors + // expect. Some of them take const ASTPtr & (makes no sense), and some + // take ASTPtr & and modify it. I don't understand how the latter is + // compatible with also having an owning `children` array -- apparently it + // leads to some dangling children that are not referenced by the fields of + // the AST class itself. Some older code hints at the idea of having + // ownership in `children` only, and making the class fields to be raw + // pointers of proper type (see e.g. IAST::set), but this is not compatible + // with the visitor interface. + + // ASTIdentifier + ASTPtr window_name; + + // ASTExpressionList + ASTPtr window_partition_by; + + // ASTExpressionList of + ASTPtr window_order_by; /// do not print empty parentheses if there are no args - compatibility with new AST for data types and engine names. bool no_empty_args = false; diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 726e28005e3..7c82c4aca1e 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -419,7 +419,8 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe ParserIdentifier window_name_parser; if (window_name_parser.parse(pos, window_name_ast, expected)) { - function->set(function->window_name, window_name_ast); + function->children.push_back(window_name_ast); + function->window_name = window_name_ast; return true; } else @@ -442,7 +443,8 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe ASTPtr partition_by_ast; if (columns_partition_by.parse(pos, partition_by_ast, expected)) { - function->set(function->window_partition_by, partition_by_ast); + function->children.push_back(partition_by_ast); + function->window_partition_by = partition_by_ast; } else { @@ -455,7 +457,8 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe ASTPtr order_by_ast; if (columns_order_by.parse(pos, order_by_ast, expected)) { - function->set(function->window_order_by, order_by_ast); + function->children.push_back(order_by_ast); + function->window_order_by = order_by_ast; } else { diff --git a/tests/queries/0_stateless/01591_window_functions.reference b/tests/queries/0_stateless/01591_window_functions.reference index 6eb94856645..ce56860ed8b 100644 --- a/tests/queries/0_stateless/01591_window_functions.reference +++ b/tests/queries/0_stateless/01591_window_functions.reference @@ -122,14 +122,14 @@ select * from (select * from numbers(5) order by rand()) order by count() over ( 2 3 4 -select * from (select * from numbers(5) order by rand()) group by number order by sum(any(number)) over (order by min(number) desc) desc; +select * from (select * from numbers(5) order by rand()) group by number order by sum(any(number + 1)) over (order by min(number) desc) desc; -- different windows -- an explain test would also be helpful, but it's too immature now and I don't -- want to change reference all the time -1 0 +1 2 3 4 @@ -204,4 +204,16 @@ select distinct sum(0) over () from numbers(2); 0 select distinct any(number) over () from numbers(2); +-- Various kinds of aliases are properly substituted into various parts of window +-- function definition. + 0 +with number + 1 as x select intDiv(number, 3) as y, sum(x + y) over (partition by y order by x) from numbers(7); + +0 1 +0 3 +0 6 +1 5 +1 11 +1 18 +2 9 diff --git a/tests/queries/0_stateless/01591_window_functions.sql b/tests/queries/0_stateless/01591_window_functions.sql index 5039925aa42..082a6652a65 100644 --- a/tests/queries/0_stateless/01591_window_functions.sql +++ b/tests/queries/0_stateless/01591_window_functions.sql @@ -41,7 +41,7 @@ select * from (select * from numbers(5) order by rand()) order by count() over ( -- Aggregate functions as window function arguments. This query is semantically -- the same as the above one, only we replace `number` with -- `any(number) group by number` and so on. -select * from (select * from numbers(5) order by rand()) group by number order by sum(any(number)) over (order by min(number) desc) desc; +select * from (select * from numbers(5) order by rand()) group by number order by sum(any(number + 1)) over (order by min(number) desc) desc; -- different windows -- an explain test would also be helpful, but it's too immature now and I don't @@ -66,3 +66,7 @@ select count(1) over (), max(number + 1) over () from numbers(3); -- Should work in DISTINCT select distinct sum(0) over () from numbers(2); select distinct any(number) over () from numbers(2); + +-- Various kinds of aliases are properly substituted into various parts of window +-- function definition. +with number + 1 as x select intDiv(number, 3) as y, sum(x + y) over (partition by y order by x) from numbers(7); From 4adbdf364e64a20001925528dd1f4aeddbec577a Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Thu, 17 Dec 2020 17:33:47 +0800 Subject: [PATCH 178/284] Fix bug for orderby withfill with limit clause --- src/Interpreters/InterpreterSelectQuery.cpp | 15 ++++++++++++++- .../01614_with_fill_with_limit.reference | 4 ++++ .../0_stateless/01614_with_fill_with_limit.sql | 15 +++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01614_with_fill_with_limit.reference create mode 100644 tests/queries/0_stateless/01614_with_fill_with_limit.sql diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 38cc19a00d6..5e895bf732b 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1029,10 +1029,23 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu /** Optimization - if there are several sources and there is LIMIT, then first apply the preliminary LIMIT, * limiting the number of rows in each up to `offset + limit`. */ + bool has_withfill = false; + if (query.orderBy()) + { + SortDescription order_descr = getSortDescription(query, *context); + for (auto & desc : order_descr) + if (desc.with_fill) + { + has_withfill = true; + break; + } + } + bool has_prelimit = false; if (!to_aggregation_stage && query.limitLength() && !query.limit_with_ties && !hasWithTotalsInAnySubqueryInFromClause(query) && - !query.arrayJoinExpressionList() && !query.distinct && !expressions.hasLimitBy() && !settings.extremes) + !query.arrayJoinExpressionList() && !query.distinct && !expressions.hasLimitBy() && !settings.extremes && + !has_withfill) { executePreLimit(query_plan, false); has_prelimit = true; diff --git a/tests/queries/0_stateless/01614_with_fill_with_limit.reference b/tests/queries/0_stateless/01614_with_fill_with_limit.reference new file mode 100644 index 00000000000..451e076552e --- /dev/null +++ b/tests/queries/0_stateless/01614_with_fill_with_limit.reference @@ -0,0 +1,4 @@ +1 original +2 +1 original +2 diff --git a/tests/queries/0_stateless/01614_with_fill_with_limit.sql b/tests/queries/0_stateless/01614_with_fill_with_limit.sql new file mode 100644 index 00000000000..119117af287 --- /dev/null +++ b/tests/queries/0_stateless/01614_with_fill_with_limit.sql @@ -0,0 +1,15 @@ +SELECT + toFloat32(number % 10) AS n, + 'original' AS source +FROM numbers(10) +WHERE (number % 3) = 1 +ORDER BY n ASC WITH FILL STEP 1 +LIMIT 2; + +SELECT + toFloat32(number % 10) AS n, + 'original' AS source +FROM numbers(10) +WHERE (number % 3) = 1 +ORDER BY n ASC WITH FILL STEP 1 +LIMIT 2 WITH TIES; From 1c52fdb265ffbbd8387c10fd26e444c75451f634 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 28 Dec 2020 13:08:38 +0300 Subject: [PATCH 179/284] cleanup --- src/Interpreters/QueryNormalizer.cpp | 4 ++-- tests/performance/window_functions.xml | 20 +++++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/Interpreters/QueryNormalizer.cpp b/src/Interpreters/QueryNormalizer.cpp index 848681953e4..38b54de2130 100644 --- a/src/Interpreters/QueryNormalizer.cpp +++ b/src/Interpreters/QueryNormalizer.cpp @@ -176,12 +176,12 @@ void QueryNormalizer::visitChildren(IAST * node, Data & data) visit(child, data); } } - + if (func_node->window_partition_by) { visitChildren(func_node->window_partition_by.get(), data); } - + if (func_node->window_order_by) { visitChildren(func_node->window_order_by.get(), data); diff --git a/tests/performance/window_functions.xml b/tests/performance/window_functions.xml index dcc873b5029..f42345d0696 100644 --- a/tests/performance/window_functions.xml +++ b/tests/performance/window_functions.xml @@ -12,11 +12,11 @@ First with LIMIT BY, next with window functions. --> @@ -24,16 +24,14 @@ From 67798c8811b63cd78e728499f17ea2e7fdb6d3b3 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Mon, 28 Dec 2020 13:39:51 +0300 Subject: [PATCH 180/284] Update docs/ru/commercial/cloud.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/commercial/cloud.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/commercial/cloud.md b/docs/ru/commercial/cloud.md index 885aac97312..024297da7d8 100644 --- a/docs/ru/commercial/cloud.md +++ b/docs/ru/commercial/cloud.md @@ -22,7 +22,7 @@ toc_title: "\u041f\u043e\u0441\u0442\u0430\u0432\u0449\u0438\u043a\u0438\u0020\u [Altinity.Cloud](https://altinity.com/cloud-database/) — это полностью управляемый ClickHouse-as-a-Service для публичного облака Amazon. -- Быстрое развертывание кластеров ClickHouse на ресурсах Amazon +- быстрое развертывание кластеров ClickHouse на ресурсах Amazon. - Легкое горизонтальное масштабирование также, как и вертикальное масштабирование узлов - Изолированные виртуальные сети для каждого клиента с общедоступным эндпоинтом или пирингом VPC - Настраиваемые типы и объемы хранилищ From 5943a251b1f59cdd9f2d52209478f7f852d07254 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Mon, 28 Dec 2020 13:39:57 +0300 Subject: [PATCH 181/284] Update docs/ru/commercial/cloud.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/commercial/cloud.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/commercial/cloud.md b/docs/ru/commercial/cloud.md index 024297da7d8..8fca835450c 100644 --- a/docs/ru/commercial/cloud.md +++ b/docs/ru/commercial/cloud.md @@ -23,7 +23,7 @@ toc_title: "\u041f\u043e\u0441\u0442\u0430\u0432\u0449\u0438\u043a\u0438\u0020\u [Altinity.Cloud](https://altinity.com/cloud-database/) — это полностью управляемый ClickHouse-as-a-Service для публичного облака Amazon. - быстрое развертывание кластеров ClickHouse на ресурсах Amazon. -- Легкое горизонтальное масштабирование также, как и вертикальное масштабирование узлов +- легкое горизонтальное масштабирование также, как и вертикальное масштабирование узлов. - Изолированные виртуальные сети для каждого клиента с общедоступным эндпоинтом или пирингом VPC - Настраиваемые типы и объемы хранилищ - Cross-AZ масштабирование для повышения производительности и обеспечения высокой доступности From 281c767f30b029d5af19de80207cd8aec1d48a65 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Mon, 28 Dec 2020 13:40:05 +0300 Subject: [PATCH 182/284] Update docs/ru/commercial/cloud.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/commercial/cloud.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/commercial/cloud.md b/docs/ru/commercial/cloud.md index 8fca835450c..b132aa58067 100644 --- a/docs/ru/commercial/cloud.md +++ b/docs/ru/commercial/cloud.md @@ -24,7 +24,7 @@ toc_title: "\u041f\u043e\u0441\u0442\u0430\u0432\u0449\u0438\u043a\u0438\u0020\u - быстрое развертывание кластеров ClickHouse на ресурсах Amazon. - легкое горизонтальное масштабирование также, как и вертикальное масштабирование узлов. -- Изолированные виртуальные сети для каждого клиента с общедоступным эндпоинтом или пирингом VPC +- изолированные виртуальные сети для каждого клиента с общедоступным эндпоинтом или пирингом VPC. - Настраиваемые типы и объемы хранилищ - Cross-AZ масштабирование для повышения производительности и обеспечения высокой доступности - Встроенный мониторинг и редактор SQL-запросов From ccaea5fa423fe2d363fcecc943bbf944d6858010 Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Mon, 28 Dec 2020 19:09:41 +0800 Subject: [PATCH 183/284] Make Date > 2106-02-07 overflow as 0 --- base/common/DateLUTImpl.h | 4 ++++ .../01631_date_overflow_as_partition_key.reference | 2 ++ .../01631_date_overflow_as_partition_key.sql | 11 +++++++++++ 3 files changed, 17 insertions(+) create mode 100644 tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference create mode 100644 tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 97301fb1bee..1d82f668a49 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -673,6 +673,10 @@ public: if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31)) return DayNum(0); // TODO (nemkov, DateTime64 phase 2): implement creating real date for year outside of LUT range. + // The day after 2106-02-07 will not stored fully as struct Values, so just overflow it as 0 + if (unlikely(year == DATE_LUT_MAX_YEAR && (month > 2 || day_of_month > 7))) + return DayNum(0); + return DayNum(years_months_lut[(year - DATE_LUT_MIN_YEAR) * 12 + month - 1] + day_of_month - 1); } diff --git a/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference new file mode 100644 index 00000000000..dbcd92da11c --- /dev/null +++ b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference @@ -0,0 +1,2 @@ +1970-01-01 1 +1970-01-01 1 diff --git a/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql new file mode 100644 index 00000000000..f252e10806a --- /dev/null +++ b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql @@ -0,0 +1,11 @@ +drop table if exists dt_overflow; + +create table dt_overflow(d Date, i int) engine MergeTree partition by d order by i; + +insert into dt_overflow values('2106-11-11', 1); + +insert into dt_overflow values('2106-11-12', 1); + +select * from dt_overflow; + +drop table if exists dt_overflow; From 2c146fe2cd255469426bc96db2aeaf9cde881597 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Mon, 28 Dec 2020 14:24:24 +0300 Subject: [PATCH 184/284] DOCSUP-5054: Fix PR comments and update list items. --- docs/en/commercial/cloud.md | 1 + docs/ru/commercial/cloud.md | 16 ++++++++-------- .../server-configuration-parameters/settings.md | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/en/commercial/cloud.md b/docs/en/commercial/cloud.md index 03c8b7645a3..9dd08124f64 100644 --- a/docs/en/commercial/cloud.md +++ b/docs/en/commercial/cloud.md @@ -21,6 +21,7 @@ toc_title: Cloud ## Altinity.Cloud {#altinity.cloud} [Altinity.Cloud](https://altinity.com/cloud-database/) is a fully managed ClickHouse-as-a-Service for the Amazon public cloud. + - Fast deployment of ClickHouse clusters on Amazon resources - Easy scale-out/scale-in as well as vertical scaling of nodes - Isolated per-tenant VPCs with public endpoint or VPC peering diff --git a/docs/ru/commercial/cloud.md b/docs/ru/commercial/cloud.md index b132aa58067..4f57592b4c7 100644 --- a/docs/ru/commercial/cloud.md +++ b/docs/ru/commercial/cloud.md @@ -12,11 +12,11 @@ toc_title: "\u041f\u043e\u0441\u0442\u0430\u0432\u0449\u0438\u043a\u0438\u0020\u [Yandex Managed Service for ClickHouse](https://cloud.yandex.ru/services/managed-clickhouse?utm_source=referrals&utm_medium=clickhouseofficialsite&utm_campaign=link3) предоставляет следующие ключевые возможности: -- Полностью управляемый сервис ZooKeeper для [репликации ClickHouse](../engines/table-engines/mergetree-family/replication.md) -- Выбор типа хранилища -- Реплики в разных зонах доступности -- Шифрование и изоляция -- Автоматизированное техническое обслуживание +- полностью управляемый сервис ZooKeeper для [репликации ClickHouse](../engines/table-engines/mergetree-family/replication.md) +- выбор типа хранилища +- реплики в разных зонах доступности +- шифрование и изоляция +- автоматизированное техническое обслуживание ## Altinity.Cloud {#altinity.cloud} @@ -25,8 +25,8 @@ toc_title: "\u041f\u043e\u0441\u0442\u0430\u0432\u0449\u0438\u043a\u0438\u0020\u - быстрое развертывание кластеров ClickHouse на ресурсах Amazon. - легкое горизонтальное масштабирование также, как и вертикальное масштабирование узлов. - изолированные виртуальные сети для каждого клиента с общедоступным эндпоинтом или пирингом VPC. -- Настраиваемые типы и объемы хранилищ -- Cross-AZ масштабирование для повышения производительности и обеспечения высокой доступности -- Встроенный мониторинг и редактор SQL-запросов +- настраиваемые типы и объемы хранилищ +- cross-az масштабирование для повышения производительности и обеспечения высокой доступности +- встроенный мониторинг и редактор SQL-запросов {## [Оригинальная статья](https://clickhouse.tech/docs/ru/commercial/cloud/) ##} diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index b75f0371223..d1911437baf 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -86,7 +86,7 @@ ClickHouse проверяет условия для `min_part_size` и `min_part Возможные значения: -- Положительное целое число. +- положительное целое число. Значение по умолчанию: `1073741824` (1 ГБ). From 50afe274614a86c400c2f50ea7f9ff5284b480ec Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 28 Dec 2020 14:46:22 +0300 Subject: [PATCH 185/284] More stable test --- .../01593_concurrent_alter_mutations_kill.sh | 18 +++++++++++++++++- ...rrent_alter_mutations_kill_many_replicas.sh | 17 ++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh index fe243a7e929..a521d90b768 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh @@ -45,7 +45,23 @@ timeout $TIMEOUT bash -c kill_mutation_thread 2> /dev/null & wait $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_mutate_kill" -$CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_kill MODIFY COLUMN value Int64 SETTINGS replication_alter_partitions_sync=2" + +# with timeout alter query can be not finished yet, so to execute new alter +# we use retries +counter=0 +while true; do + if $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_kill MODIFY COLUMN value Int64 SETTINGS replication_alter_partitions_sync=2" 2> /dev/null ; then + break + fi + + if [ "$counter" -gt 120 ] + then + break + fi + sleep 0.5 + counter=$(($counter + 1)) +done + $CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE concurrent_mutate_kill" $CLICKHOUSE_CLIENT --query "OPTIMIZE TABLE concurrent_mutate_kill FINAL" $CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM concurrent_mutate_kill" diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh index e9513557722..f5f751d6408 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh @@ -59,7 +59,22 @@ for i in $(seq $REPLICAS); do $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_kill_$i" done -$CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_kill_$i MODIFY COLUMN value Int64 SETTINGS replication_alter_partitions_sync=2" +# with timeout alter query can be not finished yet, so to execute new alter +# we use retries +counter=0 +while true; do + if $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_kill_1 MODIFY COLUMN value Int64 SETTINGS replication_alter_partitions_sync=2" 2> /dev/null ; then + break + fi + + if [ "$counter" -gt 120 ] + then + break + fi + sleep 0.5 + counter=$(($counter + 1)) +done + metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/test_01593_concurrent_kill/replicas/$i/' and name = 'metadata_version'") for i in $(seq $REPLICAS); do From 964e12d8b79d787e78c7bd7a1391e260bf3b65f8 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Mon, 28 Dec 2020 15:53:58 +0300 Subject: [PATCH 186/284] Fix --- .../ParallelParsingBlockInputStream.cpp | 313 ------------------ .../ParallelParsingBlockInputStream.h | 181 ---------- .../Impl/ParallelFormattingOutputFormat.h | 16 +- .../Impl/ParallelParsingInputFormat.cpp | 25 +- .../Formats/Impl/ParallelParsingInputFormat.h | 8 +- 5 files changed, 34 insertions(+), 509 deletions(-) delete mode 100644 src/DataStreams/ParallelParsingBlockInputStream.cpp delete mode 100644 src/DataStreams/ParallelParsingBlockInputStream.h diff --git a/src/DataStreams/ParallelParsingBlockInputStream.cpp b/src/DataStreams/ParallelParsingBlockInputStream.cpp deleted file mode 100644 index b7a0c3cab99..00000000000 --- a/src/DataStreams/ParallelParsingBlockInputStream.cpp +++ /dev/null @@ -1,313 +0,0 @@ -#include -#include -#include -#include -#include - -namespace DB -{ - -ParallelParsingBlockInputStream::ParallelParsingBlockInputStream(const Params & params) - : header(params.input_creator_params.sample), - row_input_format_params(params.input_creator_params.row_input_format_params), - format_settings(params.input_creator_params.settings), - input_processor_creator(params.input_processor_creator), - min_chunk_bytes(params.min_chunk_bytes), - original_buffer(params.read_buffer), - // Subtract one thread that we use for segmentation and one for - // reading. After that, must have at least two threads left for - // parsing. See the assertion below. - pool(std::max(2, static_cast(params.max_threads) - 2)), - file_segmentation_engine(params.file_segmentation_engine) -{ - // See comment above. - assert(params.max_threads >= 4); - - // One unit for each thread, including segmentator and reader, plus a - // couple more units so that the segmentation thread doesn't spuriously - // bump into reader thread on wraparound. - processing_units.resize(params.max_threads + 2); - - segmentator_thread = ThreadFromGlobalPool( - &ParallelParsingBlockInputStream::segmentatorThreadFunction, this, CurrentThread::getGroup()); -} - -ParallelParsingBlockInputStream::~ParallelParsingBlockInputStream() -{ - finishAndWait(); -} - -void ParallelParsingBlockInputStream::cancel(bool kill) -{ - /** - * Can be called multiple times, from different threads. Saturate the - * the kill flag with OR. - */ - if (kill) - is_killed = true; - is_cancelled = true; - - /* - * The format parsers themselves are not being cancelled here, so we'll - * have to wait until they process the current block. Given that the - * chunk size is on the order of megabytes, this shouldn't be too long. - * We can't call IInputFormat->cancel here, because the parser object is - * local to the parser thread, and we don't want to introduce any - * synchronization between parser threads and the other threads to get - * better performance. An ideal solution would be to add a callback to - * IInputFormat that checks whether it was cancelled. - */ - - finishAndWait(); -} - -void ParallelParsingBlockInputStream::scheduleParserThreadForUnitWithNumber(size_t ticket_number) -{ - pool.scheduleOrThrowOnError([this, ticket_number, group = CurrentThread::getGroup()]() - { - parserThreadFunction(group, ticket_number); - }); -} - -void ParallelParsingBlockInputStream::finishAndWait() -{ - finished = true; - - { - std::unique_lock lock(mutex); - segmentator_condvar.notify_all(); - reader_condvar.notify_all(); - } - - if (segmentator_thread.joinable()) - segmentator_thread.join(); - - try - { - pool.wait(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } -} - -void ParallelParsingBlockInputStream::segmentatorThreadFunction(ThreadGroupStatusPtr thread_group) -{ - SCOPE_EXIT( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); - if (thread_group) - CurrentThread::attachTo(thread_group); - - setThreadName("Segmentator"); - - try - { - while (!finished) - { - const auto current_unit_number = segmentator_ticket_number % processing_units.size(); - auto & unit = processing_units[current_unit_number]; - - { - std::unique_lock lock(mutex); - segmentator_condvar.wait(lock, - [&]{ return unit.status == READY_TO_INSERT || finished; }); - } - - if (finished) - { - break; - } - - assert(unit.status == READY_TO_INSERT); - - // Segmentating the original input. - unit.segment.resize(0); - - auto [have_more_data, currently_read_rows] = file_segmentation_engine( - original_buffer, unit.segment, min_chunk_bytes); - - unit.offset = successfully_read_rows_count; - successfully_read_rows_count += currently_read_rows; - - unit.is_last = !have_more_data; - unit.status = READY_TO_PARSE; - scheduleParserThreadForUnitWithNumber(segmentator_ticket_number); - ++segmentator_ticket_number; - - if (!have_more_data) - { - break; - } - } - } - catch (...) - { - onBackgroundException(successfully_read_rows_count); - } -} - -void ParallelParsingBlockInputStream::parserThreadFunction(ThreadGroupStatusPtr thread_group, size_t current_ticket_number) -{ - SCOPE_EXIT( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); - if (thread_group) - CurrentThread::attachTo(thread_group); - - setThreadName("ChunkParser"); - - const auto current_unit_number = current_ticket_number % processing_units.size(); - auto & unit = processing_units[current_unit_number]; - - try - { - /* - * This is kind of suspicious -- the input_process_creator contract with - * respect to multithreaded use is not clear, but we hope that it is - * just a 'normal' factory class that doesn't have any state, and so we - * can use it from multiple threads simultaneously. - */ - ReadBuffer read_buffer(unit.segment.data(), unit.segment.size(), 0); - auto format = input_processor_creator(read_buffer, header, row_input_format_params, format_settings); - format->setCurrentUnitNumber(current_ticket_number); - auto parser = std::make_unique(std::move(format)); - - unit.block_ext.block.clear(); - unit.block_ext.block_missing_values.clear(); - - // We don't know how many blocks will be. So we have to read them all - // until an empty block occurred. - Block block; - while (!finished && (block = parser->read()) != Block()) - { - unit.block_ext.block.emplace_back(block); - unit.block_ext.block_missing_values.emplace_back(parser->getMissingValues()); - } - - // We suppose we will get at least some blocks for a non-empty buffer, - // except at the end of file. Also see a matching assert in readImpl(). - assert(unit.is_last || !unit.block_ext.block.empty()); - - std::unique_lock lock(mutex); - unit.status = READY_TO_READ; - reader_condvar.notify_all(); - } - catch (...) - { - onBackgroundException(unit.offset); - } -} - -void ParallelParsingBlockInputStream::onBackgroundException(size_t offset) -{ - std::unique_lock lock(mutex); - if (!background_exception) - { - background_exception = std::current_exception(); - - if (ParsingException * e = exception_cast(background_exception)) - if (e->getLineNumber() != -1) - e->setLineNumber(e->getLineNumber() + offset); - } - tryLogCurrentException(__PRETTY_FUNCTION__); - finished = true; - reader_condvar.notify_all(); - segmentator_condvar.notify_all(); -} - -Block ParallelParsingBlockInputStream::readImpl() -{ - if (isCancelledOrThrowIfKilled() || finished) - { - /** - * Check for background exception and rethrow it before we return. - */ - std::unique_lock lock(mutex); - if (background_exception) - { - lock.unlock(); - cancel(false); - std::rethrow_exception(background_exception); - } - - return Block{}; - } - - const auto current_unit_number = reader_ticket_number % processing_units.size(); - auto & unit = processing_units[current_unit_number]; - - if (!next_block_in_current_unit.has_value()) - { - // We have read out all the Blocks from the previous Processing Unit, - // wait for the current one to become ready. - std::unique_lock lock(mutex); - reader_condvar.wait(lock, [&](){ return unit.status == READY_TO_READ || finished; }); - - if (finished) - { - /** - * Check for background exception and rethrow it before we return. - */ - if (background_exception) - { - lock.unlock(); - cancel(false); - std::rethrow_exception(background_exception); - } - - return Block{}; - } - - assert(unit.status == READY_TO_READ); - next_block_in_current_unit = 0; - } - - if (unit.block_ext.block.empty()) - { - /* - * Can we get zero blocks for an entire segment, when the format parser - * skips it entire content and does not create any blocks? Probably not, - * but if we ever do, we should add a loop around the above if, to skip - * these. Also see a matching assert in the parser thread. - */ - assert(unit.is_last); - finished = true; - return Block{}; - } - - assert(next_block_in_current_unit.value() < unit.block_ext.block.size()); - - Block res = std::move(unit.block_ext.block.at(*next_block_in_current_unit)); - last_block_missing_values = std::move(unit.block_ext.block_missing_values[*next_block_in_current_unit]); - - next_block_in_current_unit.value() += 1; - - if (*next_block_in_current_unit == unit.block_ext.block.size()) - { - // Finished reading this Processing Unit, move to the next one. - next_block_in_current_unit.reset(); - ++reader_ticket_number; - - if (unit.is_last) - { - // It it was the last unit, we're finished. - finished = true; - } - else - { - // Pass the unit back to the segmentator. - std::unique_lock lock(mutex); - unit.status = READY_TO_INSERT; - segmentator_condvar.notify_all(); - } - } - - return res; -} - - -} diff --git a/src/DataStreams/ParallelParsingBlockInputStream.h b/src/DataStreams/ParallelParsingBlockInputStream.h deleted file mode 100644 index 749de83b583..00000000000 --- a/src/DataStreams/ParallelParsingBlockInputStream.h +++ /dev/null @@ -1,181 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -namespace DB -{ - -class ReadBuffer; - -/** - * ORDER-PRESERVING parallel parsing of data formats. - * It splits original data into chunks. Then each chunk is parsed by different thread. - * The number of chunks equals to the number or parser threads. - * The size of chunk is equal to min_chunk_bytes_for_parallel_parsing setting. - * - * This stream has three kinds of threads: one segmentator, multiple parsers, - * and one reader thread -- that is, the one from which readImpl() is called. - * They operate one after another on parts of data called "processing units". - * One unit consists of buffer with raw data from file, filled by segmentator - * thread. This raw data is then parsed by a parser thread to form a number of - * Blocks. These Blocks are returned to the parent stream from readImpl(). - * After being read out, a processing unit is reused, to save on allocating - * memory for the raw buffer. The processing units are organized into a circular - * array to facilitate reuse and to apply backpressure on the segmentator thread - * -- after it runs out of processing units, it has to wait for the reader to - * read out the previous blocks. - * The outline of what the threads do is as follows: - * segmentator thread: - * 1) wait for the next processing unit to become empty - * 2) fill it with a part of input file - * 3) start a parser thread - * 4) repeat until eof - * parser thread: - * 1) parse the given raw buffer without any synchronization - * 2) signal that the given unit is ready to read - * 3) finish - * readImpl(): - * 1) wait for the next processing unit to become ready to read - * 2) take the blocks from the processing unit to return them to the caller - * 3) signal that the processing unit is empty - * 4) repeat until it encounters unit that is marked as "past_the_end" - * All threads must also check for cancel/eof/exception flags. - */ -class ParallelParsingBlockInputStream : public IBlockInputStream -{ -private: - using ReadCallback = std::function; - - using InputProcessorCreator = std::function; -public: - struct InputCreatorParams - { - const Block & sample; - const RowInputFormatParams & row_input_format_params; - const FormatSettings &settings; - }; - - struct Params - { - ReadBuffer & read_buffer; - const InputProcessorCreator & input_processor_creator; - const InputCreatorParams & input_creator_params; - FormatFactory::FileSegmentationEngine file_segmentation_engine; - size_t max_threads; - size_t min_chunk_bytes; - }; - - explicit ParallelParsingBlockInputStream(const Params & params); - ~ParallelParsingBlockInputStream() override; - - String getName() const override { return "ParallelParsing"; } - Block getHeader() const override { return header; } - - void cancel(bool kill) override; - -protected: - // Reader routine - Block readImpl() override; - - const BlockMissingValues & getMissingValues() const override - { - return last_block_missing_values; - } - -private: - const Block header; - const RowInputFormatParams row_input_format_params; - const FormatSettings format_settings; - const InputProcessorCreator input_processor_creator; - - const size_t min_chunk_bytes; - - /* - * This is declared as atomic to avoid UB, because parser threads access it - * without synchronization. - */ - std::atomic finished{false}; - - BlockMissingValues last_block_missing_values; - - // Original ReadBuffer to read from. - ReadBuffer & original_buffer; - - //Non-atomic because it is used in one thread. - std::optional next_block_in_current_unit; - size_t segmentator_ticket_number{0}; - size_t reader_ticket_number{0}; - - std::mutex mutex; - std::condition_variable reader_condvar; - std::condition_variable segmentator_condvar; - - // There are multiple "parsers", that's why we use thread pool. - ThreadPool pool; - // Reading and segmentating the file - ThreadFromGlobalPool segmentator_thread; - - // Function to segment the file. Then "parsers" will parse that segments. - FormatFactory::FileSegmentationEngine file_segmentation_engine; - - enum ProcessingUnitStatus - { - READY_TO_INSERT, - READY_TO_PARSE, - READY_TO_READ - }; - - struct BlockExt - { - std::vector block; - std::vector block_missing_values; - }; - - struct ProcessingUnit - { - explicit ProcessingUnit() - : status(ProcessingUnitStatus::READY_TO_INSERT) - { - } - - BlockExt block_ext; - Memory<> segment; - std::atomic status; - /// Needed for better exception message. - size_t offset = 0; - bool is_last{false}; - }; - - std::exception_ptr background_exception = nullptr; - - // We use deque instead of vector, because it does not require a move - // constructor, which is absent for atomics that are inside ProcessingUnit. - std::deque processing_units; - - - /// Compute it to have a more understandable error message. - size_t successfully_read_rows_count{0}; - - - void scheduleParserThreadForUnitWithNumber(size_t ticket_number); - void finishAndWait(); - - void segmentatorThreadFunction(ThreadGroupStatusPtr thread_group); - void parserThreadFunction(ThreadGroupStatusPtr thread_group, size_t current_ticket_number); - - // Save/log a background exception, set termination flag, wake up all - // threads. This function is used by segmentator and parsed threads. - // readImpl() is called from the main thread, so the exception handling - // is different. - void onBackgroundException(size_t offset); -}; - -} diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index bd224102bd9..705b000e27f 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -95,8 +95,11 @@ protected: IOutputFormat::finalized = true; addChunk(Chunk{}, ProcessingUnitType::FINALIZE); collector_finished.wait(); - if (background_exception) - std::rethrow_exception(background_exception); + { + std::unique_lock lock(mutex); + if (background_exception) + std::rethrow_exception(background_exception); + } } private: @@ -122,8 +125,11 @@ private: void addChunk(Chunk chunk, ProcessingUnitType type) { - if (background_exception) - std::rethrow_exception(background_exception); + { + std::unique_lock lock(mutex); + if (background_exception) + std::rethrow_exception(background_exception); + } const auto current_unit_number = writer_unit_number % processing_units.size(); auto & unit = processing_units[current_unit_number]; @@ -286,7 +292,7 @@ private: auto & unit = processing_units[current_unit_number]; assert(unit.status = READY_TO_FORMAT); - unit.segment.resize(0); + unit.segment.resize(DBMS_DEFAULT_BUFFER_SIZE); unit.actual_memory_size = 0; BufferWithOutsideMemory out_buffer(unit.segment); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 2c4a567b16f..040cbd1144f 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -38,7 +38,10 @@ void ParallelParsingInputFormat::segmentatorThreadFunction(ThreadGroupStatusPtr // Segmentating the original input. unit.segment.resize(0); - const bool have_more_data = file_segmentation_engine(in, unit.segment, min_chunk_bytes); + auto [have_more_data, currently_read_rows] = file_segmentation_engine(in, unit.segment, min_chunk_bytes); + + unit.offset = successfully_read_rows_count; + successfully_read_rows_count += currently_read_rows; unit.is_last = !have_more_data; unit.status = READY_TO_PARSE; @@ -51,7 +54,7 @@ void ParallelParsingInputFormat::segmentatorThreadFunction(ThreadGroupStatusPtr } catch (...) { - onBackgroundException(); + onBackgroundException(successfully_read_rows_count); } } @@ -64,13 +67,13 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa if (thread_group) CurrentThread::attachTo(thread_group); + const auto current_unit_number = current_ticket_number % processing_units.size(); + auto & unit = processing_units[current_unit_number]; + try { setThreadName("ChunkParser"); - const auto current_unit_number = current_ticket_number % processing_units.size(); - auto & unit = processing_units[current_unit_number]; - /* * This is kind of suspicious -- the input_process_creator contract with * respect to multithreaded use is not clear, but we hope that it is @@ -90,7 +93,9 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa Chunk chunk; while (!parsing_finished && (chunk = parser.getChunk()) != Chunk()) { - unit.chunk_ext.chunk.emplace_back(std::move(chunk)); + /// Variable chunk is moved, but it is not really used in the next iteration. + /// NOLINTNEXTLINE(bugprone-use-after-move) + unit.chunk_ext.chunk.emplace_back(std::move(chunk)); unit.chunk_ext.block_missing_values.emplace_back(parser.getMissingValues()); } @@ -104,12 +109,12 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa } catch (...) { - onBackgroundException(); + onBackgroundException(unit.offset); } } -void ParallelParsingInputFormat::onBackgroundException() +void ParallelParsingInputFormat::onBackgroundException(size_t offset) { tryLogCurrentException(__PRETTY_FUNCTION__); @@ -117,7 +122,11 @@ void ParallelParsingInputFormat::onBackgroundException() if (!background_exception) { background_exception = std::current_exception(); + if (ParsingException * e = exception_cast(background_exception)) + if (e->getLineNumber() != -1) + e->setLineNumber(e->getLineNumber() + offset); } + tryLogCurrentException(__PRETTY_FUNCTION__); parsing_finished = true; reader_condvar.notify_all(); segmentator_condvar.notify_all(); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index 0690638b97a..c33d6579eae 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -155,7 +155,6 @@ private: case IProcessor::Status::NeedData: break; case IProcessor::Status::Async: break; - case IProcessor::Status::Wait: break; case IProcessor::Status::ExpandPipeline: throw Exception("One of the parsers returned status " + IProcessor::statusToName(status) + " during parallel parsing", ErrorCodes::LOGICAL_ERROR); @@ -217,6 +216,8 @@ private: ChunkExt chunk_ext; Memory<> segment; std::atomic status; + /// Needed for better exception message. + size_t offset = 0; bool is_last{false}; }; @@ -226,6 +227,9 @@ private: /// constructor, which is absent for atomics that are inside ProcessingUnit. std::deque processing_units; + /// Compute it to have a more understandable error message. + size_t successfully_read_rows_count{0}; + void scheduleParserThreadForUnitWithNumber(size_t ticket_number) { @@ -265,7 +269,7 @@ private: /// threads. This function is used by segmentator and parsed threads. /// readImpl() is called from the main thread, so the exception handling /// is different. - void onBackgroundException(); + void onBackgroundException(size_t offset); }; } From 86678423906aa4bd202ffeb82f7d5c214cba2b71 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Mon, 28 Dec 2020 15:54:46 +0300 Subject: [PATCH 187/284] ya make update --- src/DataStreams/ya.make | 1 - src/Processors/ya.make | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/DataStreams/ya.make b/src/DataStreams/ya.make index 858bf7081e7..8648f233f26 100644 --- a/src/DataStreams/ya.make +++ b/src/DataStreams/ya.make @@ -35,7 +35,6 @@ SRCS( MongoDBBlockInputStream.cpp NativeBlockInputStream.cpp NativeBlockOutputStream.cpp - ParallelParsingBlockInputStream.cpp PushingToViewsBlockOutputStream.cpp RemoteBlockInputStream.cpp RemoteBlockOutputStream.cpp diff --git a/src/Processors/ya.make b/src/Processors/ya.make index 263c24ff35c..6e99589bf02 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -43,9 +43,9 @@ SRCS( Formats/Impl/MsgPackRowInputFormat.cpp Formats/Impl/MsgPackRowOutputFormat.cpp Formats/Impl/MySQLOutputFormat.cpp - Formats/Impl/NativeFormat.cpp Formats/Impl/NullFormat.cpp Formats/Impl/ODBCDriver2BlockOutputFormat.cpp + Formats/Impl/ParallelParsingInputFormat.cpp Formats/Impl/PostgreSQLOutputFormat.cpp Formats/Impl/PrettyBlockOutputFormat.cpp Formats/Impl/PrettyCompactBlockOutputFormat.cpp From f337be7ca2f02ed14959367e02f1e20ef98c581d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 28 Dec 2020 16:26:22 +0300 Subject: [PATCH 188/284] Remove unneeded example --- src/Interpreters/tests/CMakeLists.txt | 3 - src/Interpreters/tests/context.cpp | 90 --------------------------- 2 files changed, 93 deletions(-) delete mode 100644 src/Interpreters/tests/context.cpp diff --git a/src/Interpreters/tests/CMakeLists.txt b/src/Interpreters/tests/CMakeLists.txt index 2c8440299fb..1bc9d7fbacb 100644 --- a/src/Interpreters/tests/CMakeLists.txt +++ b/src/Interpreters/tests/CMakeLists.txt @@ -32,9 +32,6 @@ target_link_libraries (string_hash_map_aggregation PRIVATE dbms) add_executable (string_hash_set string_hash_set.cpp) target_link_libraries (string_hash_set PRIVATE dbms) -add_executable (context context.cpp) -target_link_libraries (context PRIVATE dbms) - add_executable (two_level_hash_map two_level_hash_map.cpp) target_include_directories (two_level_hash_map SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) target_link_libraries (two_level_hash_map PRIVATE dbms) diff --git a/src/Interpreters/tests/context.cpp b/src/Interpreters/tests/context.cpp deleted file mode 100644 index 9b908e26248..00000000000 --- a/src/Interpreters/tests/context.cpp +++ /dev/null @@ -1,90 +0,0 @@ -#include -/// #define BOOST_USE_UCONTEXT -#include -// #include -// #include -#include -#include - -void __attribute__((__noinline__)) foo(std::exception_ptr exception) -{ - if (exception) - std::rethrow_exception(exception); -} - -void __attribute__((__noinline__)) bar(int a) -{ - std::cout << StackTrace().toString() << std::endl; - - if (a > 0) - throw DB::Exception(0, "hello"); -} - -void __attribute__((__noinline__)) gar(int a) -{ - char buf[1024]; - buf[1023] = a & 255; - if (a > 2) - return gar(a - 1); - else - bar(a); -} - -int main(int, char **) -try { - namespace ctx=boost::context; - int a; - std::exception_ptr exception; - // ctx::protected_fixedsize allocator - // ctx::pooled_fixedsize_stack(1024 * 64 + 2 * 2 * 1024 * 1024 * 16, 1) - ctx::fiber source{std::allocator_arg_t(), FiberStack(), [&](ctx::fiber&& sink) - { - a=0; - int b=1; - for (size_t i = 0; i < 9; ++i) - { - sink=std::move(sink).resume(); - int next=a+b; - a=b; - b=next; - } - try - { - gar(1024); - } - catch (...) - { - std::cout << "Saving exception\n"; - exception = std::current_exception(); - } - return std::move(sink); - }}; - - for (int j=0;j<10;++j) - { - try - { - source=std::move(source).resume(); - } - catch (DB::Exception & e) - { - std::cout << "Caught exception in resume " << e.getStackTraceString() << std::endl; - } - std::cout << a << " "; - } - - std::cout << std::endl; - - try - { - foo(exception); - } - catch (const DB::Exception & e) - { - std::cout << e.getStackTraceString() << std::endl; - } -} -catch (...) -{ - std::cerr << "Uncaught exception\n"; -} From 819bd8059b9dcc3804b69f3c5a9c35953b5c55b8 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 28 Dec 2020 16:57:47 +0300 Subject: [PATCH 189/284] Update docs/ru/operations/server-configuration-parameters/settings.md --- docs/ru/operations/server-configuration-parameters/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index d1911437baf..902dab0310d 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -82,7 +82,7 @@ ClickHouse проверяет условия для `min_part_size` и `min_part ## core_dump {#server_configuration_parameters-core_dump} -Задает мягкое ограничение для размера файла дампа ядра. +Задает мягкое ограничение для размера файла дампа памяти. Возможные значения: From 35d62b73a7f766b7e9447e5e9a32e32baf2cf907 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Mon, 28 Dec 2020 18:47:44 +0300 Subject: [PATCH 190/284] Check for CLICKHOUSE_CLIENT_OPT env before setting it (#18574) --- tests/clickhouse-test | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 3e4429449a7..da7d055f250 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -873,6 +873,7 @@ if __name__ == '__main__': parser.add_argument('--use-skip-list', action='store_true', default=False, help="Use skip list to skip tests if found") parser.add_argument('--db-engine', help='Database engine name') + parser.add_argument('--antlr', action='store_true', default=False, dest='antlr', help='Use new ANTLR parser in tests') parser.add_argument('--no-stateless', action='store_true', help='Disable all stateless tests') parser.add_argument('--no-stateful', action='store_true', help='Disable all stateful tests') parser.add_argument('--skip', nargs='+', help="Skip these tests") @@ -886,7 +887,6 @@ if __name__ == '__main__': group=parser.add_mutually_exclusive_group(required=False) group.add_argument('--shard', action='store_true', default=None, dest='shard', help='Run sharding related tests (required to clickhouse-server listen 127.0.0.2 127.0.0.3)') group.add_argument('--no-shard', action='store_false', default=None, dest='shard', help='Do not run shard related tests') - parser.add_argument('--antlr', action='store_true', default=False, dest='antlr', help='Use new ANTLR parser in tests') args = parser.parse_args() @@ -967,7 +967,10 @@ if __name__ == '__main__': os.environ['CLICKHOUSE_URL_PARAMS'] += get_additional_client_options_url(args) if args.antlr: - os.environ['CLICKHOUSE_CLIENT_OPT'] += ' --use_antlr_parser=1' + if 'CLICKHOUSE_CLIENT_OPT' in os.environ: + os.environ['CLICKHOUSE_CLIENT_OPT'] += ' --use_antlr_parser=1' + else: + os.environ['CLICKHOUSE_CLIENT_OPT'] = '--use_antlr_parser=1' if args.extract_from_config is None: if os.access(args.binary + '-extract-from-config', os.X_OK): From 2dde73f70007b9bf139348bb55ff59bd33d9540e Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Mon, 28 Dec 2020 19:52:54 +0300 Subject: [PATCH 191/284] better --- programs/client/Client.cpp | 5 +---- src/Formats/FormatFactory.cpp | 5 ++--- .../Formats/Impl/JSONEachRowRowInputFormat.cpp | 2 +- src/Processors/Formats/Impl/NativeFormat.h | 1 + .../Formats/Impl/ParallelFormattingOutputFormat.h | 15 ++++++++++----- .../Formats/Impl/ParallelParsingInputFormat.cpp | 2 +- .../Formats/Impl/ParallelParsingInputFormat.h | 8 +++++++- 7 files changed, 23 insertions(+), 15 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 777db6126a9..328855adcc0 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1364,9 +1364,6 @@ private: if (with_output && with_output->settings_ast) apply_query_settings(*with_output->settings_ast); - // if (context.getSettingsRef().output_format_parallel_formatting) - // need_render_progress = false; - connection->forceConnected(connection_parameters.timeouts); ASTPtr input_function; @@ -1939,7 +1936,7 @@ private: if (!is_interactive && !need_render_progress) block_out_stream = context.getOutputFormatParallelIfPossible(current_format, *out_buf, block); - + if (!block_out_stream) block_out_stream = context.getOutputFormat(current_format, *out_buf, block); diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 17962ba9a97..cfcbd9fb1c1 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -224,7 +224,7 @@ BlockOutputStreamPtr FormatFactory::getOutputParallelIfPossible(const String & n /** TODO: Materialization is needed, because formats can use the functions `IDataType`, * which only work with full columns. */ - + auto formatter_creator = [output_getter, sample, callback, format_settings] (WriteBuffer & output) -> OutputFormatPtr { return output_getter(output, sample, {std::move(callback)}, format_settings);}; @@ -244,7 +244,6 @@ BlockOutputStreamPtr FormatFactory::getOutputParallelIfPossible(const String & n } - BlockOutputStreamPtr FormatFactory::getOutput(const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback, const std::optional & _format_settings) const @@ -256,7 +255,7 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, { throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); } - + auto format = getOutputFormat(name, buf, sample, context, std::move(callback), format_settings); return std::make_shared(std::make_shared(format), sample); } diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp index 7cf0a306899..bd22718b57e 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp @@ -246,7 +246,7 @@ bool JSONEachRowRowInputFormat::readRow(MutableColumns & columns, RowReadExtensi /// Semicolon is added for convenience as it could be used at end of INSERT query. bool is_first_row = getTotalRows() == 1; - if (!in.eof()) + if (!in.eof()) { /// There may be optional ',' (but not before the first row) if (!is_first_row && *in.position() == ',') diff --git a/src/Processors/Formats/Impl/NativeFormat.h b/src/Processors/Formats/Impl/NativeFormat.h index 86fafd4e2c3..664252c268b 100644 --- a/src/Processors/Formats/Impl/NativeFormat.h +++ b/src/Processors/Formats/Impl/NativeFormat.h @@ -1,3 +1,4 @@ +#pragma once #include #include #include diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 705b000e27f..6461caab17d 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -52,7 +52,7 @@ public: ~ParallelFormattingOutputFormat() override { need_flush = true; - if (!IOutputFormat::finalized) + if (!IOutputFormat::finalized) finalize(); finishAndWait(); } @@ -64,12 +64,12 @@ public: need_flush = true; } - void doWritePrefix() override + void doWritePrefix() override { addChunk(Chunk{}, ProcessingUnitType::START); } - void onCancel() override + void onCancel() override { finishAndWait(); } @@ -260,7 +260,7 @@ private: IOutputFormat::flush(); ++collector_unit_number; - + { /// Notify other threads. std::lock_guard lock(mutex); @@ -292,13 +292,18 @@ private: auto & unit = processing_units[current_unit_number]; assert(unit.status = READY_TO_FORMAT); + /// We want to preallocate memory buffer (increase capacity) + /// and put the pointer at the beginning of the buffer + /// FIXME: Implement reserve() method in Memory. unit.segment.resize(DBMS_DEFAULT_BUFFER_SIZE); + unit.segment.resize(0); + unit.actual_memory_size = 0; BufferWithOutsideMemory out_buffer(unit.segment); auto formatter = internal_formatter_creator(out_buffer); - switch (unit.type) + switch (unit.type) { case ProcessingUnitType::START : { diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 040cbd1144f..cdefd00ac69 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -95,7 +95,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa { /// Variable chunk is moved, but it is not really used in the next iteration. /// NOLINTNEXTLINE(bugprone-use-after-move) - unit.chunk_ext.chunk.emplace_back(std::move(chunk)); + unit.chunk_ext.chunk.emplace_back(std::move(chunk)); unit.chunk_ext.block_missing_values.emplace_back(parser.getMissingValues()); } diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index c33d6579eae..4efe8b465de 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -13,6 +13,12 @@ namespace DB { + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + class Context; /** @@ -52,7 +58,7 @@ class Context; class ParallelParsingInputFormat : public IInputFormat { public: - /* Used to recreate parser on every new data piece. */ + /* Used to recreate parser on every new data piece.*/ using InternalParserCreator = std::function; struct Params From d293ae5e8e2b7c1d3f8df47064c301fa72000192 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Mon, 28 Dec 2020 20:18:53 +0300 Subject: [PATCH 192/284] style --- src/Storages/MergeTree/MergeTreeData.cpp | 1 - src/Storages/StorageS3.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 5ecce954dee..dba8e3ac1d9 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -46,7 +46,6 @@ #include #include #include -#include #include diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index efd04e53557..380c98264e0 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -17,7 +17,6 @@ #include #include -#include #include #include From ecc03394724bf4c2bf2cab8f34fb68cacf3d392d Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 28 Dec 2020 23:21:57 +0300 Subject: [PATCH 193/284] add test --- .../materialize_with_ddl.py | 29 +++++++++---------- tests/integration/test_storage_mysql/test.py | 7 +++++ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py index 47e325d0e96..c04194c8ebb 100644 --- a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py @@ -48,16 +48,15 @@ def dml_with_materialize_mysql_database(clickhouse_node, mysql_node, service_nam "/* Need ClickHouse support read mysql decimal unsigned_decimal DECIMAL(19, 10) UNSIGNED, _decimal DECIMAL(19, 10), */" "unsigned_float FLOAT UNSIGNED, _float FLOAT, " "unsigned_double DOUBLE UNSIGNED, _double DOUBLE, " - "_varchar VARCHAR(10), _char CHAR(10), " + "_varchar VARCHAR(10), _char CHAR(10), binary_col BINARY(8), " "/* Need ClickHouse support Enum('a', 'b', 'v') _enum ENUM('a', 'b', 'c'), */" "_date Date, _datetime DateTime, _timestamp TIMESTAMP, _bool BOOLEAN) ENGINE = InnoDB;") # it already has some data mysql_node.query(""" - INSERT INTO test_database.test_table_1 VALUES(1, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', + INSERT INTO test_database.test_table_1 VALUES(1, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', 'binary', '2020-01-01', '2020-01-01 00:00:00', '2020-01-01 00:00:00', true); """) - clickhouse_node.query( "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format( service_name)) @@ -65,51 +64,51 @@ def dml_with_materialize_mysql_database(clickhouse_node, mysql_node, service_nam assert "test_database" in clickhouse_node.query("SHOW DATABASES") check_query(clickhouse_node, "SELECT * FROM test_database.test_table_1 ORDER BY key FORMAT TSV", - "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t" + "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t" "2020-01-01 00:00:00\t2020-01-01 00:00:00\t1\n") mysql_node.query(""" - INSERT INTO test_database.test_table_1 VALUES(2, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', + INSERT INTO test_database.test_table_1 VALUES(2, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', 'binary', '2020-01-01', '2020-01-01 00:00:00', '2020-01-01 00:00:00', false); """) check_query(clickhouse_node, "SELECT * FROM test_database.test_table_1 ORDER BY key FORMAT TSV", - "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t" + "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t" "2020-01-01 00:00:00\t2020-01-01 00:00:00\t1\n2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t" - "varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t2020-01-01 00:00:00\t0\n") + "varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t2020-01-01 00:00:00\t0\n") mysql_node.query("UPDATE test_database.test_table_1 SET unsigned_tiny_int = 2 WHERE `key` = 1") check_query(clickhouse_node, """ SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int, small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, - unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, + unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, binary_col, _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ _bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV """, - "1\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t" + "1\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t" "2020-01-01 00:00:00\t1\n2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t" - "varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t0\n") + "varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t0\n") # update primary key mysql_node.query("UPDATE test_database.test_table_1 SET `key` = 3 WHERE `unsigned_tiny_int` = 2") check_query(clickhouse_node, "SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int," " small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, " - " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, " + " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, binary_col, " " _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ " " _bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV", "2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t" - "varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t0\n3\t2\t-1\t2\t-2\t3\t-3\t" - "4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t1\n") + "varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t0\n3\t2\t-1\t2\t-2\t3\t-3\t" + "4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t1\n") mysql_node.query('DELETE FROM test_database.test_table_1 WHERE `key` = 2') check_query(clickhouse_node, "SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int," " small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, " - " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, " + " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, binary_col, " " _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ " " _bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV", - "3\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t" + "3\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t" "2020-01-01 00:00:00\t1\n") mysql_node.query('DELETE FROM test_database.test_table_1 WHERE `unsigned_tiny_int` = 2') diff --git a/tests/integration/test_storage_mysql/test.py b/tests/integration/test_storage_mysql/test.py index 87033381e2c..7b23e20e200 100644 --- a/tests/integration/test_storage_mysql/test.py +++ b/tests/integration/test_storage_mysql/test.py @@ -148,6 +148,13 @@ def test_table_function(started_cluster): assert node1.query("SELECT sum(`money`) FROM {}".format(table_function)).rstrip() == '60000' conn.close() +def test_binary_type(started_cluster): + conn = get_mysql_conn() + with conn.cursor() as cursor: + cursor.execute("CREATE TABLE clickhouse.binary_type (id INT PRIMARY KEY, data BINARY(16) NOT NULL)") + table_function = "mysql('mysql1:3306', 'clickhouse', '{}', 'root', 'clickhouse')".format('binary_type') + node1.query("INSERT INTO {} VALUES (42, 'clickhouse')".format('TABLE FUNCTION ' + table_function)) + assert node1.query("SELECT * FROM {}".format(table_function)) == '42\tclickhouse\\0\\0\\0\\0\\0\\0\n' def test_enum_type(started_cluster): table_name = 'test_enum_type' From 91a9604ab00a3ae044c6aaf999bafde4eb32bb62 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Tue, 29 Dec 2020 02:43:47 +0300 Subject: [PATCH 194/284] Docs for the INSERT support --- .../en/sql-reference/table-functions/mysql.md | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/docs/en/sql-reference/table-functions/mysql.md b/docs/en/sql-reference/table-functions/mysql.md index 9220b31bbae..7be328a5475 100644 --- a/docs/en/sql-reference/table-functions/mysql.md +++ b/docs/en/sql-reference/table-functions/mysql.md @@ -5,10 +5,12 @@ toc_title: mysql # mysql {#mysql} -Allows `SELECT` queries to be performed on data that is stored on a remote MySQL server. +Allows `SELECT` and `INSERT` queries to be performed on data that is stored on a remote MySQL server. + +**Syntax** ``` sql -mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']); +mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']) ``` **Parameters** @@ -39,7 +41,7 @@ The rest of the conditions and the `LIMIT` sampling constraint are executed in C A table object with the same columns as the original MySQL table. -## Usage Example {#usage-example} +**Examples** Table in MySQL: @@ -50,18 +52,15 @@ mysql> CREATE TABLE `test`.`test` ( -> `float` FLOAT NOT NULL, -> `float_nullable` FLOAT NULL DEFAULT NULL, -> PRIMARY KEY (`int_id`)); -Query OK, 0 rows affected (0,09 sec) mysql> insert into test (`int_id`, `float`) VALUES (1,2); -Query OK, 1 row affected (0,00 sec) mysql> select * from test; -+------+----------+-----+----------+ ++--------+--------------+-------+----------------+ | int_id | int_nullable | float | float_nullable | -+------+----------+-----+----------+ ++--------+--------------+-------+----------------+ | 1 | NULL | 2 | NULL | -+------+----------+-----+----------+ -1 row in set (0,00 sec) ++--------+--------------+-------+----------------+ ``` Selecting data from ClickHouse: @@ -76,9 +75,24 @@ SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123') └────────┴──────────────┴───────┴────────────────┘ ``` -## See Also {#see-also} +Replacing and inserting: + +```sql +INSERT INTO mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 1) ('int_id', 'float') VALUES (1, 3); +INSERT INTO mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 0, 'UPDATE int_id = int_id + 1') ('int_id', 'float') VALUES (1, 4); +SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123'); +``` + +```text +┌─int_id─┬─int_nullable─┬─float─┬─float_nullable─┐ +│ 1 │ ᴺᵁᴸᴸ │ 3 │ ᴺᵁᴸᴸ │ +│ 2 │ ᴺᵁᴸᴸ │ 4 │ ᴺᵁᴸᴸ │ +└────────┴──────────────┴───────┴────────────────┘ +``` + +**See Also** - [The ‘MySQL’ table engine](../../engines/table-engines/integrations/mysql.md) - [Using MySQL as a source of external dictionary](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-mysql) -[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/mysql/) +[Original article](https://clickhouse.tech/docs/en/sql-reference/table_functions/mysql/) From c0eef84c5013ed22780ff9339cfe58bb6fe7c656 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Tue, 29 Dec 2020 04:15:26 +0300 Subject: [PATCH 195/284] better --- src/Common/ThreadPool.h | 8 ++++-- .../Impl/ParallelFormattingOutputFormat.h | 26 ++++++++++++------- src/Storages/MergeTree/MergeTreeData.cpp | 1 - 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h index 3fec6a29e10..da31921e542 100644 --- a/src/Common/ThreadPool.h +++ b/src/Common/ThreadPool.h @@ -194,14 +194,18 @@ public: ~ThreadFromGlobalPool() { - if (joinable()) + if (joinable()) { + std::cerr << StackTrace().toString() << std::endl; std::terminate(); + } } void join() { - if (!joinable()) + if (!joinable()) { + std::cerr << StackTrace().toString() << std::endl; std::terminate(); + } state->wait(); state.reset(); diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 6461caab17d..6f4b65a072a 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -36,6 +36,8 @@ public: const size_t max_threads_for_parallel_formatting; }; + ParallelFormattingOutputFormat() = delete; + explicit ParallelFormattingOutputFormat(Params params) : IOutputFormat(params.header, params.out) , internal_formatter_creator(params.internal_formatter_creator) @@ -51,9 +53,6 @@ public: ~ParallelFormattingOutputFormat() override { - need_flush = true; - if (!IOutputFormat::finalized) - finalize(); finishAndWait(); } @@ -66,7 +65,7 @@ public: void doWritePrefix() override { - addChunk(Chunk{}, ProcessingUnitType::START); + addChunk(Chunk{}, ProcessingUnitType::START, /*can_throw_exception*/ true); } void onCancel() override @@ -77,24 +76,31 @@ public: protected: void consume(Chunk chunk) override final { - addChunk(std::move(chunk), ProcessingUnitType::PLAIN); + addChunk(std::move(chunk), ProcessingUnitType::PLAIN, /*can_throw_exception*/ true); } void consumeTotals(Chunk totals) override { - addChunk(std::move(totals), ProcessingUnitType::TOTALS); + addChunk(std::move(totals), ProcessingUnitType::TOTALS, /*can_throw_exception*/ true); } void consumeExtremes(Chunk extremes) override { - addChunk(std::move(extremes), ProcessingUnitType::EXTREMES); + addChunk(std::move(extremes), ProcessingUnitType::EXTREMES, /*can_throw_exception*/ true); } void finalize() override { + need_flush = true; IOutputFormat::finalized = true; - addChunk(Chunk{}, ProcessingUnitType::FINALIZE); + /// Don't throw any background_exception here, because we want to finalize the execution. + /// Exception will be checked after main thread is finished. + addChunk(Chunk{}, ProcessingUnitType::FINALIZE, /*can_throw_exception*/ false); collector_finished.wait(); + + if (collector_thread.joinable()) + collector_thread.join(); + { std::unique_lock lock(mutex); if (background_exception) @@ -123,11 +129,11 @@ private: FINALIZE }; - void addChunk(Chunk chunk, ProcessingUnitType type) + void addChunk(Chunk chunk, ProcessingUnitType type, bool can_throw_exception) { { std::unique_lock lock(mutex); - if (background_exception) + if (background_exception && can_throw_exception) std::rethrow_exception(background_exception); } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index dba8e3ac1d9..4a40b9e4abd 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -60,7 +60,6 @@ #include #include #include -#include namespace ProfileEvents { From 07e9a25bd8b7736045f660bbc13532c72b22dcdd Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Tue, 29 Dec 2020 04:19:57 +0300 Subject: [PATCH 196/284] style --- src/Common/ThreadPool.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h index da31921e542..cceadc4ea2c 100644 --- a/src/Common/ThreadPool.h +++ b/src/Common/ThreadPool.h @@ -194,7 +194,8 @@ public: ~ThreadFromGlobalPool() { - if (joinable()) { + if (joinable()) + { std::cerr << StackTrace().toString() << std::endl; std::terminate(); } @@ -202,7 +203,8 @@ public: void join() { - if (!joinable()) { + if (!joinable()) + { std::cerr << StackTrace().toString() << std::endl; std::terminate(); } From be884a89f890ea41546a587d4e537392a70b9c31 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 29 Dec 2020 13:16:22 +0300 Subject: [PATCH 197/284] Minor fixes for min/sim hash --- src/Common/ThreadStatus.cpp | 2 +- src/Functions/ExtractString.h | 19 +---------- src/Functions/FunctionsStringHash.cpp | 33 +++---------------- src/Functions/FunctionsStringHash.h | 4 +-- src/Processors/LimitTransform.cpp | 2 +- src/Processors/Merges/Algorithms/MergedData.h | 2 +- src/Processors/Pipe.h | 4 +-- 7 files changed, 13 insertions(+), 53 deletions(-) diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index 4bba3d8f4eb..3809f84711e 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -96,7 +96,7 @@ ThreadStatus::~ThreadStatus() catch (const DB::Exception &) { /// It's a minor tracked memory leak here (not the memory itself but it's counter). - /// We've already allocated a little bit more then the limit and cannot track it in the thread memory tracker or its parent. + /// We've already allocated a little bit more than the limit and cannot track it in the thread memory tracker or its parent. } if (deleter) diff --git a/src/Functions/ExtractString.h b/src/Functions/ExtractString.h index 8b88a5a7c37..aa0e1b04835 100644 --- a/src/Functions/ExtractString.h +++ b/src/Functions/ExtractString.h @@ -20,8 +20,7 @@ namespace DB // includes extracting ASCII ngram, UTF8 ngram, ASCII word and UTF8 word struct ExtractStringImpl { - // read a ASCII word - static ALWAYS_INLINE inline const UInt8 * readOneASCIIWord(const UInt8 *& pos, const UInt8 * end) + static ALWAYS_INLINE inline const UInt8 * readOneWord(const UInt8 *& pos, const UInt8 * end) { // jump separators while (pos < end && isUTF8Sep(*pos)) @@ -35,22 +34,6 @@ struct ExtractStringImpl return word_start; } - // read one UTF8 word from pos to word - static ALWAYS_INLINE inline const UInt8 * readOneUTF8Word(const UInt8 *& pos, const UInt8 * end) - { - // jump UTF8 separator - while (pos < end && isUTF8Sep(*pos)) - ++pos; - - // UTF8 word's character number - const UInt8 * word_start = pos; - - while (pos < end && !isUTF8Sep(*pos)) - readOneUTF8Code(pos, end); - - return word_start; - } - // we use ASCII non-alphanum character as UTF8 separator static ALWAYS_INLINE inline bool isUTF8Sep(const UInt8 c) { return c < 128 && !isAlphaNumericASCII(c); } diff --git a/src/Functions/FunctionsStringHash.cpp b/src/Functions/FunctionsStringHash.cpp index e389e2f7f98..e5ed5dc77c0 100644 --- a/src/Functions/FunctionsStringHash.cpp +++ b/src/Functions/FunctionsStringHash.cpp @@ -249,13 +249,7 @@ struct SimHashImpl // get first word shingle while (start < end && words.size() < shingle_size) { - const UInt8 * word_start; - - if constexpr (UTF8) - word_start = ExtractStringImpl::readOneUTF8Word(start, end); - else - word_start = ExtractStringImpl::readOneASCIIWord(start, end); - + const UInt8 * word_start = ExtractStringImpl::readOneWord(start, end); size_t length = start - word_start; if (length >= min_word_size) @@ -271,13 +265,7 @@ struct SimHashImpl size_t offset = 0; while (start < end) { - const UInt8 * word_start; - - if constexpr (UTF8) - word_start = ExtractStringImpl::readOneUTF8Word(start, end); - else - word_start = ExtractStringImpl::readOneASCIIWord(start, end); - + const UInt8 * word_start = ExtractStringImpl::readOneWord(start, end); size_t length = start - word_start; if (length < min_word_size) @@ -340,7 +328,7 @@ struct MinHashImpl { static constexpr size_t min_word_size = 4; - template + template struct Heap { void update(UInt64 hash, BytesRef ref, size_t limit) @@ -478,13 +466,7 @@ struct MinHashImpl // get first word shingle while (start < end && words.size() < shingle_size) { - const UInt8 * word_start; - - if constexpr (UTF8) - word_start = ExtractStringImpl::readOneUTF8Word(start, end); - else - word_start = ExtractStringImpl::readOneASCIIWord(start, end); - + const UInt8 * word_start = ExtractStringImpl::readOneWord(start, end); size_t length = start - word_start; if (length >= min_word_size) @@ -506,12 +488,7 @@ struct MinHashImpl size_t offset = 0; while (start < end) { - const UInt8 * word_start; - - if constexpr (UTF8) - word_start = ExtractStringImpl::readOneUTF8Word(start, end); - else - word_start = ExtractStringImpl::readOneASCIIWord(start, end); + const UInt8 * word_start = ExtractStringImpl::readOneWord(start, end); size_t length = start - word_start; diff --git a/src/Functions/FunctionsStringHash.h b/src/Functions/FunctionsStringHash.h index 2bcb7fa1013..c09abc33319 100644 --- a/src/Functions/FunctionsStringHash.h +++ b/src/Functions/FunctionsStringHash.h @@ -77,7 +77,7 @@ public: { if constexpr (is_simhash) throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION, - "Function {} expect no more then two arguments (text, shingle size), got {}", + "Function {} expect no more than two arguments (text, shingle size), got {}", getName(), arguments.size()); if (!isUnsignedInteger(arguments[2].type)) @@ -95,7 +95,7 @@ public: if (arguments.size() > 3) { throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION, - "Function {} expect no more then three arguments (text, shingle size, num hashes), got {}", + "Function {} expect no more than three arguments (text, shingle size, num hashes), got {}", getName(), arguments.size()); } diff --git a/src/Processors/LimitTransform.cpp b/src/Processors/LimitTransform.cpp index f7043cbfec5..36c58e1454e 100644 --- a/src/Processors/LimitTransform.cpp +++ b/src/Processors/LimitTransform.cpp @@ -237,7 +237,7 @@ LimitTransform::Status LimitTransform::preparePair(PortsData & data) previous_row_chunk = makeChunkWithPreviousRow(data.current_chunk, data.current_chunk.getNumRows() - 1); } else - /// This function may be heavy to execute in prepare. But it happens no more then twice, and make code simpler. + /// This function may be heavy to execute in prepare. But it happens no more than twice, and make code simpler. splitChunk(data); bool may_need_more_data_for_ties = previous_row_chunk || rows_read - rows <= offset + limit; diff --git a/src/Processors/Merges/Algorithms/MergedData.h b/src/Processors/Merges/Algorithms/MergedData.h index 5075174db62..9bf33d72f31 100644 --- a/src/Processors/Merges/Algorithms/MergedData.h +++ b/src/Processors/Merges/Algorithms/MergedData.h @@ -82,7 +82,7 @@ public: if (need_flush) return true; - /// Never return more then max_block_size. + /// Never return more than max_block_size. if (merged_rows >= max_block_size) return true; diff --git a/src/Processors/Pipe.h b/src/Processors/Pipe.h index 065a89d7c0f..2d64de3e664 100644 --- a/src/Processors/Pipe.h +++ b/src/Processors/Pipe.h @@ -71,8 +71,8 @@ public: enum class StreamType { Main = 0, /// Stream for query data. There may be several streams of this type. - Totals, /// Stream for totals. No more then one. - Extremes, /// Stream for extremes. No more then one. + Totals, /// Stream for totals. No more than one. + Extremes, /// Stream for extremes. No more than one. }; using ProcessorGetter = std::function; From 58b1e47e0d57ad4782856b50676c6803c272f7fa Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 29 Dec 2020 16:27:47 +0300 Subject: [PATCH 198/284] Update string-functions.md --- docs/en/sql-reference/functions/string-functions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index ddb27756df4..83f2705693a 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -560,9 +560,9 @@ Result: ## encodeXMLComponent {#encode-xml-component} -Escape characters to place string into XML text node or attribute. +Escapes characters to place string into XML text node or attribute. -The following five XML predefined entities will be replaced: `<`, `&`, `>`, `"`, `\'`. +The following five XML predefined entities will be replaced: `<`, `&`, `>`, `"`, `'`. **Syntax** From 206660631cc757eae77ca4b96736681fd6854128 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 29 Dec 2020 17:50:25 +0300 Subject: [PATCH 199/284] Update datetime64.md --- docs/ru/sql-reference/data-types/datetime64.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/data-types/datetime64.md b/docs/ru/sql-reference/data-types/datetime64.md index b5e185c9132..6576bf9dc0d 100644 --- a/docs/ru/sql-reference/data-types/datetime64.md +++ b/docs/ru/sql-reference/data-types/datetime64.md @@ -96,7 +96,7 @@ FROM dt - [Функции для работы с датой и временем](../../sql-reference/functions/date-time-functions.md) - [Функции для работы с массивами](../../sql-reference/functions/array-functions.md) - [Настройка `date_time_input_format`](../../operations/settings/settings.md#settings-date_time_input_format) -- [Настройка `date_time_output_format`](../../operations/settings/settings.md#settings-date_time_output_format) +- [Настройка `date_time_output_format`](../../operations/settings/settings.md) - [Конфигурационный параметр сервера `timezone`](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) - [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime) - [Тип данных `Date`](date.md) From def9c817797343bba62708c22fa1967ed92ba7d2 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Tue, 29 Dec 2020 17:51:42 +0300 Subject: [PATCH 200/284] disable values for parallel_formatting --- src/Processors/Formats/Impl/ValuesRowOutputFormat.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Processors/Formats/Impl/ValuesRowOutputFormat.cpp b/src/Processors/Formats/Impl/ValuesRowOutputFormat.cpp index 7f249200404..7791e1296e0 100644 --- a/src/Processors/Formats/Impl/ValuesRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ValuesRowOutputFormat.cpp @@ -51,7 +51,6 @@ void registerOutputFormatProcessorValues(FormatFactory & factory) { return std::make_shared(buf, sample, params, settings); }); - factory.markOutputFormatSupportsParallelFormatting("Values"); } } From 310c8138a69109f3e46e608df536081c58bace43 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 29 Dec 2020 17:56:55 +0300 Subject: [PATCH 201/284] Update formats.md --- docs/zh/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/interfaces/formats.md b/docs/zh/interfaces/formats.md index 808bff42335..b37ef559aa7 100644 --- a/docs/zh/interfaces/formats.md +++ b/docs/zh/interfaces/formats.md @@ -511,7 +511,7 @@ ClickHouse支持[NULL](../sql-reference/syntax.md), 在JSON输出中显示为`nu 在这种格式中，一个JSON对象被解释为一个值。如果输入有几个JSON对象(逗号分隔)，它们将被解释为独立的行。 -这种格式只能对具有单个字段类型的表进行解析[String](../sql-reference/data-types/string.md)。其余的列必须设置为[DEFAULT](../sql-reference/statements/create/table.md#default)或[MATERIALIZED](../sql-reference/statements/create/table.md#materialized)，或者忽略。一旦将整个JSON对象收集为字符串，就可以使用[JSON函数](../sql-reference/functions/json-functions.md)运行它。 +这种格式只能对具有单个字段类型的表进行解析[String](../sql-reference/data-types/string.md)。其余的列必须设置为[DEFAULT](../sql-reference/statements/create.md)或[MATERIALIZED](../sql-reference/statements/create.md)，或者忽略。一旦将整个JSON对象收集为字符串，就可以使用[JSON函数](../sql-reference/functions/json-functions.md)运行它。 **示例** From f07df76c59ef72e8380d8b091eea31f9ea4a30c2 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 24 Dec 2020 18:41:37 +0300 Subject: [PATCH 202/284] fix ans --- utils/db-generator/query_db_generator.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/db-generator/query_db_generator.cpp b/utils/db-generator/query_db_generator.cpp index b6e287278ff..33b8e6ce8af 100644 --- a/utils/db-generator/query_db_generator.cpp +++ b/utils/db-generator/query_db_generator.cpp @@ -56,9 +56,9 @@ std::string randomDate() int32_t year = rng() % 136 + 1970; int32_t month = rng() % 12 + 1; int32_t day = rng() % 12 + 1; - char ans[13]; - sprintf(ans, "'%04u-%02u-%02u'", year, month, day); - return std::string(ans); + char answer[13]; + sprintf(answer, "'%04u-%02u-%02u'", year, month, day); + return std::string(answer); } std::string randomDatetime() @@ -69,9 +69,9 @@ std::string randomDatetime() int32_t hours = rng() % 24; int32_t minutes = rng() % 60; int32_t seconds = rng() % 60; - char ans[22]; + char answer[22]; sprintf( - ans, + answer, "'%04u-%02u-%02u %02u:%02u:%02u'", year, month, @@ -79,7 +79,7 @@ std::string randomDatetime() hours, minutes, seconds); - return std::string(ans); + return std::string(answer); } TableAndColumn get_table_a_column(const std::string & c) { From 7e37743c37313b0c89a629335d8fd6aea1c057c3 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 24 Dec 2020 18:43:53 +0300 Subject: [PATCH 203/284] run check-typos separately for now --- utils/check-style/check-style | 4 ---- 1 file changed, 4 deletions(-) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index d3653deb980..1b1678dee05 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -85,10 +85,6 @@ find $ROOT_PATH -name '.gitmodules' | while read i; do grep -F 'url = ' $i | gre # There shouldn't be any code snippets under GPL or LGPL find $ROOT_PATH/{src,base,programs} -name '*.h' -or -name '*.cpp' 2>/dev/null | xargs grep -i -F 'General Public License' && echo "There shouldn't be any code snippets under GPL or LGPL" -# Check for typos in code -CURDIR=$(dirname "${BASH_SOURCE[0]}") -"${CURDIR}"/check-typos - # Check sh tests with Shellcheck (cd $ROOT_PATH/tests/queries/0_stateless/ && shellcheck --check-sourced --external-sources --severity info --exclude SC1071,SC2086 *.sh ../1_stateful/*.sh) From a25fa1996b4e7d7db4c469b6fa484774202b71f8 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 28 Dec 2020 14:46:53 +0300 Subject: [PATCH 204/284] fix style checks --- docker/test/coverage/run.sh | 14 +++++++------- docker/test/fasttest/run.sh | 2 +- docker/test/performance-comparison/compare.sh | 16 ++++++++-------- docker/test/stateless/run.sh | 6 +++--- docker/test/style/Dockerfile | 3 ++- src/Storages/MergeTree/MergeTreeIOSettings.h | 2 +- .../00029_test_zookeeper_optimize_exception.sh | 1 + .../0_stateless/00039_inserts_through_http.sh | 1 + .../00070_insert_fewer_columns_http.sh | 1 + .../0_stateless/00090_union_race_conditions_1.sh | 1 + .../0_stateless/00091_union_race_conditions_2.sh | 1 + .../0_stateless/00092_union_race_conditions_3.sh | 1 + .../0_stateless/00093_union_race_conditions_4.sh | 1 + .../0_stateless/00094_union_race_conditions_5.sh | 1 + .../00097_long_storage_buffer_race_condition.sh | 1 + ...0097_long_storage_buffer_race_condition_mt.sh | 1 + .../00100_subquery_table_identifier.sh | 1 + .../00115_shard_in_incomplete_result.sh | 1 + ..._shard_memory_tracker_and_exception_safety.sh | 1 + tests/queries/0_stateless/00155_long_merges.sh | 1 + .../00177_inserts_through_http_parts.sh | 1 + .../0_stateless/00186_very_long_arrays.sh | 1 + .../00210_insert_select_extremes_http.sh | 1 + .../00265_http_content_type_format_timezone.sh | 1 + tests/queries/0_stateless/00301_csv.sh | 1 + .../0_stateless/00302_http_compression.sh | 1 + .../0_stateless/00304_http_external_data.sh | 1 + .../0_stateless/00305_http_and_readonly.sh | 1 + tests/queries/0_stateless/00310_tskv.sh | 1 + .../0_stateless/00313_const_totals_extremes.sh | 1 + .../0_stateless/00322_disable_checksumming.sh | 1 + tests/queries/0_stateless/00335_bom.sh | 1 + .../0_stateless/00336_shard_stack_trace.sh | 1 + .../0_stateless/00339_parsing_bad_arrays.sh | 1 + .../00354_host_command_line_option.sh | 1 + .../0_stateless/00365_statistics_in_formats.sh | 1 + .../0_stateless/00366_multi_statements.sh | 1 + .../0_stateless/00368_format_option_collision.sh | 1 + tests/queries/0_stateless/00372_cors_header.sh | 1 + ...0374_json_each_row_input_with_noisy_fields.sh | 1 + .../0_stateless/00379_system_processes_port.sh | 1 + ...80_client_break_at_exception_in_batch_mode.sh | 1 + ...0385_storage_file_and_clickhouse-local_app.sh | 1 + tests/queries/0_stateless/00386_long_in_pk.sh | 1 + .../0_stateless/00387_use_client_time_zone.sh | 1 + .../0_stateless/00400_client_external_options.sh | 1 + .../0_stateless/00405_PrettyCompactMonoBlock.sh | 1 + tests/queries/0_stateless/00407_parsing_nulls.sh | 1 + .../queries/0_stateless/00408_http_keep_alive.sh | 1 + ...0411_long_accurate_number_comparison_float.sh | 1 + ...00411_long_accurate_number_comparison_int1.sh | 1 + ...00411_long_accurate_number_comparison_int2.sh | 1 + ...00411_long_accurate_number_comparison_int3.sh | 1 + ...00411_long_accurate_number_comparison_int4.sh | 1 + tests/queries/0_stateless/00415_into_outfile.sh | 1 + .../00416_pocopatch_progress_in_http_headers.sh | 1 + tests/queries/0_stateless/00417_kill_query.sh | 1 + .../0_stateless/00417_system_build_options.sh | 1 + .../00418_input_format_allow_errors.sh | 1 + .../0_stateless/00419_show_sql_queries.sh | 1 + .../00421_storage_merge__table_index.sh | 1 + .../0_stateless/00427_alter_primary_key.sh | 1 + .../0_stateless/00429_long_http_bufferization.sh | 1 + tests/queries/0_stateless/00430_https_server.sh | 1 + .../00443_optimize_final_vertical_merge.sh | 1 + .../00443_preferred_block_size_bytes.sh | 1 + ...r_column_in_partition_concurrent_zookeeper.sh | 1 + .../00463_long_sessions_in_http_interface.sh | 1 + .../00473_output_format_json_quote_denormals.sh | 1 + .../0_stateless/00474_readonly_settings.sh | 1 + .../0_stateless/00485_http_insert_format.sh | 1 + .../0_stateless/00497_whitespaces_in_insert.sh | 1 + tests/queries/0_stateless/00501_http_head.sh | 1 + .../0_stateless/00504_insert_miss_columns.sh | 1 + tests/queries/0_stateless/00505_secure.sh | 1 + tests/queries/0_stateless/00505_shard_secure.sh | 1 + .../queries/0_stateless/00507_array_no_params.sh | 1 + .../0_stateless/00512_fractional_time_zones.sh | 1 + tests/queries/0_stateless/00520_http_nullable.sh | 1 + .../0_stateless/00531_client_ignore_error.sh | 1 + .../0_stateless/00534_client_ignore_error.sh | 1 + tests/queries/0_stateless/00534_filimonov.sh | 1 + .../00534_functions_bad_arguments1.sh | 2 ++ .../00534_functions_bad_arguments10.sh | 2 ++ .../00534_functions_bad_arguments11.sh | 2 ++ .../00534_functions_bad_arguments12.sh | 2 ++ .../00534_functions_bad_arguments13.sh | 2 ++ .../00534_functions_bad_arguments2.sh | 2 ++ .../00534_functions_bad_arguments3.sh | 2 ++ .../00534_functions_bad_arguments4.sh | 2 ++ .../00534_functions_bad_arguments5.sh | 2 ++ .../00534_functions_bad_arguments6.sh | 2 ++ .../00534_functions_bad_arguments7.sh | 2 ++ .../00534_functions_bad_arguments8.sh | 2 ++ .../00534_functions_bad_arguments9.sh | 2 ++ .../queries/0_stateless/00540_bad_data_types.sh | 1 + ...access_to_temporary_table_in_readonly_mode.sh | 1 + .../0_stateless/00550_join_insert_select.sh | 1 + tests/queries/0_stateless/00557_remote_port.sh | 1 + tests/queries/0_stateless/00564_enum_order.sh | 1 + tests/queries/0_stateless/00565_enum_order.sh | 1 + .../00574_empty_strings_deserialization.sh | 1 + ...al_column_exception_when_drop_depen_column.sh | 1 + .../0_stateless/00595_insert_into_view.sh | 1 + .../0_stateless/00596_limit_on_expanded_ast.sh | 1 + .../0_stateless/00598_create_as_select_http.sh | 1 + .../0_stateless/00600_replace_running_query.sh | 1 + .../0_stateless/00601_kill_running_query.sh | 1 + tests/queries/0_stateless/00602_throw_if.sh | 1 + .../0_stateless/00612_http_max_query_size.sh | 1 + .../0_stateless/00612_pk_in_tuple_perf.sh | 1 + .../00623_truncate_table_throw_exception.sh | 1 + .../0_stateless/00625_query_in_form_data.sh | 1 + ...626_replace_partition_from_table_zookeeper.sh | 1 + .../0_stateless/00630_arbitrary_csv_delimiter.sh | 1 + ...rialized_view_and_too_many_parts_zookeeper.sh | 1 + tests/queries/0_stateless/00634_logging_shard.sh | 1 + ...0634_performance_introspection_and_logging.sh | 1 + .../00636_partition_key_parts_pruning.sh | 1 + ...37_sessions_in_http_interface_and_settings.sh | 1 + tests/queries/0_stateless/00646_url_engine.sh | 1 + .../00650_csv_with_specified_quote_rule.sh | 1 + ...00651_default_database_on_client_reconnect.sh | 1 + .../0_stateless/00652_mergetree_mutations.sh | 2 ++ .../0_stateless/00652_mutations_alter_update.sh | 1 + .../00652_mutations_default_database.sh | 1 + ...cated_mutations_default_database_zookeeper.sh | 2 ++ .../00652_replicated_mutations_zookeeper.sh | 2 ++ .../00653_verification_monotonic_data_load.sh | 1 + .../0_stateless/00682_empty_parts_merge.sh | 1 + .../0_stateless/00686_client_exit_code.sh | 2 ++ .../queries/0_stateless/00687_top_and_offset.sh | 1 + ...insert_select_converting_exception_message.sh | 1 + .../00699_materialized_view_mutations.sh | 1 + .../00704_drop_truncate_memory_table.sh | 1 + .../0_stateless/00705_drop_create_merge_tree.sh | 1 + ...715_fetch_merged_or_mutated_part_zookeeper.sh | 2 ++ .../00715_json_each_row_input_nested.sh | 1 + .../00719_insert_block_without_column.sh | 1 + .../queries/0_stateless/00719_parallel_ddl_db.sh | 1 + .../0_stateless/00719_parallel_ddl_table.sh | 1 + .../0_stateless/00728_json_each_row_parsing.sh | 1 + .../00731_long_merge_tree_select_opened_files.sh | 1 + .../0_stateless/00738_lock_for_inner_table.sh | 1 + tests/queries/0_stateless/00746_sql_fuzzy.sh | 1 + ...uted_optimize_skip_select_on_unused_shards.sh | 1 + ...skip_select_on_unused_shards_with_prewhere.sh | 1 + tests/queries/0_stateless/00763_lock_buffer.sh | 1 + ...3_long_lock_buffer_alter_destination_table.sh | 1 + .../00764_max_query_size_allocation.sh | 1 + .../00816_long_concurrent_alter_column.sh | 1 + .../queries/0_stateless/00823_capnproto_input.sh | 1 + .../0_stateless/00825_http_header_query_id.sh | 1 + .../0_stateless/00825_protobuf_format_input.sh | 3 +++ .../0_stateless/00825_protobuf_format_output.sh | 1 + ...ncel_http_readonly_queries_on_client_close.sh | 1 + ...w_to_set_two_configuration_files_in_client.sh | 1 + .../00834_hints_for_type_function_typos.sh | 1 + tests/queries/0_stateless/00834_kill_mutation.sh | 2 ++ .../00834_kill_mutation_replicated_zookeeper.sh | 2 ++ tests/queries/0_stateless/00837_minmax_index.sh | 1 + .../00838_system_tables_drop_table_race.sh | 1 + tests/queries/0_stateless/00838_unique_index.sh | 1 + ...0_long_concurrent_select_and_drop_deadlock.sh | 1 + .../00851_http_insert_json_defaults.sh | 1 + .../00898_parsing_bad_diagnostic_message.sh | 1 + tests/queries/0_stateless/00900_orc_load.sh | 1 + tests/queries/0_stateless/00900_parquet.sh | 1 + .../queries/0_stateless/00900_parquet_decimal.sh | 1 + tests/queries/0_stateless/00900_parquet_load.sh | 1 + .../0_stateless/00907_set_index_max_rows.sh | 1 + .../0_stateless/00908_bloom_filter_index.sh | 1 + .../0_stateless/00908_long_http_insert.sh | 1 + .../00909_kill_not_initialized_query.sh | 1 + .../00910_client_window_size_detection.sh | 1 + .../00921_datetime64_compatibility.sh | 1 + .../0_stateless/00927_asof_join_other_types.sh | 1 + ...33_test_fix_extra_seek_on_compressed_cache.sh | 1 + .../0_stateless/00937_template_output_format.sh | 1 + .../0_stateless/00937_test_use_header_csv.sh | 1 + .../0_stateless/00937_test_use_header_tsv.sh | 1 + .../0_stateless/00938_fix_rwlock_segfault.sh | 1 + .../0_stateless/00938_template_input_format.sh | 1 + .../00941_system_columns_race_condition.sh | 1 + tests/queries/0_stateless/00942_dataparts_500.sh | 1 + tests/queries/0_stateless/00942_mutate_index.sh | 1 + .../0_stateless/00943_materialize_index.sh | 1 + .../00944_clear_index_in_partition.sh | 1 + ..._create_bloom_filter_index_with_merge_tree.sh | 1 + .../00948_format_in_with_single_element.sh | 1 + .../0_stateless/00952_basic_constraints.sh | 1 + .../queries/0_stateless/00952_input_function.sh | 1 + .../0_stateless/00953_constraints_operations.sh | 1 + .../00953_indices_alter_exceptions.sh | 1 + .../00953_zookeeper_suetin_deduplication_bug.sh | 1 + .../00954_client_prepared_statements.sh | 1 + .../00955_complex_prepared_statements.sh | 1 + .../0_stateless/00955_test_final_mark_use.sh | 1 + .../00956_http_prepared_statements.sh | 1 + .../0_stateless/00956_sensitive_data_masking.sh | 1 + .../00957_format_with_clashed_aliases.sh | 1 + .../00958_format_of_tuple_array_element.sh | 1 + .../00959_format_with_different_aliases.sh | 1 + .../00964_bloom_index_string_functions.sh | 1 + .../0_stateless/00965_logs_level_bugfix.sh | 1 + .../00965_send_logs_level_concurrent_queries.sh | 1 + .../00965_set_index_string_functions.sh | 1 + .../0_stateless/00971_query_id_in_logs.sh | 1 + .../00974_primary_key_for_lowCardinality.sh | 1 + .../00974_text_log_table_not_empty.sh | 1 + ...0975_indices_mutation_replicated_zookeeper.sh | 2 ++ .../0_stateless/00980_alter_settings_race.sh | 1 + .../0_stateless/00981_in_subquery_with_tuple.sh | 1 + .../0_stateless/00984_parser_stack_overflow.sh | 1 + tests/queries/0_stateless/00990_hasToken.sh | 1 + .../00991_system_parts_race_condition.sh | 1 + ...0992_system_parts_race_condition_zookeeper.sh | 1 + ...system_parts_race_condition_drop_zookeeper.sh | 1 + .../0_stateless/00995_exception_while_insert.sh | 1 + .../01000_unneeded_substitutions_client.sh | 1 + .../01001_rename_merge_race_condition.sh | 1 + ...2_alter_nullable_adaptive_granularity_long.sh | 1 + .../01003_kill_query_race_condition.sh | 1 + .../queries/0_stateless/01004_rename_deadlock.sh | 1 + .../0_stateless/01005_rwr_shard_deadlock.sh | 1 + ...1006_simpod_empty_part_single_column_write.sh | 2 ++ .../0_stateless/01007_r1r2_w_r2r1_deadlock.sh | 1 + .../01010_low_cardinality_and_native_http.sh | 1 + .../01013_sync_replica_timeout_zookeeper.sh | 1 + .../0_stateless/01014_format_custom_separated.sh | 1 + .../0_stateless/01014_lazy_database_basic.sh | 1 + ...ncurrent_recreate_reattach_and_show_tables.sh | 1 + .../01015_insert_values_parametrized.sh | 1 + .../0_stateless/01016_input_null_as_default.sh | 1 + ..._with_nondeterministic_functions_zookeeper.sh | 1 + .../0_stateless/01017_tsv_empty_as_default.sh | 1 + .../01018_ddl_dictionaries_bad_queries.sh | 1 + ...1018_ddl_dictionaries_concurrent_requrests.sh | 1 + ...01018_insert_multiple_blocks_with_defaults.sh | 1 + .../01019_alter_materialized_view_atomic.sh | 1 + .../01019_alter_materialized_view_consistent.sh | 1 + .../0_stateless/01019_parallel_parsing_cancel.sh | 1 + .../01030_limit_by_with_ties_error.sh | 1 + .../01031_mutations_interpreter_and_context.sh | 1 + .../01034_move_partition_from_table_zookeeper.sh | 1 + .../0_stateless/01034_values_parse_float_bug.sh | 1 + .../0_stateless/01035_avg_weighted_long.sh | 1 + ...urrent_move_partition_from_table_zookeeper.sh | 1 + .../01035_enum_conversion_native_format.sh | 1 + .../0_stateless/01035_lc_empty_part_bug.sh | 1 + .../01037_polygon_dicts_correctness_all.sh | 1 + .../01037_polygon_dicts_correctness_fast.sh | 1 + .../01037_polygon_dicts_simple_functions.sh | 1 + .../01038_dictionary_lifetime_min_zero_sec.sh | 1 + ...ictionary_invalidate_query_switchover_long.sh | 1 + ...ystem_reload_dictionary_reloads_completely.sh | 1 + .../01045_order_by_pk_special_storages.sh | 1 + ...ookeeper_system_mutations_with_parts_names.sh | 2 ++ .../01052_compression_buffer_overrun.sh | 1 + .../01054_cache_dictionary_bunch_update.sh | 1 + .../01054_random_printable_ascii_ubsan.sh | 1 + .../01055_compact_parts_granularity.sh | 1 + .../01055_minmax_index_compact_parts.sh | 1 + ...1056_prepared_statements_null_and_escaping.sh | 1 + .../01057_http_compression_prefer_brotli.sh | 1 + .../0_stateless/01058_zlib_ng_level1_bug.sh | 1 + tests/queries/0_stateless/01060_avro.sh | 1 + .../0_stateless/01062_max_parser_depth.sh | 1 + .../01071_http_header_exception_code.sh | 1 + ...76_cache_dictionary_datarace_exception_ptr.sh | 1 + .../0_stateless/01076_json_each_row_array.sh | 1 + .../01076_parallel_alter_replicated_zookeeper.sh | 1 + .../01077_mutations_index_consistency.sh | 1 + .../0_stateless/01079_bad_alters_zookeeper.sh | 1 + ...9_parallel_alter_add_drop_column_zookeeper.sh | 1 + ...1079_parallel_alter_detach_table_zookeeper.sh | 1 + .../01079_parallel_alter_modify_zookeeper.sh | 1 + .../01085_max_distributed_connections.sh | 1 + .../01085_max_distributed_connections_http.sh | 1 + .../0_stateless/01085_regexp_input_format.sh | 1 + .../queries/0_stateless/01086_odbc_roundtrip.sh | 1 + .../01086_regexp_input_format_skip_unmatched.sh | 1 + .../0_stateless/01088_benchmark_query_id.sh | 1 + .../queries/0_stateless/01098_msgpack_format.sh | 1 + .../01098_temporary_and_external_tables.sh | 1 + .../01103_check_cpu_instructions_at_startup.sh | 1 + .../01103_optimize_drop_race_zookeeper.sh | 1 + .../0_stateless/01107_atomic_db_detach_attach.sh | 1 + .../01107_tuples_arrays_parsing_exceptions.sh | 1 + ...restart_replicas_rename_deadlock_zookeeper.sh | 1 + .../queries/0_stateless/01114_database_atomic.sh | 1 + .../0_stateless/01146_clickhouse_local_data.sh | 1 + tests/queries/0_stateless/01150_ddl_guard_rwr.sh | 1 + .../0_stateless/01187_set_profile_as_setting.sh | 1 + .../01192_rename_database_zookeeper.sh | 3 ++- .../0_stateless/01193_metadata_loading.sh | 1 + tests/queries/0_stateless/01194_http_query_id.sh | 1 + .../0_stateless/01195_formats_diagnostic_info.sh | 3 ++- .../0_stateless/01196_max_parser_depth.sh | 1 + .../0_stateless/01198_client_quota_key.sh | 1 + .../01213_alter_rename_column_zookeeper.sh | 1 + .../0_stateless/01232_json_as_string_format.sh | 1 + .../01232_preparing_sets_race_condition.sh | 1 + .../0_stateless/01238_http_memory_tracking.sh | 1 + .../0_stateless/01249_flush_interactive.sh | 1 + tests/queries/0_stateless/01258_bom_tsv.sh | 1 + .../queries/0_stateless/01268_procfs_metrics.sh | 1 + .../0_stateless/01271_http_code_parse_error.sh | 1 + tests/queries/0_stateless/01273_arrow.sh | 1 + tests/queries/0_stateless/01273_arrow_load.sh | 1 + tests/queries/0_stateless/01273_arrow_stream.sh | 2 ++ .../0_stateless/01274_generate_random_nested.sh | 1 + .../0_stateless/01278_format_multiple_queries.sh | 1 + ...ert_block_size_rows_for_materialized_views.sh | 1 + .../0_stateless/01279_empty_external_table.sh | 1 + .../0_stateless/01280_ttl_where_group_by.sh | 1 + .../01281_group_by_limit_memory_tracking.sh | 5 +++-- .../0_stateless/01285_engine_join_donmikel.sh | 1 + ...1293_client_interactive_vertical_multiline.sh | 1 + ...293_client_interactive_vertical_singleline.sh | 1 + .../0_stateless/01293_optimize_final_force.sh | 1 + ...ncurrent_recreate_reattach_and_show_tables.sh | 1 + .../01300_client_save_history_when_terminated.sh | 1 + ...1301_aggregate_state_exception_memory_leak.sh | 1 + ...1302_aggregate_state_exception_memory_leak.sh | 1 + tests/queries/0_stateless/01304_direct_io.sh | 1 + .../01305_replica_create_drop_zookeeper.sh | 1 + .../queries/0_stateless/01306_benchmark_json.sh | 1 + .../01307_multiple_leaders_zookeeper.sh | 1 + .../0_stateless/01307_orc_output_format.sh | 1 + .../01308_orc_output_format_arrays.sh | 1 + .../0_stateless/01312_skip_empty_params.sh | 1 + .../01316_create_user_syntax_hilite.sh | 1 + .../01317_no_password_in_command_line.sh | 1 + .../01318_alter_add_constraint_format.sh | 1 + ...01318_long_unsuccessful_mutation_zookeeper.sh | 1 + ...01320_create_sync_race_condition_zookeeper.sh | 1 + .../0_stateless/01338_long_select_and_alter.sh | 1 + .../01338_long_select_and_alter_zookeeper.sh | 1 + .../01339_client_unrecognized_option.sh | 1 + .../0_stateless/01342_query_parameters_alias.sh | 1 + .../01355_CSV_input_format_allow_errors.sh | 1 + tests/queries/0_stateless/01358_lc_parquet.sh | 1 + .../0_stateless/01361_fover_remote_num_tries.sh | 1 + ..._client_autocomplete_word_break_characters.sh | 1 + .../01375_output_format_tsv_csv_with_names.sh | 1 + ...orage_file_tsv_csv_with_names_write_prefix.sh | 1 + .../0_stateless/01383_log_broken_table.sh | 1 + .../0_stateless/01393_benchmark_secure_port.sh | 1 + .../0_stateless/01395_limit_more_cases.sh | 1 + ...6_inactive_replica_cleanup_nodes_zookeeper.sh | 3 ++- .../0_stateless/01399_http_request_headers.sh | 1 + .../queries/0_stateless/01401_FORMAT_SETTINGS.sh | 1 + .../01406_carriage_return_in_tsv_csv.sh | 1 + .../0_stateless/01412_cache_dictionary_race.sh | 1 + .../01414_mutations_and_errors_zookeeper.sh | 1 + .../0_stateless/01415_sticking_mutations.sh | 1 + .../01417_freeze_partition_verbose.sh | 3 ++- .../01417_freeze_partition_verbose_zookeeper.sh | 3 ++- .../01417_query_time_in_system_events.sh | 3 ++- .../0_stateless/01429_empty_arrow_and_parquet.sh | 2 ++ .../0_stateless/01442_merge_detach_attach.sh | 2 ++ .../queries/0_stateless/01443_merge_truncate.sh | 1 + .../01444_create_table_drop_database_race.sh | 2 ++ .../01445_create_table_as_table_function.sh | 2 ++ .../0_stateless/01446_json_strings_each_row.sh | 1 + tests/queries/0_stateless/01451_dist_logs.sh | 1 + .../0_stateless/01451_wrong_error_long_query.sh | 5 +++-- .../01454_storagememory_data_race_challenge.sh | 1 + .../01455_opentelemetry_distributed.sh | 1 + .../01459_manual_write_to_replicas.sh | 1 + .../01459_manual_write_to_replicas_quorum.sh | 1 + .../0_stateless/01460_line_as_string_format.sh | 5 +++-- .../queries/0_stateless/01472_obfuscator_uuid.sh | 1 + .../queries/0_stateless/01474_custom_null_tsv.sh | 1 + .../0_stateless/01500_StorageFile_write_to_fd.sh | 1 + .../01501_clickhouse_client_INSERT_exception.sh | 1 + .../0_stateless/01502_jemalloc_percpu_arena.sh | 1 + .../01502_log_tinylog_deadlock_race.sh | 1 + .../0_stateless/01505_pipeline_executor_UAF.sh | 1 + ...ickhouse_server_start_with_embedded_config.sh | 1 + .../0_stateless/01508_format_regexp_raw.sh | 1 + .../0_stateless/01508_partition_pruning.sh | 1 + .../0_stateless/01508_query_obfuscator.sh | 1 + ...1508_race_condition_rename_clear_zookeeper.sh | 1 + .../01509_check_many_parallel_quorum_inserts.sh | 1 + .../01509_check_parallel_quorum_inserts.sh | 1 + .../queries/0_stateless/01509_format_raw_blob.sh | 1 + .../01509_parallel_quorum_and_merge.sh | 1 + .../01510_format_regexp_raw_low_cardinality.sh | 1 + .../01514_distributed_cancel_query_on_error.sh | 9 +++++---- .../0_stateless/01515_logtrace_function.sh | 1 + .../0_stateless/01516_drop_table_stress.sh | 1 + .../0_stateless/01520_client_print_query_id.sh | 1 + .../01523_client_local_queries_file_parameter.sh | 1 + .../0_stateless/01526_client_start_and_exit.sh | 1 + .../0_stateless/01526_initial_query_id.sh | 1 + .../0_stateless/01526_max_untracked_memory.sh | 1 + tests/queries/0_stateless/01526_param_uuid.sh | 1 + .../01527_clickhouse_local_optimize.sh | 1 + .../01528_clickhouse_local_prepare_parts.sh | 1 + tests/queries/0_stateless/01528_play.sh | 1 + .../0_stateless/01529_bad_memory_tracking.sh | 1 + .../01532_clickhouse_local_tmp_folder.sh | 1 + .../01541_max_memory_usage_for_user.sh | 1 + .../01542_dictionary_load_exception_race.sh | 1 + .../01543_avro_deserialization_with_lc.sh | 1 + .../0_stateless/01544_file_engine_settings.sh | 1 + tests/queries/0_stateless/01545_system_errors.sh | 1 + .../01548_create_table_compound_column_format.sh | 1 + .../01548_parallel_parsing_max_memory.sh | 1 + .../01550_query_identifier_parameters.sh | 1 + .../0_stateless/01550_type_map_formats_input.sh | 1 + ...1554_row_number_after_cannot_read_all_data.sh | 1 + tests/queries/0_stateless/01558_ttest_scipy.sh | 1 + .../01559_misplaced_codec_diagnostics.sh | 1 + .../0_stateless/01561_clickhouse_client_stage.sh | 1 + .../0_stateless/01561_mann_whitney_scipy.sh | 1 + .../01563_distributed_query_finish.sh | 1 + .../01576_alter_low_cardinality_and_select.sh | 1 + .../0_stateless/01582_distinct_optimization.sh | 1 + ...583_parallel_parsing_exception_with_offset.sh | 1 + .../01593_concurrent_alter_mutations_kill.sh | 1 + ...current_alter_mutations_kill_many_replicas.sh | 1 + .../0_stateless/01594_too_low_memory_limits.sh | 1 + .../0_stateless/01597_columns_list_ignored.sh | 1 + .../0_stateless/01599_mutation_query_params.sh | 1 + .../queries/0_stateless/01600_benchmark_query.sh | 1 + .../0_stateless/01600_detach_permanently.sh | 3 ++- .../01600_log_queries_with_extensive_info.sh | 1 + .../0_stateless/01600_quota_by_forwarded_ip.sh | 1 + .../queries/0_stateless/01601_proxy_protocol.sh | 1 + .../01601_temporary_table_session_scope.sh | 1 + .../01602_max_distributed_connections.sh | 1 + tests/queries/0_stateless/01606_git_import.sh | 1 + .../0_stateless/01607_arrays_as_nested_csv.sh | 1 + .../1_stateful/00090_thread_pool_deadlock.sh | 1 + tests/queries/1_stateful/00092_obfuscator.sh | 1 + utils/check-style/check-style | 6 ------ utils/check-style/shellcheck-run.sh | 9 +++++++++ 440 files changed, 510 insertions(+), 44 deletions(-) create mode 100755 utils/check-style/shellcheck-run.sh diff --git a/docker/test/coverage/run.sh b/docker/test/coverage/run.sh index e2369a28a9a..11b6ce13ea1 100755 --- a/docker/test/coverage/run.sh +++ b/docker/test/coverage/run.sh @@ -102,11 +102,11 @@ else echo "No failed tests" fi -mkdir -p $COVERAGE_DIR -mv /*.profraw $COVERAGE_DIR +mkdir -p "$COVERAGE_DIR" +mv /*.profraw "$COVERAGE_DIR" -mkdir -p $SOURCE_DIR/obj-x86_64-linux-gnu -cd $SOURCE_DIR/obj-x86_64-linux-gnu && CC=clang-11 CXX=clang++-11 cmake .. && cd / -llvm-profdata-11 merge -sparse ${COVERAGE_DIR}/* -o clickhouse.profdata -llvm-cov-11 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex $IGNORE > output.lcov -genhtml output.lcov --ignore-errors source --output-directory ${OUTPUT_DIR} +mkdir -p "$SOURCE_DIR"/obj-x86_64-linux-gnu +cd "$SOURCE_DIR"/obj-x86_64-linux-gnu && CC=clang-11 CXX=clang++-11 cmake .. && cd / +llvm-profdata-11 merge -sparse "${COVERAGE_DIR}"/* -o clickhouse.profdata +llvm-cov-11 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex "$IGNORE" > output.lcov +genhtml output.lcov --ignore-errors source --output-directory "${OUTPUT_DIR}" diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index a918cc44420..c782ac49d27 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -65,7 +65,7 @@ function start_server { set -m # Spawn server in its own process groups local opts=( - --config-file="$FASTTEST_DATA/config.xml" + --config-file "$FASTTEST_DATA/config.xml" -- --path "$FASTTEST_DATA" --user_files_path "$FASTTEST_DATA/user_files" diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 59d7cc98063..674e8dd1cab 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -50,13 +50,13 @@ function configure local setup_left_server_opts=( # server options - --config-file=left/config/config.xml + --config-file left/config/config.xml -- # server *config* directives overrides --path db0 --user_files_path db0/user_files --top_level_domains_path /top_level_domains - --tcp_port $LEFT_SERVER_PORT + --tcp_port "$LEFT_SERVER_PORT" ) left/clickhouse-server "${setup_left_server_opts[@]}" &> setup-server-log.log & left_pid=$! @@ -98,13 +98,13 @@ function restart local left_server_opts=( # server options - --config-file=left/config/config.xml + --config-file left/config/config.xml -- # server *config* directives overrides --path left/db --user_files_path left/db/user_files --top_level_domains_path /top_level_domains - --tcp_port $LEFT_SERVER_PORT + --tcp_port "$LEFT_SERVER_PORT" ) left/clickhouse-server "${left_server_opts[@]}" &>> left-server-log.log & left_pid=$! @@ -113,13 +113,13 @@ function restart local right_server_opts=( # server options - --config-file=right/config/config.xml + --config-file right/config/config.xml -- # server *config* directives overrides --path right/db --user_files_path right/db/user_files --top_level_domains_path /top_level_domains - --tcp_port $RIGHT_SERVER_PORT + --tcp_port "$RIGHT_SERVER_PORT" ) right/clickhouse-server "${right_server_opts[@]}" &>> right-server-log.log & right_pid=$! @@ -1109,7 +1109,7 @@ function upload_results then echo Database for test results is not specified, will not upload them. return 0 - fi + fi # Surprisingly, clickhouse-client doesn't understand --host 127.0.0.1:9000 # so I have to extract host and port with clickhouse-local. I tried to use @@ -1117,7 +1117,7 @@ function upload_results # parse host:port. set +x # Don't show password in the log clickhouse-client \ - $(clickhouse-local --query "with '${CHPC_DATABASE_URL}' as url select '--host ' || domain(url) || ' --port ' || toString(port(url)) format TSV") \ + "$(clickhouse-local --query "with '${CHPC_DATABASE_URL}' as url select '--host ' || domain(url) || ' --port ' || toString(port(url)) format TSV")" \ --secure \ --user "${CHPC_DATABASE_USER}" \ --password "${CHPC_DATABASE_PASSWORD}" \ diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index e6e987e1d94..309328bc8e2 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -55,9 +55,9 @@ function run_tests() ADDITIONAL_OPTIONS+=('00000_no_tests_to_skip') fi - for i in $(seq 1 $NUM_TRIES); do + for _ in $(seq 1 "$NUM_TRIES"); do clickhouse-test --testname --shard --zookeeper --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a test_output/test_result.txt - if [ ${PIPESTATUS[0]} -ne "0" ]; then + if [ "${PIPESTATUS[0]}" -ne "0" ]; then break; fi done @@ -65,4 +65,4 @@ function run_tests() export -f run_tests -timeout $MAX_RUN_TIME bash -c run_tests ||: +timeout "$MAX_RUN_TIME" bash -c run_tests ||: diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index 239a074969c..f7555231ffb 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -8,4 +8,5 @@ CMD cd /ClickHouse/utils/check-style && \ ./check-style -n | tee /test_output/style_output.txt && \ ./check-typos | tee /test_output/typos_output.txt && \ ./check-whitespaces -n | tee /test_output/whitespaces_output.txt && \ - ./check-duplicate-includes.sh | tee /test_output/duplicate_output.txt + ./check-duplicate-includes.sh | tee /test_output/duplicate_output.txt && \ + ./shellcheck-run.sh | tee /test_output/shellcheck_output.txt diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.h b/src/Storages/MergeTree/MergeTreeIOSettings.h index 185211af84b..2dec16e7d10 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -46,7 +46,7 @@ struct MergeTreeWriterSettings bool rewrite_primary_key; bool blocks_are_granules_size; - /// Used for AIO threshold comparsion + /// Used for AIO threshold comparison /// FIXME currently doesn't work because WriteBufferAIO contain obscure bug(s) size_t estimated_size = 0; }; diff --git a/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh b/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh index 3f09e256b02..86f1d1f161c 100755 --- a/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh +++ b/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/00039_inserts_through_http.sh b/tests/queries/0_stateless/00039_inserts_through_http.sh index 35abcd166d7..2eaa4393935 100755 --- a/tests/queries/0_stateless/00039_inserts_through_http.sh +++ b/tests/queries/0_stateless/00039_inserts_through_http.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo 'DROP TABLE IF EXISTS long_insert' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}" -d @- diff --git a/tests/queries/0_stateless/00070_insert_fewer_columns_http.sh b/tests/queries/0_stateless/00070_insert_fewer_columns_http.sh index 0cf5f95d3d9..52076767981 100755 --- a/tests/queries/0_stateless/00070_insert_fewer_columns_http.sh +++ b/tests/queries/0_stateless/00070_insert_fewer_columns_http.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo 'DROP TABLE IF EXISTS insert_fewer_columns' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}" -d @- diff --git a/tests/queries/0_stateless/00090_union_race_conditions_1.sh b/tests/queries/0_stateless/00090_union_race_conditions_1.sh index a2da4c461dd..afec8b5bac9 100755 --- a/tests/queries/0_stateless/00090_union_race_conditions_1.sh +++ b/tests/queries/0_stateless/00090_union_race_conditions_1.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -o errexit diff --git a/tests/queries/0_stateless/00091_union_race_conditions_2.sh b/tests/queries/0_stateless/00091_union_race_conditions_2.sh index 94df04e824a..78a6cca2b2b 100755 --- a/tests/queries/0_stateless/00091_union_race_conditions_2.sh +++ b/tests/queries/0_stateless/00091_union_race_conditions_2.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -o errexit diff --git a/tests/queries/0_stateless/00092_union_race_conditions_3.sh b/tests/queries/0_stateless/00092_union_race_conditions_3.sh index 2d74f853ecd..9be2613f70a 100755 --- a/tests/queries/0_stateless/00092_union_race_conditions_3.sh +++ b/tests/queries/0_stateless/00092_union_race_conditions_3.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -o errexit diff --git a/tests/queries/0_stateless/00093_union_race_conditions_4.sh b/tests/queries/0_stateless/00093_union_race_conditions_4.sh index d33b2aacc06..ab1a025c5b2 100755 --- a/tests/queries/0_stateless/00093_union_race_conditions_4.sh +++ b/tests/queries/0_stateless/00093_union_race_conditions_4.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -o errexit diff --git a/tests/queries/0_stateless/00094_union_race_conditions_5.sh b/tests/queries/0_stateless/00094_union_race_conditions_5.sh index 774c4f1e54e..b546b6a0092 100755 --- a/tests/queries/0_stateless/00094_union_race_conditions_5.sh +++ b/tests/queries/0_stateless/00094_union_race_conditions_5.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -o errexit diff --git a/tests/queries/0_stateless/00097_long_storage_buffer_race_condition.sh b/tests/queries/0_stateless/00097_long_storage_buffer_race_condition.sh index 26502247055..b3c0b37e0c0 100755 --- a/tests/queries/0_stateless/00097_long_storage_buffer_race_condition.sh +++ b/tests/queries/0_stateless/00097_long_storage_buffer_race_condition.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh [ "$NO_SHELL_CONFIG" ] || . "$CURDIR"/../shell_config.sh seq 1 1000 | sed -r 's/.+/CREATE TABLE IF NOT EXISTS buf_00097 (a UInt8) ENGINE = Buffer('$CLICKHOUSE_DATABASE', b, 1, 1, 1, 1, 1, 1, 1); DROP TABLE buf_00097;/' | $CLICKHOUSE_CLIENT -n diff --git a/tests/queries/0_stateless/00097_long_storage_buffer_race_condition_mt.sh b/tests/queries/0_stateless/00097_long_storage_buffer_race_condition_mt.sh index b0cc62cc652..3856da7f214 100755 --- a/tests/queries/0_stateless/00097_long_storage_buffer_race_condition_mt.sh +++ b/tests/queries/0_stateless/00097_long_storage_buffer_race_condition_mt.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh export NO_SHELL_CONFIG=1 diff --git a/tests/queries/0_stateless/00100_subquery_table_identifier.sh b/tests/queries/0_stateless/00100_subquery_table_identifier.sh index 2a42e9a0e70..e20939bc992 100755 --- a/tests/queries/0_stateless/00100_subquery_table_identifier.sh +++ b/tests/queries/0_stateless/00100_subquery_table_identifier.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="SELECT sum(dummy) FROM remote('localhost', system, one) WHERE 1 GLOBAL IN (SELECT 1)" diff --git a/tests/queries/0_stateless/00115_shard_in_incomplete_result.sh b/tests/queries/0_stateless/00115_shard_in_incomplete_result.sh index 855c5f8ac86..1348989e244 100755 --- a/tests/queries/0_stateless/00115_shard_in_incomplete_result.sh +++ b/tests/queries/0_stateless/00115_shard_in_incomplete_result.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -o errexit diff --git a/tests/queries/0_stateless/00133_long_shard_memory_tracker_and_exception_safety.sh b/tests/queries/0_stateless/00133_long_shard_memory_tracker_and_exception_safety.sh index 7dbd662c54b..05ebe9d19a8 100755 --- a/tests/queries/0_stateless/00133_long_shard_memory_tracker_and_exception_safety.sh +++ b/tests/queries/0_stateless/00133_long_shard_memory_tracker_and_exception_safety.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -n --query=" diff --git a/tests/queries/0_stateless/00155_long_merges.sh b/tests/queries/0_stateless/00155_long_merges.sh index 863575daf11..c2aafaf0c95 100755 --- a/tests/queries/0_stateless/00155_long_merges.sh +++ b/tests/queries/0_stateless/00155_long_merges.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh function create { diff --git a/tests/queries/0_stateless/00177_inserts_through_http_parts.sh b/tests/queries/0_stateless/00177_inserts_through_http_parts.sh index bae77c56e13..72a9d4fa16f 100755 --- a/tests/queries/0_stateless/00177_inserts_through_http_parts.sh +++ b/tests/queries/0_stateless/00177_inserts_through_http_parts.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=DROP+TABLE" -d 'IF EXISTS insert' diff --git a/tests/queries/0_stateless/00186_very_long_arrays.sh b/tests/queries/0_stateless/00186_very_long_arrays.sh index 24ff6e098b6..26a4496a85b 100755 --- a/tests/queries/0_stateless/00186_very_long_arrays.sh +++ b/tests/queries/0_stateless/00186_very_long_arrays.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh (echo 'SELECT number FROM system.numbers WHERE transform(number, ['; seq 1 100000 | tr '\n' ','; echo '0],['; seq 1 100000 | tr '\n' ','; echo '0]) = 10000000 LIMIT 1';) | $CLICKHOUSE_CLIENT --max_query_size=100000000 diff --git a/tests/queries/0_stateless/00210_insert_select_extremes_http.sh b/tests/queries/0_stateless/00210_insert_select_extremes_http.sh index 5a944c58466..e9c6257d848 100755 --- a/tests/queries/0_stateless/00210_insert_select_extremes_http.sh +++ b/tests/queries/0_stateless/00210_insert_select_extremes_http.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1" -d @- <<< "DROP TABLE IF EXISTS test_00210" diff --git a/tests/queries/0_stateless/00265_http_content_type_format_timezone.sh b/tests/queries/0_stateless/00265_http_content_type_format_timezone.sh index 9e92a997f51..74cbbe7f71d 100755 --- a/tests/queries/0_stateless/00265_http_content_type_format_timezone.sh +++ b/tests/queries/0_stateless/00265_http_content_type_format_timezone.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh CLICKHOUSE_TIMEZONE_ESCAPED=$($CLICKHOUSE_CLIENT --query="SELECT timezone()" | sed 's/[]\/$*.^+:()[]/\\&/g') diff --git a/tests/queries/0_stateless/00301_csv.sh b/tests/queries/0_stateless/00301_csv.sh index 6d9c5fc4fed..0aee9abe25c 100755 --- a/tests/queries/0_stateless/00301_csv.sh +++ b/tests/queries/0_stateless/00301_csv.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS csv"; diff --git a/tests/queries/0_stateless/00302_http_compression.sh b/tests/queries/0_stateless/00302_http_compression.sh index 1727d5ab993..829475e8602 100755 --- a/tests/queries/0_stateless/00302_http_compression.sh +++ b/tests/queries/0_stateless/00302_http_compression.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&enable_http_compression=1" -d 'SELECT number FROM system.numbers LIMIT 10'; diff --git a/tests/queries/0_stateless/00304_http_external_data.sh b/tests/queries/0_stateless/00304_http_external_data.sh index e088540120c..41a9dea1ebb 100755 --- a/tests/queries/0_stateless/00304_http_external_data.sh +++ b/tests/queries/0_stateless/00304_http_external_data.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo -ne '1,Hello\n2,World\n' | ${CLICKHOUSE_CURL} -sSF 'file=@-' "${CLICKHOUSE_URL}&query=SELECT+*+FROM+file&file_format=CSV&file_types=UInt8,String"; diff --git a/tests/queries/0_stateless/00305_http_and_readonly.sh b/tests/queries/0_stateless/00305_http_and_readonly.sh index 969f3b42e92..dd9f116be7a 100755 --- a/tests/queries/0_stateless/00305_http_and_readonly.sh +++ b/tests/queries/0_stateless/00305_http_and_readonly.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # POST permits everything. diff --git a/tests/queries/0_stateless/00310_tskv.sh b/tests/queries/0_stateless/00310_tskv.sh index 419bb7d228d..73b6581ac0d 100755 --- a/tests/queries/0_stateless/00310_tskv.sh +++ b/tests/queries/0_stateless/00310_tskv.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS tskv"; diff --git a/tests/queries/0_stateless/00313_const_totals_extremes.sh b/tests/queries/0_stateless/00313_const_totals_extremes.sh index eff01bfd511..0c51d80abe8 100755 --- a/tests/queries/0_stateless/00313_const_totals_extremes.sh +++ b/tests/queries/0_stateless/00313_const_totals_extremes.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1&output_format_write_statistics=0" -d "SELECT 1 AS k, count() GROUP BY k WITH TOTALS"; diff --git a/tests/queries/0_stateless/00322_disable_checksumming.sh b/tests/queries/0_stateless/00322_disable_checksumming.sh index f06b8609d01..e04ec076f80 100755 --- a/tests/queries/0_stateless/00322_disable_checksumming.sh +++ b/tests/queries/0_stateless/00322_disable_checksumming.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo -ne '\x50\x74\x32\xf2\x59\xe9\x8a\xdb\x37\xc6\x4a\xa7\xfb\x22\xc4\x39''\x82\x13\x00\x00\x00\x09\x00\x00\x00''\x90SELECT 1\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&decompress=1" --data-binary @- diff --git a/tests/queries/0_stateless/00335_bom.sh b/tests/queries/0_stateless/00335_bom.sh index 75622f0ba42..b8bcbb7d635 100755 --- a/tests/queries/0_stateless/00335_bom.sh +++ b/tests/queries/0_stateless/00335_bom.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo 'DROP TABLE IF EXISTS bom' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data-binary @- diff --git a/tests/queries/0_stateless/00336_shard_stack_trace.sh b/tests/queries/0_stateless/00336_shard_stack_trace.sh index a096d7a71c6..19389ec11c1 100755 --- a/tests/queries/0_stateless/00336_shard_stack_trace.sh +++ b/tests/queries/0_stateless/00336_shard_stack_trace.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'SELECT a' | wc -l diff --git a/tests/queries/0_stateless/00339_parsing_bad_arrays.sh b/tests/queries/0_stateless/00339_parsing_bad_arrays.sh index 2794cceb65d..51ffd8f9814 100755 --- a/tests/queries/0_stateless/00339_parsing_bad_arrays.sh +++ b/tests/queries/0_stateless/00339_parsing_bad_arrays.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'DROP TABLE IF EXISTS bad_arrays' diff --git a/tests/queries/0_stateless/00354_host_command_line_option.sh b/tests/queries/0_stateless/00354_host_command_line_option.sh index 227908a7318..9d0d4d59bee 100755 --- a/tests/queries/0_stateless/00354_host_command_line_option.sh +++ b/tests/queries/0_stateless/00354_host_command_line_option.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh clickhouse_client_removed_host_parameter --host="${CLICKHOUSE_HOST}" --query="SELECT 1"; diff --git a/tests/queries/0_stateless/00365_statistics_in_formats.sh b/tests/queries/0_stateless/00365_statistics_in_formats.sh index d395f543ce7..724a5dc5fde 100755 --- a/tests/queries/0_stateless/00365_statistics_in_formats.sh +++ b/tests/queries/0_stateless/00365_statistics_in_formats.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS numbers"; diff --git a/tests/queries/0_stateless/00366_multi_statements.sh b/tests/queries/0_stateless/00366_multi_statements.sh index e8daffa79b1..9b885bb1b32 100755 --- a/tests/queries/0_stateless/00366_multi_statements.sh +++ b/tests/queries/0_stateless/00366_multi_statements.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="SELECT 1" diff --git a/tests/queries/0_stateless/00368_format_option_collision.sh b/tests/queries/0_stateless/00368_format_option_collision.sh index 425a90a1301..323de47428c 100755 --- a/tests/queries/0_stateless/00368_format_option_collision.sh +++ b/tests/queries/0_stateless/00368_format_option_collision.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh clickhouse_client_removed_host_parameter --host="${CLICKHOUSE_HOST}" --query="SELECT * FROM ext" --format=Vertical --external --file=- --structure="s String" --name=ext --format=JSONEachRow <<< '{"s":"Hello"}' diff --git a/tests/queries/0_stateless/00372_cors_header.sh b/tests/queries/0_stateless/00372_cors_header.sh index 6eec46fd942..8af6ee09876 100755 --- a/tests/queries/0_stateless/00372_cors_header.sh +++ b/tests/queries/0_stateless/00372_cors_header.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&add_http_cors_header=1" -H "Origin:smi2.ru" --data-binary @- <<< "SELECT 1" 2>&1 | grep -F "< Access-Control-Allow-Origin: *" | wc -l diff --git a/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.sh b/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.sh index c614b35bdfd..c9c53dedd69 100755 --- a/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.sh +++ b/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS json_noisy" diff --git a/tests/queries/0_stateless/00379_system_processes_port.sh b/tests/queries/0_stateless/00379_system_processes_port.sh index edbcea8c4f8..99413af98cf 100755 --- a/tests/queries/0_stateless/00379_system_processes_port.sh +++ b/tests/queries/0_stateless/00379_system_processes_port.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -sS --local-port 1390 "${CLICKHOUSE_URL}&query_id=my_id&query=SELECT+port+FROM+system.processes+WHERE+query_id%3D%27my_id%27+ORDER+BY+elapsed+LIMIT+1" diff --git a/tests/queries/0_stateless/00380_client_break_at_exception_in_batch_mode.sh b/tests/queries/0_stateless/00380_client_break_at_exception_in_batch_mode.sh index d58928ecfeb..d24d029c4e7 100755 --- a/tests/queries/0_stateless/00380_client_break_at_exception_in_batch_mode.sh +++ b/tests/queries/0_stateless/00380_client_break_at_exception_in_batch_mode.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --multiquery --query="SELECT 1; SELECT xyz; SELECT 2;" 2> /dev/null || true; diff --git a/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app.sh b/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app.sh index 7f6d5d32a46..ef0ec1ae842 100755 --- a/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app.sh +++ b/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh TABLE_HASH="cityHash64(groupArray(cityHash64(*)))" diff --git a/tests/queries/0_stateless/00386_long_in_pk.sh b/tests/queries/0_stateless/00386_long_in_pk.sh index 8cad8f93a13..66cc4ccc227 100755 --- a/tests/queries/0_stateless/00386_long_in_pk.sh +++ b/tests/queries/0_stateless/00386_long_in_pk.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # We should have correct env vars from shell_config.sh to run this test diff --git a/tests/queries/0_stateless/00387_use_client_time_zone.sh b/tests/queries/0_stateless/00387_use_client_time_zone.sh index a86f866a6ad..201277b76d6 100755 --- a/tests/queries/0_stateless/00387_use_client_time_zone.sh +++ b/tests/queries/0_stateless/00387_use_client_time_zone.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh env TZ=UTC ${CLICKHOUSE_CLIENT} --use_client_time_zone=1 --query="SELECT toDateTime(1000000000)" diff --git a/tests/queries/0_stateless/00400_client_external_options.sh b/tests/queries/0_stateless/00400_client_external_options.sh index d519ca16a63..c2c6f44d62e 100755 --- a/tests/queries/0_stateless/00400_client_external_options.sh +++ b/tests/queries/0_stateless/00400_client_external_options.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo -ne "1\n2\n3\n" | $CLICKHOUSE_CLIENT --query="SELECT * FROM _data" --external --file=- --types=Int8; diff --git a/tests/queries/0_stateless/00405_PrettyCompactMonoBlock.sh b/tests/queries/0_stateless/00405_PrettyCompactMonoBlock.sh index a353ce5ea5c..fb89199acb1 100755 --- a/tests/queries/0_stateless/00405_PrettyCompactMonoBlock.sh +++ b/tests/queries/0_stateless/00405_PrettyCompactMonoBlock.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo 'one block' diff --git a/tests/queries/0_stateless/00407_parsing_nulls.sh b/tests/queries/0_stateless/00407_parsing_nulls.sh index 8618d4da8de..cc627dbb97c 100755 --- a/tests/queries/0_stateless/00407_parsing_nulls.sh +++ b/tests/queries/0_stateless/00407_parsing_nulls.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo -ne '\\tHello\t123\t\\N\n\\N\t\t2000-01-01 00:00:00\n' | ${CLICKHOUSE_LOCAL} --input-format=TabSeparated --output-format=TabSeparated --structure='s Nullable(String), x Nullable(UInt64), t Nullable(DateTime)' --query="SELECT * FROM table" diff --git a/tests/queries/0_stateless/00408_http_keep_alive.sh b/tests/queries/0_stateless/00408_http_keep_alive.sh index d0c14720cd8..4bd0e494eb8 100755 --- a/tests/queries/0_stateless/00408_http_keep_alive.sh +++ b/tests/queries/0_stateless/00408_http_keep_alive.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh URL="${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/" diff --git a/tests/queries/0_stateless/00411_long_accurate_number_comparison_float.sh b/tests/queries/0_stateless/00411_long_accurate_number_comparison_float.sh index 17d0c7564e3..d03e02efc55 100755 --- a/tests/queries/0_stateless/00411_long_accurate_number_comparison_float.sh +++ b/tests/queries/0_stateless/00411_long_accurate_number_comparison_float.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # We should have correct env vars from shell_config.sh to run this test diff --git a/tests/queries/0_stateless/00411_long_accurate_number_comparison_int1.sh b/tests/queries/0_stateless/00411_long_accurate_number_comparison_int1.sh index 43d9d550ddf..c2c5491ed90 100755 --- a/tests/queries/0_stateless/00411_long_accurate_number_comparison_int1.sh +++ b/tests/queries/0_stateless/00411_long_accurate_number_comparison_int1.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # We should have correct env vars from shell_config.sh to run this test diff --git a/tests/queries/0_stateless/00411_long_accurate_number_comparison_int2.sh b/tests/queries/0_stateless/00411_long_accurate_number_comparison_int2.sh index 34aaf9ef7ed..539bdd297bd 100755 --- a/tests/queries/0_stateless/00411_long_accurate_number_comparison_int2.sh +++ b/tests/queries/0_stateless/00411_long_accurate_number_comparison_int2.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # We should have correct env vars from shell_config.sh to run this test diff --git a/tests/queries/0_stateless/00411_long_accurate_number_comparison_int3.sh b/tests/queries/0_stateless/00411_long_accurate_number_comparison_int3.sh index 139792944ee..a0e25e8cb0a 100755 --- a/tests/queries/0_stateless/00411_long_accurate_number_comparison_int3.sh +++ b/tests/queries/0_stateless/00411_long_accurate_number_comparison_int3.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # We should have correct env vars from shell_config.sh to run this test diff --git a/tests/queries/0_stateless/00411_long_accurate_number_comparison_int4.sh b/tests/queries/0_stateless/00411_long_accurate_number_comparison_int4.sh index f57099e77ca..b09a05baf69 100755 --- a/tests/queries/0_stateless/00411_long_accurate_number_comparison_int4.sh +++ b/tests/queries/0_stateless/00411_long_accurate_number_comparison_int4.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # We should have correct env vars from shell_config.sh to run this test diff --git a/tests/queries/0_stateless/00415_into_outfile.sh b/tests/queries/0_stateless/00415_into_outfile.sh index 6ceeb7297d6..77dc96a48e6 100755 --- a/tests/queries/0_stateless/00415_into_outfile.sh +++ b/tests/queries/0_stateless/00415_into_outfile.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh function perform() diff --git a/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh b/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh index 31f5e0f0f43..5d9cd12e4bf 100755 --- a/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh +++ b/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&max_block_size=5&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT max(number) FROM numbers(10)' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' diff --git a/tests/queries/0_stateless/00417_kill_query.sh b/tests/queries/0_stateless/00417_kill_query.sh index cb08f8c009c..ce4c5851762 100755 --- a/tests/queries/0_stateless/00417_kill_query.sh +++ b/tests/queries/0_stateless/00417_kill_query.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh QUERY_FIELND_NUM=4 diff --git a/tests/queries/0_stateless/00417_system_build_options.sh b/tests/queries/0_stateless/00417_system_build_options.sh index 34f9d4dfc5d..bfdfa7d14ce 100755 --- a/tests/queries/0_stateless/00417_system_build_options.sh +++ b/tests/queries/0_stateless/00417_system_build_options.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="SELECT * FROM system.build_options" | perl -lnE 'print $1 if /(BUILD_DATE|BUILD_TYPE|CXX_COMPILER)\s+\S+/ || /(CXX_FLAGS|LINK_FLAGS|TZDATA_VERSION)/'; diff --git a/tests/queries/0_stateless/00418_input_format_allow_errors.sh b/tests/queries/0_stateless/00418_input_format_allow_errors.sh index 762e35fa8cd..b27c6f3fe29 100755 --- a/tests/queries/0_stateless/00418_input_format_allow_errors.sh +++ b/tests/queries/0_stateless/00418_input_format_allow_errors.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS formats_test" diff --git a/tests/queries/0_stateless/00419_show_sql_queries.sh b/tests/queries/0_stateless/00419_show_sql_queries.sh index a76e9020db0..1737e874ff2 100755 --- a/tests/queries/0_stateless/00419_show_sql_queries.sh +++ b/tests/queries/0_stateless/00419_show_sql_queries.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "SHOW PROCESSLIST" &>/dev/null diff --git a/tests/queries/0_stateless/00421_storage_merge__table_index.sh b/tests/queries/0_stateless/00421_storage_merge__table_index.sh index 769be11f56b..c4af0528f7d 100755 --- a/tests/queries/0_stateless/00421_storage_merge__table_index.sh +++ b/tests/queries/0_stateless/00421_storage_merge__table_index.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh for i in $(seq -w 0 2 20); do diff --git a/tests/queries/0_stateless/00427_alter_primary_key.sh b/tests/queries/0_stateless/00427_alter_primary_key.sh index 352263aad40..4ad1166bfa4 100755 --- a/tests/queries/0_stateless/00427_alter_primary_key.sh +++ b/tests/queries/0_stateless/00427_alter_primary_key.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh function perform() diff --git a/tests/queries/0_stateless/00429_long_http_bufferization.sh b/tests/queries/0_stateless/00429_long_http_bufferization.sh index 83c9992218d..aab9aeba937 100755 --- a/tests/queries/0_stateless/00429_long_http_bufferization.sh +++ b/tests/queries/0_stateless/00429_long_http_bufferization.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh function query { diff --git a/tests/queries/0_stateless/00430_https_server.sh b/tests/queries/0_stateless/00430_https_server.sh index f562f108791..10df01e2c93 100755 --- a/tests/queries/0_stateless/00430_https_server.sh +++ b/tests/queries/0_stateless/00430_https_server.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # TODO: enable this test with self-signed server cert diff --git a/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh b/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh index cc19a608c20..32633a458cc 100755 --- a/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh +++ b/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh table="optimize_me_finally" diff --git a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh index be986a72457..724630057d9 100755 --- a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh +++ b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS preferred_block_size_bytes" diff --git a/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh b/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh index 97830b828b6..60de1822318 100755 --- a/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh +++ b/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ch="$CLICKHOUSE_CLIENT --stacktrace -q" diff --git a/tests/queries/0_stateless/00463_long_sessions_in_http_interface.sh b/tests/queries/0_stateless/00463_long_sessions_in_http_interface.sh index c49924741df..d3bdf32db74 100755 --- a/tests/queries/0_stateless/00463_long_sessions_in_http_interface.sh +++ b/tests/queries/0_stateless/00463_long_sessions_in_http_interface.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh request() { diff --git a/tests/queries/0_stateless/00473_output_format_json_quote_denormals.sh b/tests/queries/0_stateless/00473_output_format_json_quote_denormals.sh index 46f051639d1..47f5c698d17 100755 --- a/tests/queries/0_stateless/00473_output_format_json_quote_denormals.sh +++ b/tests/queries/0_stateless/00473_output_format_json_quote_denormals.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="select 1/0, -1/0, sqrt(-1), -sqrt(-1) format JSON" --output_format_json_quote_denormals=0 | grep -o null diff --git a/tests/queries/0_stateless/00474_readonly_settings.sh b/tests/queries/0_stateless/00474_readonly_settings.sh index 04134c24be7..0edde9f12ed 100755 --- a/tests/queries/0_stateless/00474_readonly_settings.sh +++ b/tests/queries/0_stateless/00474_readonly_settings.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="select toUInt64(pow(2, 62)) as value format JSON" --output_format_json_quote_64bit_integers=0 | grep value diff --git a/tests/queries/0_stateless/00485_http_insert_format.sh b/tests/queries/0_stateless/00485_http_insert_format.sh index 358fb09bf67..b5bf36e6c36 100755 --- a/tests/queries/0_stateless/00485_http_insert_format.sh +++ b/tests/queries/0_stateless/00485_http_insert_format.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS format" diff --git a/tests/queries/0_stateless/00497_whitespaces_in_insert.sh b/tests/queries/0_stateless/00497_whitespaces_in_insert.sh index 6869d79420e..7a799792dd0 100755 --- a/tests/queries/0_stateless/00497_whitespaces_in_insert.sh +++ b/tests/queries/0_stateless/00497_whitespaces_in_insert.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS ws"; diff --git a/tests/queries/0_stateless/00501_http_head.sh b/tests/queries/0_stateless/00501_http_head.sh index 318e55fdcee..60283f26833 100755 --- a/tests/queries/0_stateless/00501_http_head.sh +++ b/tests/queries/0_stateless/00501_http_head.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ( ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}&query=SELECT%201"; diff --git a/tests/queries/0_stateless/00504_insert_miss_columns.sh b/tests/queries/0_stateless/00504_insert_miss_columns.sh index 98d6249c758..ea699ab58a5 100755 --- a/tests/queries/0_stateless/00504_insert_miss_columns.sh +++ b/tests/queries/0_stateless/00504_insert_miss_columns.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # https://github.com/ClickHouse/ClickHouse/issues/1300 diff --git a/tests/queries/0_stateless/00505_secure.sh b/tests/queries/0_stateless/00505_secure.sh index d1968ffe638..3d9e28ba08d 100755 --- a/tests/queries/0_stateless/00505_secure.sh +++ b/tests/queries/0_stateless/00505_secure.sh @@ -3,6 +3,7 @@ # set -x CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # Not default server config needed diff --git a/tests/queries/0_stateless/00505_shard_secure.sh b/tests/queries/0_stateless/00505_shard_secure.sh index c9200f8804c..526176f8c39 100755 --- a/tests/queries/0_stateless/00505_shard_secure.sh +++ b/tests/queries/0_stateless/00505_shard_secure.sh @@ -3,6 +3,7 @@ # set -x CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "SELECT * FROM remoteSecure('127.0.0.{1,2}', system.one);" diff --git a/tests/queries/0_stateless/00507_array_no_params.sh b/tests/queries/0_stateless/00507_array_no_params.sh index 6ae0c1c2067..63829b30f76 100755 --- a/tests/queries/0_stateless/00507_array_no_params.sh +++ b/tests/queries/0_stateless/00507_array_no_params.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS foo;" diff --git a/tests/queries/0_stateless/00512_fractional_time_zones.sh b/tests/queries/0_stateless/00512_fractional_time_zones.sh index 748c4584d06..45be8fe8d17 100755 --- a/tests/queries/0_stateless/00512_fractional_time_zones.sh +++ b/tests/queries/0_stateless/00512_fractional_time_zones.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh TZ=Europe/Moscow ${CLICKHOUSE_LOCAL} --query="SELECT toDateTime('1990-10-19 00:00:00')" diff --git a/tests/queries/0_stateless/00520_http_nullable.sh b/tests/queries/0_stateless/00520_http_nullable.sh index 0f068d5a6fe..6dff8109b9f 100755 --- a/tests/queries/0_stateless/00520_http_nullable.sh +++ b/tests/queries/0_stateless/00520_http_nullable.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -sS "$CLICKHOUSE_URL" -d 'SELECT floor(NULL), 1;'; diff --git a/tests/queries/0_stateless/00531_client_ignore_error.sh b/tests/queries/0_stateless/00531_client_ignore_error.sh index 98cb297e237..daf636b0765 100755 --- a/tests/queries/0_stateless/00531_client_ignore_error.sh +++ b/tests/queries/0_stateless/00531_client_ignore_error.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo "SELECT 1; SELECT 2; SELECT CAST(); SELECT ';'; SELECT 3;SELECT CAST();SELECT 4;" | $CLICKHOUSE_CLIENT -n --ignore-error 2>/dev/null diff --git a/tests/queries/0_stateless/00534_client_ignore_error.sh b/tests/queries/0_stateless/00534_client_ignore_error.sh index 98cb297e237..daf636b0765 100755 --- a/tests/queries/0_stateless/00534_client_ignore_error.sh +++ b/tests/queries/0_stateless/00534_client_ignore_error.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo "SELECT 1; SELECT 2; SELECT CAST(); SELECT ';'; SELECT 3;SELECT CAST();SELECT 4;" | $CLICKHOUSE_CLIENT -n --ignore-error 2>/dev/null diff --git a/tests/queries/0_stateless/00534_filimonov.sh b/tests/queries/0_stateless/00534_filimonov.sh index 044b5c9e65d..a629c0f3a92 100755 --- a/tests/queries/0_stateless/00534_filimonov.sh +++ b/tests/queries/0_stateless/00534_filimonov.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # Server should not crash on any function trash calls diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments1.sh b/tests/queries/0_stateless/00534_functions_bad_arguments1.sh index 15bbdb1e4c7..2979e18e2a5 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments1.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments1.sh @@ -2,8 +2,10 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./00534_functions_bad_arguments.lib . "$CURDIR"/00534_functions_bad_arguments.lib test_variant 'SELECT $_;' diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments10.sh b/tests/queries/0_stateless/00534_functions_bad_arguments10.sh index 8d151d5573d..178c9dd8d94 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments10.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments10.sh @@ -2,8 +2,10 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./00534_functions_bad_arguments.lib . "$CURDIR"/00534_functions_bad_arguments.lib test_variant 'SELECT $_([NULL],[NULL]);' diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments11.sh b/tests/queries/0_stateless/00534_functions_bad_arguments11.sh index ac02a9e0a38..3004c6c84ff 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments11.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments11.sh @@ -2,8 +2,10 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./00534_functions_bad_arguments.lib . "$CURDIR"/00534_functions_bad_arguments.lib test_variant 'SELECT $_(NULL, NULL, NULL);' diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments12.sh b/tests/queries/0_stateless/00534_functions_bad_arguments12.sh index 7d8ba7e78f6..8e5bd15dc80 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments12.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments12.sh @@ -2,8 +2,10 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./00534_functions_bad_arguments.lib . "$CURDIR"/00534_functions_bad_arguments.lib test_variant 'SELECT $_([], [], []);' diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments13.sh b/tests/queries/0_stateless/00534_functions_bad_arguments13.sh index 3995ddefc15..37f4282ae79 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments13.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments13.sh @@ -2,8 +2,10 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./00534_functions_bad_arguments.lib . "$CURDIR"/00534_functions_bad_arguments.lib test_variant 'SELECT $_([NULL], [NULL], [NULL]);' diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments2.sh b/tests/queries/0_stateless/00534_functions_bad_arguments2.sh index 23b03a8d168..9c8eda6fbcb 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments2.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments2.sh @@ -2,8 +2,10 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./00534_functions_bad_arguments.lib . "$CURDIR"/00534_functions_bad_arguments.lib test_variant 'SELECT $_();' diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments3.sh b/tests/queries/0_stateless/00534_functions_bad_arguments3.sh index 316456045cd..640467e9d00 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments3.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments3.sh @@ -2,8 +2,10 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./00534_functions_bad_arguments.lib . "$CURDIR"/00534_functions_bad_arguments.lib test_variant 'SELECT $_(NULL);' diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments4.sh b/tests/queries/0_stateless/00534_functions_bad_arguments4.sh index 10fce23049b..b0381295b39 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments4.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments4.sh @@ -2,8 +2,10 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./00534_functions_bad_arguments.lib . "$CURDIR"/00534_functions_bad_arguments.lib test_variant 'SELECT $_([]);' diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments5.sh b/tests/queries/0_stateless/00534_functions_bad_arguments5.sh index fbed81b41d3..3e1d4be7af5 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments5.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments5.sh @@ -2,8 +2,10 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./00534_functions_bad_arguments.lib . "$CURDIR"/00534_functions_bad_arguments.lib test_variant 'SELECT $_([NULL]);' diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments6.sh b/tests/queries/0_stateless/00534_functions_bad_arguments6.sh index 3e1e1d38649..53dff77a390 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments6.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments6.sh @@ -2,8 +2,10 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./00534_functions_bad_arguments.lib . "$CURDIR"/00534_functions_bad_arguments.lib test_variant 'SELECT $_(-1);' diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments7.sh b/tests/queries/0_stateless/00534_functions_bad_arguments7.sh index ac024b4cb94..3625b65c381 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments7.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments7.sh @@ -2,8 +2,10 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./00534_functions_bad_arguments.lib . "$CURDIR"/00534_functions_bad_arguments.lib test_variant "SELECT \$_('');" diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments8.sh b/tests/queries/0_stateless/00534_functions_bad_arguments8.sh index 31e1eea52a4..35cb25e6f8e 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments8.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments8.sh @@ -2,8 +2,10 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./00534_functions_bad_arguments.lib . "$CURDIR"/00534_functions_bad_arguments.lib test_variant 'SELECT $_(NULL, NULL);' diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments9.sh b/tests/queries/0_stateless/00534_functions_bad_arguments9.sh index ca4fd0bb5ba..1756e968271 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments9.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments9.sh @@ -2,8 +2,10 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./00534_functions_bad_arguments.lib . "$CURDIR"/00534_functions_bad_arguments.lib test_variant 'SELECT $_([], []);' diff --git a/tests/queries/0_stateless/00540_bad_data_types.sh b/tests/queries/0_stateless/00540_bad_data_types.sh index 73245e94a52..400d1476ac4 100755 --- a/tests/queries/0_stateless/00540_bad_data_types.sh +++ b/tests/queries/0_stateless/00540_bad_data_types.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="SELECT CAST(0 AS Array)" 2>/dev/null || true; diff --git a/tests/queries/0_stateless/00543_access_to_temporary_table_in_readonly_mode.sh b/tests/queries/0_stateless/00543_access_to_temporary_table_in_readonly_mode.sh index eeae4f6fc1c..560b97a1d1b 100755 --- a/tests/queries/0_stateless/00543_access_to_temporary_table_in_readonly_mode.sh +++ b/tests/queries/0_stateless/00543_access_to_temporary_table_in_readonly_mode.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -n --query=" diff --git a/tests/queries/0_stateless/00550_join_insert_select.sh b/tests/queries/0_stateless/00550_join_insert_select.sh index 8e60ddebf8f..bfaccb613ca 100755 --- a/tests/queries/0_stateless/00550_join_insert_select.sh +++ b/tests/queries/0_stateless/00550_join_insert_select.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -n --ignore-error --query=" diff --git a/tests/queries/0_stateless/00557_remote_port.sh b/tests/queries/0_stateless/00557_remote_port.sh index abb282ca123..4b6e715438e 100755 --- a/tests/queries/0_stateless/00557_remote_port.sh +++ b/tests/queries/0_stateless/00557_remote_port.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh if [ "$CLICKHOUSE_HOST" == "localhost" ]; then diff --git a/tests/queries/0_stateless/00564_enum_order.sh b/tests/queries/0_stateless/00564_enum_order.sh index dc47b3c3773..f50b1c39590 100755 --- a/tests/queries/0_stateless/00564_enum_order.sh +++ b/tests/queries/0_stateless/00564_enum_order.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "DROP TABLE IF EXISTS enum"; diff --git a/tests/queries/0_stateless/00565_enum_order.sh b/tests/queries/0_stateless/00565_enum_order.sh index 2851bcaaca2..6958a403246 100755 --- a/tests/queries/0_stateless/00565_enum_order.sh +++ b/tests/queries/0_stateless/00565_enum_order.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e -o pipefail diff --git a/tests/queries/0_stateless/00574_empty_strings_deserialization.sh b/tests/queries/0_stateless/00574_empty_strings_deserialization.sh index 95b3c2d9e5f..1cbc9456ab0 100755 --- a/tests/queries/0_stateless/00574_empty_strings_deserialization.sh +++ b/tests/queries/0_stateless/00574_empty_strings_deserialization.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS empty_strings_deserialization" diff --git a/tests/queries/0_stateless/00575_illegal_column_exception_when_drop_depen_column.sh b/tests/queries/0_stateless/00575_illegal_column_exception_when_drop_depen_column.sh index a526fe00e1b..94a9d331b6a 100755 --- a/tests/queries/0_stateless/00575_illegal_column_exception_when_drop_depen_column.sh +++ b/tests/queries/0_stateless/00575_illegal_column_exception_when_drop_depen_column.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/00595_insert_into_view.sh b/tests/queries/0_stateless/00595_insert_into_view.sh index ad16b6147d7..16cc843f9b7 100755 --- a/tests/queries/0_stateless/00595_insert_into_view.sh +++ b/tests/queries/0_stateless/00595_insert_into_view.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh exception_pattern="Code: 48.*Method write is not supported by storage View" diff --git a/tests/queries/0_stateless/00596_limit_on_expanded_ast.sh b/tests/queries/0_stateless/00596_limit_on_expanded_ast.sh index 0c4430f3705..ef0436e55f2 100755 --- a/tests/queries/0_stateless/00596_limit_on_expanded_ast.sh +++ b/tests/queries/0_stateless/00596_limit_on_expanded_ast.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh exception_pattern="too big" diff --git a/tests/queries/0_stateless/00598_create_as_select_http.sh b/tests/queries/0_stateless/00598_create_as_select_http.sh index 58c9593919e..34a70efacd0 100755 --- a/tests/queries/0_stateless/00598_create_as_select_http.sh +++ b/tests/queries/0_stateless/00598_create_as_select_http.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e -o pipefail diff --git a/tests/queries/0_stateless/00600_replace_running_query.sh b/tests/queries/0_stateless/00600_replace_running_query.sh index ea70866538d..be5523e06ea 100755 --- a/tests/queries/0_stateless/00600_replace_running_query.sh +++ b/tests/queries/0_stateless/00600_replace_running_query.sh @@ -3,6 +3,7 @@ CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/00601_kill_running_query.sh b/tests/queries/0_stateless/00601_kill_running_query.sh index 9a6a3a476a4..e4e3ee98877 100755 --- a/tests/queries/0_stateless/00601_kill_running_query.sh +++ b/tests/queries/0_stateless/00601_kill_running_query.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e -o pipefail diff --git a/tests/queries/0_stateless/00602_throw_if.sh b/tests/queries/0_stateless/00602_throw_if.sh index 5a133ce57a7..fe8feab0303 100755 --- a/tests/queries/0_stateless/00602_throw_if.sh +++ b/tests/queries/0_stateless/00602_throw_if.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh default_exception_message="Value passed to 'throwIf' function is non zero" diff --git a/tests/queries/0_stateless/00612_http_max_query_size.sh b/tests/queries/0_stateless/00612_http_max_query_size.sh index 78ae4eba1dc..0a37d7ec766 100755 --- a/tests/queries/0_stateless/00612_http_max_query_size.sh +++ b/tests/queries/0_stateless/00612_http_max_query_size.sh @@ -2,6 +2,7 @@ # shellcheck disable=SC2028 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo 'select 1' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}&max_query_size=8" -d @- 2>&1 | grep -o "Max query size exceeded" diff --git a/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh b/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh index 59617c81db9..99813d894ae 100755 --- a/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh +++ b/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh @@ -2,6 +2,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/00623_truncate_table_throw_exception.sh b/tests/queries/0_stateless/00623_truncate_table_throw_exception.sh index f4e88b21b75..59092ff28f2 100755 --- a/tests/queries/0_stateless/00623_truncate_table_throw_exception.sh +++ b/tests/queries/0_stateless/00623_truncate_table_throw_exception.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS test_truncate;" diff --git a/tests/queries/0_stateless/00625_query_in_form_data.sh b/tests/queries/0_stateless/00625_query_in_form_data.sh index 035f4a7ee21..38fb166d846 100755 --- a/tests/queries/0_stateless/00625_query_in_form_data.sh +++ b/tests/queries/0_stateless/00625_query_in_form_data.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} "${CLICKHOUSE_URL}&query=select" -X POST --form-string 'query= 1;' 2>/dev/null diff --git a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh index 0c56a02b894..8945fc3e56b 100755 --- a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh @@ -7,6 +7,7 @@ CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh function query_with_retry diff --git a/tests/queries/0_stateless/00630_arbitrary_csv_delimiter.sh b/tests/queries/0_stateless/00630_arbitrary_csv_delimiter.sh index cb3f70367cf..978b7bdb671 100755 --- a/tests/queries/0_stateless/00630_arbitrary_csv_delimiter.sh +++ b/tests/queries/0_stateless/00630_arbitrary_csv_delimiter.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS csv"; diff --git a/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh b/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh index a9be70be5b9..817da08bfa0 100755 --- a/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh +++ b/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS root" diff --git a/tests/queries/0_stateless/00634_logging_shard.sh b/tests/queries/0_stateless/00634_logging_shard.sh index 821ae363218..ab210e5a373 100755 --- a/tests/queries/0_stateless/00634_logging_shard.sh +++ b/tests/queries/0_stateless/00634_logging_shard.sh @@ -5,6 +5,7 @@ set -e export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL="trace" CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh cur_name=$(basename "${BASH_SOURCE[0]}") diff --git a/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh b/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh index add1f037436..c645bea23b3 100755 --- a/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh +++ b/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh @@ -5,6 +5,7 @@ set -e export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL="trace" CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh cur_name=$(basename "${BASH_SOURCE[0]}") diff --git a/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh b/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh index 8150d52abc9..fdaecd87f53 100755 --- a/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh +++ b/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query="SELECT '*** Single column partition key ***'" diff --git a/tests/queries/0_stateless/00637_sessions_in_http_interface_and_settings.sh b/tests/queries/0_stateless/00637_sessions_in_http_interface_and_settings.sh index 901f3150ce6..014fc696da3 100755 --- a/tests/queries/0_stateless/00637_sessions_in_http_interface_and_settings.sh +++ b/tests/queries/0_stateless/00637_sessions_in_http_interface_and_settings.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh user="readonly" diff --git a/tests/queries/0_stateless/00646_url_engine.sh b/tests/queries/0_stateless/00646_url_engine.sh index bf20e0c1222..357e0d74fd0 100755 --- a/tests/queries/0_stateless/00646_url_engine.sh +++ b/tests/queries/0_stateless/00646_url_engine.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # We should have correct env vars from shell_config.sh to run this test diff --git a/tests/queries/0_stateless/00650_csv_with_specified_quote_rule.sh b/tests/queries/0_stateless/00650_csv_with_specified_quote_rule.sh index 8540267a809..3971510f7f6 100755 --- a/tests/queries/0_stateless/00650_csv_with_specified_quote_rule.sh +++ b/tests/queries/0_stateless/00650_csv_with_specified_quote_rule.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS csv"; diff --git a/tests/queries/0_stateless/00651_default_database_on_client_reconnect.sh b/tests/queries/0_stateless/00651_default_database_on_client_reconnect.sh index 154ee654a81..23ce8bb9677 100755 --- a/tests/queries/0_stateless/00651_default_database_on_client_reconnect.sh +++ b/tests/queries/0_stateless/00651_default_database_on_client_reconnect.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --ignore-error --multiquery --query "DROP TABLE IF EXISTS tab_00651; CREATE TABLE tab_00651 (val UInt64) engine = Memory; SHOW CREATE TABLE tab_00651 format abcd; DESC tab_00651; DROP TABLE tab_00651;" ||: 2> /dev/null diff --git a/tests/queries/0_stateless/00652_mergetree_mutations.sh b/tests/queries/0_stateless/00652_mergetree_mutations.sh index a49a0c8f500..7c7117d5f75 100755 --- a/tests/queries/0_stateless/00652_mergetree_mutations.sh +++ b/tests/queries/0_stateless/00652_mergetree_mutations.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./mergetree_mutations.lib . "$CURDIR"/mergetree_mutations.lib ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mutations" diff --git a/tests/queries/0_stateless/00652_mutations_alter_update.sh b/tests/queries/0_stateless/00652_mutations_alter_update.sh index 83a5e18d4ae..67c024336a0 100755 --- a/tests/queries/0_stateless/00652_mutations_alter_update.sh +++ b/tests/queries/0_stateless/00652_mutations_alter_update.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS alter_update" diff --git a/tests/queries/0_stateless/00652_mutations_default_database.sh b/tests/queries/0_stateless/00652_mutations_default_database.sh index 78aa0e88c36..eed45540f9b 100755 --- a/tests/queries/0_stateless/00652_mutations_default_database.sh +++ b/tests/queries/0_stateless/00652_mutations_default_database.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --multiquery --mutations_sync=1 << EOF diff --git a/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh b/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh index e40c877e8a2..02f552c250d 100755 --- a/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh +++ b/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./mergetree_mutations.lib . "$CURDIR"/mergetree_mutations.lib ${CLICKHOUSE_CLIENT} --multiquery << EOF diff --git a/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh b/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh index 9e4bdba1294..08a39c58c3e 100755 --- a/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh +++ b/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./mergetree_mutations.lib . "$CURDIR"/mergetree_mutations.lib ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mutations_r1" diff --git a/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh b/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh index ff73669bbd0..f49aeb93184 100755 --- a/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh +++ b/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh @@ -11,6 +11,7 @@ #-------------------------------------------- CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS string_test_table;" diff --git a/tests/queries/0_stateless/00682_empty_parts_merge.sh b/tests/queries/0_stateless/00682_empty_parts_merge.sh index 1a336580b3a..0213f31ea94 100755 --- a/tests/queries/0_stateless/00682_empty_parts_merge.sh +++ b/tests/queries/0_stateless/00682_empty_parts_merge.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS ordinary_00682" diff --git a/tests/queries/0_stateless/00686_client_exit_code.sh b/tests/queries/0_stateless/00686_client_exit_code.sh index dea82af2211..eab534dd8d5 100755 --- a/tests/queries/0_stateless/00686_client_exit_code.sh +++ b/tests/queries/0_stateless/00686_client_exit_code.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./mergetree_mutations.lib . "$CURDIR"/mergetree_mutations.lib echo "INSERT INTO test FORMAT CSV" | ${CLICKHOUSE_CLIENT} -n 2>/dev/null diff --git a/tests/queries/0_stateless/00687_top_and_offset.sh b/tests/queries/0_stateless/00687_top_and_offset.sh index fb7dc2b4d90..4355f746ac8 100755 --- a/tests/queries/0_stateless/00687_top_and_offset.sh +++ b/tests/queries/0_stateless/00687_top_and_offset.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/00690_insert_select_converting_exception_message.sh b/tests/queries/0_stateless/00690_insert_select_converting_exception_message.sh index 290bde5b4ac..3f22726877a 100755 --- a/tests/queries/0_stateless/00690_insert_select_converting_exception_message.sh +++ b/tests/queries/0_stateless/00690_insert_select_converting_exception_message.sh @@ -3,6 +3,7 @@ CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_00690;" diff --git a/tests/queries/0_stateless/00699_materialized_view_mutations.sh b/tests/queries/0_stateless/00699_materialized_view_mutations.sh index a8166ca29c0..f55b8ac10ed 100755 --- a/tests/queries/0_stateless/00699_materialized_view_mutations.sh +++ b/tests/queries/0_stateless/00699_materialized_view_mutations.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh b/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh index e406b603717..bdb4627ae30 100755 --- a/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh +++ b/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh @@ -4,6 +4,7 @@ set -e CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --multiquery --query=" diff --git a/tests/queries/0_stateless/00705_drop_create_merge_tree.sh b/tests/queries/0_stateless/00705_drop_create_merge_tree.sh index c817767c0b5..ede490cf8f1 100755 --- a/tests/queries/0_stateless/00705_drop_create_merge_tree.sh +++ b/tests/queries/0_stateless/00705_drop_create_merge_tree.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh function stress() diff --git a/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh b/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh index 8a8a9ed6615..54b6c80f2ac 100755 --- a/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh +++ b/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./mergetree_mutations.lib . "$CURDIR"/mergetree_mutations.lib diff --git a/tests/queries/0_stateless/00715_json_each_row_input_nested.sh b/tests/queries/0_stateless/00715_json_each_row_input_nested.sh index c38f5b2ba28..72e01aef742 100755 --- a/tests/queries/0_stateless/00715_json_each_row_input_nested.sh +++ b/tests/queries/0_stateless/00715_json_each_row_input_nested.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS json_each_row_nested" diff --git a/tests/queries/0_stateless/00719_insert_block_without_column.sh b/tests/queries/0_stateless/00719_insert_block_without_column.sh index 384445b1ae6..11aaab98cdc 100755 --- a/tests/queries/0_stateless/00719_insert_block_without_column.sh +++ b/tests/queries/0_stateless/00719_insert_block_without_column.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh [ -e "${CLICKHOUSE_TMP}"/test_squashing_block_without_column.out ] && rm "${CLICKHOUSE_TMP}"/test_squashing_block_without_column.out diff --git a/tests/queries/0_stateless/00719_parallel_ddl_db.sh b/tests/queries/0_stateless/00719_parallel_ddl_db.sh index c9f86fa0a41..5608a57eecc 100755 --- a/tests/queries/0_stateless/00719_parallel_ddl_db.sh +++ b/tests/queries/0_stateless/00719_parallel_ddl_db.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS parallel_ddl" diff --git a/tests/queries/0_stateless/00719_parallel_ddl_table.sh b/tests/queries/0_stateless/00719_parallel_ddl_table.sh index c06b6f6cb12..2a542ea21f6 100755 --- a/tests/queries/0_stateless/00719_parallel_ddl_table.sh +++ b/tests/queries/0_stateless/00719_parallel_ddl_table.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS parallel_ddl" diff --git a/tests/queries/0_stateless/00728_json_each_row_parsing.sh b/tests/queries/0_stateless/00728_json_each_row_parsing.sh index 73ea72a5e2c..6a43fc2d8da 100755 --- a/tests/queries/0_stateless/00728_json_each_row_parsing.sh +++ b/tests/queries/0_stateless/00728_json_each_row_parsing.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS json_parse;" diff --git a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh index d4cbc753aee..25a742a481a 100755 --- a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh +++ b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh settings="--log_queries=1 --log_query_threads=1 --log_profile_events=1 --log_query_settings=1" diff --git a/tests/queries/0_stateless/00738_lock_for_inner_table.sh b/tests/queries/0_stateless/00738_lock_for_inner_table.sh index 4570c853f31..9540d566ac3 100755 --- a/tests/queries/0_stateless/00738_lock_for_inner_table.sh +++ b/tests/queries/0_stateless/00738_lock_for_inner_table.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo "DROP TABLE IF EXISTS tab_00738; diff --git a/tests/queries/0_stateless/00746_sql_fuzzy.sh b/tests/queries/0_stateless/00746_sql_fuzzy.sh index aed00c905d7..9fa64d10057 100755 --- a/tests/queries/0_stateless/00746_sql_fuzzy.sh +++ b/tests/queries/0_stateless/00746_sql_fuzzy.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh export SQL_FUZZY_FILE_FUNCTIONS=${CLICKHOUSE_TMP}/clickhouse-functions diff --git a/tests/queries/0_stateless/00754_distributed_optimize_skip_select_on_unused_shards.sh b/tests/queries/0_stateless/00754_distributed_optimize_skip_select_on_unused_shards.sh index d3aa04c9095..55b74e055db 100755 --- a/tests/queries/0_stateless/00754_distributed_optimize_skip_select_on_unused_shards.sh +++ b/tests/queries/0_stateless/00754_distributed_optimize_skip_select_on_unused_shards.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS mergetree_00754;" diff --git a/tests/queries/0_stateless/00754_distributed_optimize_skip_select_on_unused_shards_with_prewhere.sh b/tests/queries/0_stateless/00754_distributed_optimize_skip_select_on_unused_shards_with_prewhere.sh index 5cdc150dace..ecf75352ca9 100755 --- a/tests/queries/0_stateless/00754_distributed_optimize_skip_select_on_unused_shards_with_prewhere.sh +++ b/tests/queries/0_stateless/00754_distributed_optimize_skip_select_on_unused_shards_with_prewhere.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS distributed_00754;" diff --git a/tests/queries/0_stateless/00763_lock_buffer.sh b/tests/queries/0_stateless/00763_lock_buffer.sh index 7e1a1e87917..44660035208 100755 --- a/tests/queries/0_stateless/00763_lock_buffer.sh +++ b/tests/queries/0_stateless/00763_lock_buffer.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mt_00763_2" diff --git a/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh b/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh index 1605689fd11..65dd2474580 100755 --- a/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh +++ b/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh @@ -4,6 +4,7 @@ set -e CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mt_00763_1" diff --git a/tests/queries/0_stateless/00764_max_query_size_allocation.sh b/tests/queries/0_stateless/00764_max_query_size_allocation.sh index f074077b6ea..e42a5bd9fb9 100755 --- a/tests/queries/0_stateless/00764_max_query_size_allocation.sh +++ b/tests/queries/0_stateless/00764_max_query_size_allocation.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_query_size=1000000000&max_memory_usage=10000000" -d "SELECT 1" diff --git a/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh b/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh index 7f4df58d764..63b687d072d 100755 --- a/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh +++ b/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo "DROP TABLE IF EXISTS concurrent_alter_column" | ${CLICKHOUSE_CLIENT} diff --git a/tests/queries/0_stateless/00823_capnproto_input.sh b/tests/queries/0_stateless/00823_capnproto_input.sh index a53d54b11d8..b86c8882bbd 100755 --- a/tests/queries/0_stateless/00823_capnproto_input.sh +++ b/tests/queries/0_stateless/00823_capnproto_input.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh #create the schema file diff --git a/tests/queries/0_stateless/00825_http_header_query_id.sh b/tests/queries/0_stateless/00825_http_header_query_id.sh index 2b0f199baf2..aad7c038bce 100755 --- a/tests/queries/0_stateless/00825_http_header_query_id.sh +++ b/tests/queries/0_stateless/00825_http_header_query_id.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL_COMMAND} -q -I -sSg "${CLICKHOUSE_URL}&query=SELECT%201" | grep -o X-ClickHouse-Query-Id diff --git a/tests/queries/0_stateless/00825_protobuf_format_input.sh b/tests/queries/0_stateless/00825_protobuf_format_input.sh index b9912b2b849..5a85a852cb1 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_input.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_input.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -eo pipefail @@ -43,6 +44,7 @@ EOF # To generate the file 00825_protobuf_format_input.insh use the following commands: # ninja ProtobufDelimitedMessagesSerializer # build/utils/test-data-generator/ProtobufDelimitedMessagesSerializer +# shellcheck source=./00825_protobuf_format_input.insh source "$CURDIR"/00825_protobuf_format_input.insh $CLICKHOUSE_CLIENT --query "SELECT * FROM in_persons_00825 ORDER BY uuid;" @@ -51,6 +53,7 @@ $CLICKHOUSE_CLIENT --query "SELECT * FROM in_squares_00825 ORDER BY number;" $CLICKHOUSE_CLIENT --query "TRUNCATE TABLE in_persons_00825;" $CLICKHOUSE_CLIENT --query "TRUNCATE TABLE in_squares_00825;" +# shellcheck source=./00825_protobuf_format_input_single.insh source "$CURDIR"/00825_protobuf_format_input_single.insh $CLICKHOUSE_CLIENT --query "SELECT * FROM in_persons_00825 ORDER BY uuid;" diff --git a/tests/queries/0_stateless/00825_protobuf_format_output.sh b/tests/queries/0_stateless/00825_protobuf_format_output.sh index 889d3a9d2ae..f2d0c60b393 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_output.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_output.sh @@ -5,6 +5,7 @@ # build/utils/test-data-generator/ProtobufDelimitedMessagesSerializer CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e -o pipefail diff --git a/tests/queries/0_stateless/00834_cancel_http_readonly_queries_on_client_close.sh b/tests/queries/0_stateless/00834_cancel_http_readonly_queries_on_client_close.sh index 0088effcf34..3da40f16786 100755 --- a/tests/queries/0_stateless/00834_cancel_http_readonly_queries_on_client_close.sh +++ b/tests/queries/0_stateless/00834_cancel_http_readonly_queries_on_client_close.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} --max-time 1 -sS "${CLICKHOUSE_URL}&query_id=cancel_http_readonly_queries_on_client_close&cancel_http_readonly_queries_on_client_close=1&query=SELECT+count()+FROM+system.numbers" 2>&1 | grep -cF 'curl: (28)' diff --git a/tests/queries/0_stateless/00834_dont_allow_to_set_two_configuration_files_in_client.sh b/tests/queries/0_stateless/00834_dont_allow_to_set_two_configuration_files_in_client.sh index 421d898eb83..be57757af06 100755 --- a/tests/queries/0_stateless/00834_dont_allow_to_set_two_configuration_files_in_client.sh +++ b/tests/queries/0_stateless/00834_dont_allow_to_set_two_configuration_files_in_client.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh OUTPUT=$($CLICKHOUSE_CLIENT_BINARY -c 1 -C 2 2>&1) diff --git a/tests/queries/0_stateless/00834_hints_for_type_function_typos.sh b/tests/queries/0_stateless/00834_hints_for_type_function_typos.sh index e00590692ba..6640e0003cd 100755 --- a/tests/queries/0_stateless/00834_hints_for_type_function_typos.sh +++ b/tests/queries/0_stateless/00834_hints_for_type_function_typos.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "select c23ount(*) from system.functions;" 2>&1 | grep "Maybe you meant: \['count'" &>/dev/null; diff --git a/tests/queries/0_stateless/00834_kill_mutation.sh b/tests/queries/0_stateless/00834_kill_mutation.sh index 886433e7ba8..a17d85cf9a5 100755 --- a/tests/queries/0_stateless/00834_kill_mutation.sh +++ b/tests/queries/0_stateless/00834_kill_mutation.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./mergetree_mutations.lib . "$CURDIR"/mergetree_mutations.lib ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS kill_mutation" diff --git a/tests/queries/0_stateless/00834_kill_mutation_replicated_zookeeper.sh b/tests/queries/0_stateless/00834_kill_mutation_replicated_zookeeper.sh index 8414db1fee5..d1f938f73fe 100755 --- a/tests/queries/0_stateless/00834_kill_mutation_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/00834_kill_mutation_replicated_zookeeper.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./mergetree_mutations.lib . "$CURDIR"/mergetree_mutations.lib ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS kill_mutation_r1" diff --git a/tests/queries/0_stateless/00837_minmax_index.sh b/tests/queries/0_stateless/00837_minmax_index.sh index 9f947db6b6f..39e39f9d628 100755 --- a/tests/queries/0_stateless/00837_minmax_index.sh +++ b/tests/queries/0_stateless/00837_minmax_index.sh @@ -2,6 +2,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;" diff --git a/tests/queries/0_stateless/00838_system_tables_drop_table_race.sh b/tests/queries/0_stateless/00838_system_tables_drop_table_race.sh index c64e1b41ca4..2e070d685a4 100755 --- a/tests/queries/0_stateless/00838_system_tables_drop_table_race.sh +++ b/tests/queries/0_stateless/00838_system_tables_drop_table_race.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS table" diff --git a/tests/queries/0_stateless/00838_unique_index.sh b/tests/queries/0_stateless/00838_unique_index.sh index 330eb4ed346..36504b754a7 100755 --- a/tests/queries/0_stateless/00838_unique_index.sh +++ b/tests/queries/0_stateless/00838_unique_index.sh @@ -2,6 +2,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS set_idx;" diff --git a/tests/queries/0_stateless/00840_long_concurrent_select_and_drop_deadlock.sh b/tests/queries/0_stateless/00840_long_concurrent_select_and_drop_deadlock.sh index 481bcc043d7..60a2d8eb9a0 100755 --- a/tests/queries/0_stateless/00840_long_concurrent_select_and_drop_deadlock.sh +++ b/tests/queries/0_stateless/00840_long_concurrent_select_and_drop_deadlock.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh for _ in {1..200}; do echo "drop table if exists view_00840" | $CLICKHOUSE_CLIENT; echo "create view view_00840 as select count(*),database,table from system.columns group by database,table" | $CLICKHOUSE_CLIENT; done & diff --git a/tests/queries/0_stateless/00851_http_insert_json_defaults.sh b/tests/queries/0_stateless/00851_http_insert_json_defaults.sh index 08218cec5b9..08458822cc0 100755 --- a/tests/queries/0_stateless/00851_http_insert_json_defaults.sh +++ b/tests/queries/0_stateless/00851_http_insert_json_defaults.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS defaults" diff --git a/tests/queries/0_stateless/00898_parsing_bad_diagnostic_message.sh b/tests/queries/0_stateless/00898_parsing_bad_diagnostic_message.sh index c75b752576d..0eeabde917c 100755 --- a/tests/queries/0_stateless/00898_parsing_bad_diagnostic_message.sh +++ b/tests/queries/0_stateless/00898_parsing_bad_diagnostic_message.sh @@ -3,6 +3,7 @@ # set -x CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh echo -ne '0\t1\t2\t3\t4\t5\t6\t7\t8\t9\t10\ta' | $CLICKHOUSE_LOCAL --structure 'c0 UInt8, c1 UInt8, c2 UInt8, c3 UInt8, c4 UInt8, c5 UInt8, c6 UInt8, c7 UInt8, c8 UInt8, c9 UInt8, c10 UInt8, c11 UInt8' --input-format TSV --query 'SELECT * FROM table' 2>&1 | grep -F 'Column 11' diff --git a/tests/queries/0_stateless/00900_orc_load.sh b/tests/queries/0_stateless/00900_orc_load.sh index 6e08b415397..a0bacff43e5 100755 --- a/tests/queries/0_stateless/00900_orc_load.sh +++ b/tests/queries/0_stateless/00900_orc_load.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh DATA_FILE=$CUR_DIR/data_orc/test.orc diff --git a/tests/queries/0_stateless/00900_parquet.sh b/tests/queries/0_stateless/00900_parquet.sh index 1af893c0b45..4b06001429f 100755 --- a/tests/queries/0_stateless/00900_parquet.sh +++ b/tests/queries/0_stateless/00900_parquet.sh @@ -3,6 +3,7 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh #${CLICKHOUSE_CLIENT} --max_block_size=1 --query="SELECT * FROM system.numbers LIMIT 10 FORMAT Parquet" > ${CLICKHOUSE_TMP}/t1.pq diff --git a/tests/queries/0_stateless/00900_parquet_decimal.sh b/tests/queries/0_stateless/00900_parquet_decimal.sh index e6174a1f3a9..67561f484bd 100755 --- a/tests/queries/0_stateless/00900_parquet_decimal.sh +++ b/tests/queries/0_stateless/00900_parquet_decimal.sh @@ -3,6 +3,7 @@ # set -x CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS decimal;" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS decimal2;" diff --git a/tests/queries/0_stateless/00900_parquet_load.sh b/tests/queries/0_stateless/00900_parquet_load.sh index 43b738aab83..52213f066e1 100755 --- a/tests/queries/0_stateless/00900_parquet_load.sh +++ b/tests/queries/0_stateless/00900_parquet_load.sh @@ -14,6 +14,7 @@ # set -x CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh CB_DIR=$(dirname "$CLICKHOUSE_CLIENT_BINARY") diff --git a/tests/queries/0_stateless/00907_set_index_max_rows.sh b/tests/queries/0_stateless/00907_set_index_max_rows.sh index 8109223db97..f780517934d 100755 --- a/tests/queries/0_stateless/00907_set_index_max_rows.sh +++ b/tests/queries/0_stateless/00907_set_index_max_rows.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS set_idx;" diff --git a/tests/queries/0_stateless/00908_bloom_filter_index.sh b/tests/queries/0_stateless/00908_bloom_filter_index.sh index d989ed0fa6b..a7631db759c 100755 --- a/tests/queries/0_stateless/00908_bloom_filter_index.sh +++ b/tests/queries/0_stateless/00908_bloom_filter_index.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS bloom_filter_idx;" diff --git a/tests/queries/0_stateless/00908_long_http_insert.sh b/tests/queries/0_stateless/00908_long_http_insert.sh index 06a28bc0820..5e793410412 100755 --- a/tests/queries/0_stateless/00908_long_http_insert.sh +++ b/tests/queries/0_stateless/00908_long_http_insert.sh @@ -2,6 +2,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo 'DROP TABLE IF EXISTS table_for_insert' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}" -d @- diff --git a/tests/queries/0_stateless/00909_kill_not_initialized_query.sh b/tests/queries/0_stateless/00909_kill_not_initialized_query.sh index bc2a42bd708..531652a33e7 100755 --- a/tests/queries/0_stateless/00909_kill_not_initialized_query.sh +++ b/tests/queries/0_stateless/00909_kill_not_initialized_query.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/00910_client_window_size_detection.sh b/tests/queries/0_stateless/00910_client_window_size_detection.sh index eb0b14b0006..47d913de152 100755 --- a/tests/queries/0_stateless/00910_client_window_size_detection.sh +++ b/tests/queries/0_stateless/00910_client_window_size_detection.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.sh b/tests/queries/0_stateless/00921_datetime64_compatibility.sh index 3e5de1a552c..564c456bc7f 100755 --- a/tests/queries/0_stateless/00921_datetime64_compatibility.sh +++ b/tests/queries/0_stateless/00921_datetime64_compatibility.sh @@ -3,6 +3,7 @@ CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL="none" # We should have correct env vars from shell_config.sh to run this test CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # In order to check queries individually (don't stop on the first one that fails): diff --git a/tests/queries/0_stateless/00927_asof_join_other_types.sh b/tests/queries/0_stateless/00927_asof_join_other_types.sh index 13a8e798f38..c002d092b40 100755 --- a/tests/queries/0_stateless/00927_asof_join_other_types.sh +++ b/tests/queries/0_stateless/00927_asof_join_other_types.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh for typename in "UInt32" "UInt64" "Float64" "Float32" "DateTime" "Decimal32(5)" "Decimal64(5)" "Decimal128(5)" "DateTime64(3)" diff --git a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh index c5f5c61b41d..2b9a69d19d4 100755 --- a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh +++ b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/00937_template_output_format.sh b/tests/queries/0_stateless/00937_template_output_format.sh index 230bc0c0205..6030887cfba 100755 --- a/tests/queries/0_stateless/00937_template_output_format.sh +++ b/tests/queries/0_stateless/00937_template_output_format.sh @@ -2,6 +2,7 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS template"; diff --git a/tests/queries/0_stateless/00937_test_use_header_csv.sh b/tests/queries/0_stateless/00937_test_use_header_csv.sh index c7c0d1f99c0..8ab37b30b60 100755 --- a/tests/queries/0_stateless/00937_test_use_header_csv.sh +++ b/tests/queries/0_stateless/00937_test_use_header_csv.sh @@ -2,6 +2,7 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS csv" diff --git a/tests/queries/0_stateless/00937_test_use_header_tsv.sh b/tests/queries/0_stateless/00937_test_use_header_tsv.sh index a272e70d32b..aa06f2be3b8 100755 --- a/tests/queries/0_stateless/00937_test_use_header_tsv.sh +++ b/tests/queries/0_stateless/00937_test_use_header_tsv.sh @@ -2,6 +2,7 @@ # shellcheck disable=SC2016 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS tsv" diff --git a/tests/queries/0_stateless/00938_fix_rwlock_segfault.sh b/tests/queries/0_stateless/00938_fix_rwlock_segfault.sh index 9df9cce29ce..5c4253e682b 100755 --- a/tests/queries/0_stateless/00938_fix_rwlock_segfault.sh +++ b/tests/queries/0_stateless/00938_fix_rwlock_segfault.sh @@ -3,6 +3,7 @@ # Test fix for issue #5066 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/00938_template_input_format.sh b/tests/queries/0_stateless/00938_template_input_format.sh index fde74ad5512..75616b35af0 100755 --- a/tests/queries/0_stateless/00938_template_input_format.sh +++ b/tests/queries/0_stateless/00938_template_input_format.sh @@ -2,6 +2,7 @@ # shellcheck disable=SC2016,SC2028 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS template1"; diff --git a/tests/queries/0_stateless/00941_system_columns_race_condition.sh b/tests/queries/0_stateless/00941_system_columns_race_condition.sh index 952c1f78c98..0a3fc7f3b3f 100755 --- a/tests/queries/0_stateless/00941_system_columns_race_condition.sh +++ b/tests/queries/0_stateless/00941_system_columns_race_condition.sh @@ -3,6 +3,7 @@ # Test fix for issue #5066 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/00942_dataparts_500.sh b/tests/queries/0_stateless/00942_dataparts_500.sh index 9d6dc06cb75..19cb1138aa8 100755 --- a/tests/queries/0_stateless/00942_dataparts_500.sh +++ b/tests/queries/0_stateless/00942_dataparts_500.sh @@ -3,6 +3,7 @@ # Test fix for issue #5066 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} --include -sS "${CLICKHOUSE_URL_INTERSERVER}?endpoint=DataPartsExchange%3A%2Fclickhouse%2Ftables%2F01-01%2Fvisits%2Freplicas%2Fsome.server.com&part=0&compress=false" 2>&1 | grep -F 'HTTP/1.1 500 Internal Server Error' diff --git a/tests/queries/0_stateless/00942_mutate_index.sh b/tests/queries/0_stateless/00942_mutate_index.sh index df02361af78..4eebdd1147f 100755 --- a/tests/queries/0_stateless/00942_mutate_index.sh +++ b/tests/queries/0_stateless/00942_mutate_index.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;" diff --git a/tests/queries/0_stateless/00943_materialize_index.sh b/tests/queries/0_stateless/00943_materialize_index.sh index 92947a76b97..43c9af84672 100755 --- a/tests/queries/0_stateless/00943_materialize_index.sh +++ b/tests/queries/0_stateless/00943_materialize_index.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;" diff --git a/tests/queries/0_stateless/00944_clear_index_in_partition.sh b/tests/queries/0_stateless/00944_clear_index_in_partition.sh index 8687e2044f0..1f349cf5946 100755 --- a/tests/queries/0_stateless/00944_clear_index_in_partition.sh +++ b/tests/queries/0_stateless/00944_clear_index_in_partition.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;" diff --git a/tests/queries/0_stateless/00944_create_bloom_filter_index_with_merge_tree.sh b/tests/queries/0_stateless/00944_create_bloom_filter_index_with_merge_tree.sh index 9cdc3da4bac..bb4a04eb15e 100755 --- a/tests/queries/0_stateless/00944_create_bloom_filter_index_with_merge_tree.sh +++ b/tests/queries/0_stateless/00944_create_bloom_filter_index_with_merge_tree.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/00948_format_in_with_single_element.sh b/tests/queries/0_stateless/00948_format_in_with_single_element.sh index 03a77ed7f49..0edfb3c2d8b 100755 --- a/tests/queries/0_stateless/00948_format_in_with_single_element.sh +++ b/tests/queries/0_stateless/00948_format_in_with_single_element.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/00952_basic_constraints.sh b/tests/queries/0_stateless/00952_basic_constraints.sh index d8d44a9e77d..0a7ba91909c 100755 --- a/tests/queries/0_stateless/00952_basic_constraints.sh +++ b/tests/queries/0_stateless/00952_basic_constraints.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh EXCEPTION_TEXT=violated diff --git a/tests/queries/0_stateless/00952_input_function.sh b/tests/queries/0_stateless/00952_input_function.sh index 4915a469997..54496ba09e0 100755 --- a/tests/queries/0_stateless/00952_input_function.sh +++ b/tests/queries/0_stateless/00952_input_function.sh @@ -3,6 +3,7 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS input_function_table_1" diff --git a/tests/queries/0_stateless/00953_constraints_operations.sh b/tests/queries/0_stateless/00953_constraints_operations.sh index a7880055244..b4335e1fd41 100755 --- a/tests/queries/0_stateless/00953_constraints_operations.sh +++ b/tests/queries/0_stateless/00953_constraints_operations.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh EXCEPTION_TEXT=violated diff --git a/tests/queries/0_stateless/00953_indices_alter_exceptions.sh b/tests/queries/0_stateless/00953_indices_alter_exceptions.sh index 1fee87f5cde..8d4097c0722 100755 --- a/tests/queries/0_stateless/00953_indices_alter_exceptions.sh +++ b/tests/queries/0_stateless/00953_indices_alter_exceptions.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh EXCEPTION_SUCCESS_TEXT=ok diff --git a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh index 3caba663df6..bbc2d957937 100755 --- a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh +++ b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/00954_client_prepared_statements.sh b/tests/queries/0_stateless/00954_client_prepared_statements.sh index 1bff57c574e..460b0d44943 100755 --- a/tests/queries/0_stateless/00954_client_prepared_statements.sh +++ b/tests/queries/0_stateless/00954_client_prepared_statements.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS ps"; diff --git a/tests/queries/0_stateless/00955_complex_prepared_statements.sh b/tests/queries/0_stateless/00955_complex_prepared_statements.sh index affb73a1ec3..2096272df91 100755 --- a/tests/queries/0_stateless/00955_complex_prepared_statements.sh +++ b/tests/queries/0_stateless/00955_complex_prepared_statements.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh EXCEPTION_TEXT="Code: 457." diff --git a/tests/queries/0_stateless/00955_test_final_mark_use.sh b/tests/queries/0_stateless/00955_test_final_mark_use.sh index d14c9e49814..3c7db3249b3 100755 --- a/tests/queries/0_stateless/00955_test_final_mark_use.sh +++ b/tests/queries/0_stateless/00955_test_final_mark_use.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query=" diff --git a/tests/queries/0_stateless/00956_http_prepared_statements.sh b/tests/queries/0_stateless/00956_http_prepared_statements.sh index 091ad9a2ebc..80c7aa69ae3 100755 --- a/tests/queries/0_stateless/00956_http_prepared_statements.sh +++ b/tests/queries/0_stateless/00956_http_prepared_statements.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -sS "$CLICKHOUSE_URL" -d "DROP TABLE IF EXISTS ps"; diff --git a/tests/queries/0_stateless/00956_sensitive_data_masking.sh b/tests/queries/0_stateless/00956_sensitive_data_masking.sh index c2118be444a..799941e94bf 100755 --- a/tests/queries/0_stateless/00956_sensitive_data_masking.sh +++ b/tests/queries/0_stateless/00956_sensitive_data_masking.sh @@ -4,6 +4,7 @@ export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL="trace" CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh cur_name=$(basename "${BASH_SOURCE[0]}") diff --git a/tests/queries/0_stateless/00957_format_with_clashed_aliases.sh b/tests/queries/0_stateless/00957_format_with_clashed_aliases.sh index f877289da62..dc6908659a0 100755 --- a/tests/queries/0_stateless/00957_format_with_clashed_aliases.sh +++ b/tests/queries/0_stateless/00957_format_with_clashed_aliases.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/00958_format_of_tuple_array_element.sh b/tests/queries/0_stateless/00958_format_of_tuple_array_element.sh index bcc54d1e2d2..decd44da8a3 100755 --- a/tests/queries/0_stateless/00958_format_of_tuple_array_element.sh +++ b/tests/queries/0_stateless/00958_format_of_tuple_array_element.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/00959_format_with_different_aliases.sh b/tests/queries/0_stateless/00959_format_with_different_aliases.sh index f1aa43ba917..53b994c27e8 100755 --- a/tests/queries/0_stateless/00959_format_with_different_aliases.sh +++ b/tests/queries/0_stateless/00959_format_with_different_aliases.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/00964_bloom_index_string_functions.sh b/tests/queries/0_stateless/00964_bloom_index_string_functions.sh index 80117e2bdec..7697578ea66 100755 --- a/tests/queries/0_stateless/00964_bloom_index_string_functions.sh +++ b/tests/queries/0_stateless/00964_bloom_index_string_functions.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS bloom_filter_idx;" diff --git a/tests/queries/0_stateless/00965_logs_level_bugfix.sh b/tests/queries/0_stateless/00965_logs_level_bugfix.sh index 0c618cf1bf0..5666e69656d 100755 --- a/tests/queries/0_stateless/00965_logs_level_bugfix.sh +++ b/tests/queries/0_stateless/00965_logs_level_bugfix.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT_BINARY} --send_logs_level="trace" --query="SELECT 1" 2>&1 | awk '{ print $8 }' | grep "Trace" | head -n 1 diff --git a/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.sh b/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.sh index 34dd1e5c083..c1590047ffa 100755 --- a/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.sh +++ b/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh for _ in {1..10}; do diff --git a/tests/queries/0_stateless/00965_set_index_string_functions.sh b/tests/queries/0_stateless/00965_set_index_string_functions.sh index 811d484d5ba..dba33d9abcf 100755 --- a/tests/queries/0_stateless/00965_set_index_string_functions.sh +++ b/tests/queries/0_stateless/00965_set_index_string_functions.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS set_idx;" diff --git a/tests/queries/0_stateless/00971_query_id_in_logs.sh b/tests/queries/0_stateless/00971_query_id_in_logs.sh index 2f9101a91b6..9e927f36a9c 100755 --- a/tests/queries/0_stateless/00971_query_id_in_logs.sh +++ b/tests/queries/0_stateless/00971_query_id_in_logs.sh @@ -3,6 +3,7 @@ CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=trace CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh b/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh index 2c06168e1af..a817acd88a6 100755 --- a/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh +++ b/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS lowString;" diff --git a/tests/queries/0_stateless/00974_text_log_table_not_empty.sh b/tests/queries/0_stateless/00974_text_log_table_not_empty.sh index 73e4de4a926..ab1b32ad90e 100755 --- a/tests/queries/0_stateless/00974_text_log_table_not_empty.sh +++ b/tests/queries/0_stateless/00974_text_log_table_not_empty.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query="SELECT 6103" diff --git a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh index 6e476042731..81c0c563db1 100755 --- a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./mergetree_mutations.lib . "$CURDIR"/mergetree_mutations.lib $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS indices_mutaions1;" diff --git a/tests/queries/0_stateless/00980_alter_settings_race.sh b/tests/queries/0_stateless/00980_alter_settings_race.sh index 0876d846fc9..004504b1227 100755 --- a/tests/queries/0_stateless/00980_alter_settings_race.sh +++ b/tests/queries/0_stateless/00980_alter_settings_race.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS table_for_concurrent_alter" diff --git a/tests/queries/0_stateless/00981_in_subquery_with_tuple.sh b/tests/queries/0_stateless/00981_in_subquery_with_tuple.sh index 803123d1045..99173062595 100755 --- a/tests/queries/0_stateless/00981_in_subquery_with_tuple.sh +++ b/tests/queries/0_stateless/00981_in_subquery_with_tuple.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS bug"; diff --git a/tests/queries/0_stateless/00984_parser_stack_overflow.sh b/tests/queries/0_stateless/00984_parser_stack_overflow.sh index cf67ab7e2f1..167678db5ec 100755 --- a/tests/queries/0_stateless/00984_parser_stack_overflow.sh +++ b/tests/queries/0_stateless/00984_parser_stack_overflow.sh @@ -3,6 +3,7 @@ CLICKHOUSE_CURL_TIMEOUT=30 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # Too deep recursion diff --git a/tests/queries/0_stateless/00990_hasToken.sh b/tests/queries/0_stateless/00990_hasToken.sh index 4b42a570e99..6a1d4ff5ccf 100755 --- a/tests/queries/0_stateless/00990_hasToken.sh +++ b/tests/queries/0_stateless/00990_hasToken.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # We should have correct env vars from shell_config.sh to run this test diff --git a/tests/queries/0_stateless/00991_system_parts_race_condition.sh b/tests/queries/0_stateless/00991_system_parts_race_condition.sh index 0689641b0ae..55ff4d97149 100755 --- a/tests/queries/0_stateless/00991_system_parts_race_condition.sh +++ b/tests/queries/0_stateless/00991_system_parts_race_condition.sh @@ -5,6 +5,7 @@ # https://github.com/google/sanitizers/issues/950 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh index c3c3a21f96f..613e032f42a 100755 --- a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh +++ b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh index 0c13ad2d573..1731148f71f 100755 --- a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh +++ b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/00995_exception_while_insert.sh b/tests/queries/0_stateless/00995_exception_while_insert.sh index 0c318c727ea..28351078df3 100755 --- a/tests/queries/0_stateless/00995_exception_while_insert.sh +++ b/tests/queries/0_stateless/00995_exception_while_insert.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=none/g') diff --git a/tests/queries/0_stateless/01000_unneeded_substitutions_client.sh b/tests/queries/0_stateless/01000_unneeded_substitutions_client.sh index c4c44e0e963..f20a0d8d3bd 100755 --- a/tests/queries/0_stateless/01000_unneeded_substitutions_client.sh +++ b/tests/queries/0_stateless/01000_unneeded_substitutions_client.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "SELECT '\${}'" diff --git a/tests/queries/0_stateless/01001_rename_merge_race_condition.sh b/tests/queries/0_stateless/01001_rename_merge_race_condition.sh index 8b2cb026187..5aeea34e7c1 100755 --- a/tests/queries/0_stateless/01001_rename_merge_race_condition.sh +++ b/tests/queries/0_stateless/01001_rename_merge_race_condition.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.sh b/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.sh index f09d74adb3d..d2766a14c24 100755 --- a/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.sh +++ b/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01003_kill_query_race_condition.sh b/tests/queries/0_stateless/01003_kill_query_race_condition.sh index ec7b87c112b..2d21aabf91a 100755 --- a/tests/queries/0_stateless/01003_kill_query_race_condition.sh +++ b/tests/queries/0_stateless/01003_kill_query_race_condition.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01004_rename_deadlock.sh b/tests/queries/0_stateless/01004_rename_deadlock.sh index ebb6cc42792..aa9e6f8a5bc 100755 --- a/tests/queries/0_stateless/01004_rename_deadlock.sh +++ b/tests/queries/0_stateless/01004_rename_deadlock.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh b/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh index 4a6a6e7eb1c..5c2ca22715d 100755 --- a/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh +++ b/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01006_simpod_empty_part_single_column_write.sh b/tests/queries/0_stateless/01006_simpod_empty_part_single_column_write.sh index 5473185af3d..fbd287478b6 100755 --- a/tests/queries/0_stateless/01006_simpod_empty_part_single_column_write.sh +++ b/tests/queries/0_stateless/01006_simpod_empty_part_single_column_write.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./mergetree_mutations.lib . "$CURDIR"/mergetree_mutations.lib diff --git a/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh b/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh index ae3c9e62fdd..8773a180822 100755 --- a/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh +++ b/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01010_low_cardinality_and_native_http.sh b/tests/queries/0_stateless/01010_low_cardinality_and_native_http.sh index 3a933e1fb21..1bf34445dbd 100755 --- a/tests/queries/0_stateless/01010_low_cardinality_and_native_http.sh +++ b/tests/queries/0_stateless/01010_low_cardinality_and_native_http.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh b/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh index 078b49da940..724caa7f414 100755 --- a/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh +++ b/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01014_format_custom_separated.sh b/tests/queries/0_stateless/01014_format_custom_separated.sh index 4f98d55a078..42599bcc944 100755 --- a/tests/queries/0_stateless/01014_format_custom_separated.sh +++ b/tests/queries/0_stateless/01014_format_custom_separated.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS custom_separated" diff --git a/tests/queries/0_stateless/01014_lazy_database_basic.sh b/tests/queries/0_stateless/01014_lazy_database_basic.sh index 2712e3d05bb..11d698e764e 100755 --- a/tests/queries/0_stateless/01014_lazy_database_basic.sh +++ b/tests/queries/0_stateless/01014_lazy_database_basic.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} -n -q "DROP DATABASE IF EXISTS testlazy" diff --git a/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh b/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh index ef001b0988a..2003effb71b 100755 --- a/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh +++ b/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh export CURR_DATABASE="test_lazy_01014_concurrent_${CLICKHOUSE_DATABASE}" diff --git a/tests/queries/0_stateless/01015_insert_values_parametrized.sh b/tests/queries/0_stateless/01015_insert_values_parametrized.sh index 896ad97156a..bdc4fe01698 100755 --- a/tests/queries/0_stateless/01015_insert_values_parametrized.sh +++ b/tests/queries/0_stateless/01015_insert_values_parametrized.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS insert_values_parametrized"; diff --git a/tests/queries/0_stateless/01016_input_null_as_default.sh b/tests/queries/0_stateless/01016_input_null_as_default.sh index f31e6591e97..137e25b6a12 100755 --- a/tests/queries/0_stateless/01016_input_null_as_default.sh +++ b/tests/queries/0_stateless/01016_input_null_as_default.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS null_as_default"; diff --git a/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh b/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh index 992f5614772..d7d0dab71b9 100755 --- a/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh +++ b/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01017_tsv_empty_as_default.sh b/tests/queries/0_stateless/01017_tsv_empty_as_default.sh index a0404b89801..84faf7600f5 100755 --- a/tests/queries/0_stateless/01017_tsv_empty_as_default.sh +++ b/tests/queries/0_stateless/01017_tsv_empty_as_default.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS empty_as_default"; diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_bad_queries.sh b/tests/queries/0_stateless/01018_ddl_dictionaries_bad_queries.sh index 33cde0d5961..37e9fd774d7 100755 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_bad_queries.sh +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_bad_queries.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh b/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh index 6e47787b33a..bc13e44934a 100755 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01018_insert_multiple_blocks_with_defaults.sh b/tests/queries/0_stateless/01018_insert_multiple_blocks_with_defaults.sh index ced0f269327..5733bbfc6d2 100755 --- a/tests/queries/0_stateless/01018_insert_multiple_blocks_with_defaults.sh +++ b/tests/queries/0_stateless/01018_insert_multiple_blocks_with_defaults.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS defaults" diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh index 46f1e82f54c..54a7e940377 100755 --- a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh +++ b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --multiquery <&1 | grep -oP "Maximum parse depth .* exceeded." diff --git a/tests/queries/0_stateless/01071_http_header_exception_code.sh b/tests/queries/0_stateless/01071_http_header_exception_code.sh index 9e882bcf542..22444f46cbc 100755 --- a/tests/queries/0_stateless/01071_http_header_exception_code.sh +++ b/tests/queries/0_stateless/01071_http_header_exception_code.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh if [[ $(${CLICKHOUSE_CURL_COMMAND} -q -I "${CLICKHOUSE_URL}&query=BADREQUEST" 2>&1 | grep -c 'X-ClickHouse-Exception-Code: 62') -eq 1 ]]; then diff --git a/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh b/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh index 4f71ef383c7..2562e701a25 100755 --- a/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh +++ b/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh @@ -3,6 +3,7 @@ # This is a monkey test used to trigger sanitizers. CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="CREATE DATABASE IF NOT EXISTS dictdb_01076; " diff --git a/tests/queries/0_stateless/01076_json_each_row_array.sh b/tests/queries/0_stateless/01076_json_each_row_array.sh index 32327e57b5e..cbbaafacace 100755 --- a/tests/queries/0_stateless/01076_json_each_row_array.sh +++ b/tests/queries/0_stateless/01076_json_each_row_array.sh @@ -3,6 +3,7 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh ${CLICKHOUSE_LOCAL} --query "SELECT '[' || arrayStringConcat(arrayMap(x -> '{\"id\": 1, \"name\": \"name1\"}', range(1000000)), ',') || ']'" | ${CLICKHOUSE_LOCAL} --query "SELECT count() FROM table" --input-format JSONEachRow --structure 'id UInt32, name String' diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh index c9e5d7b9447..ca453ee8f0d 100755 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -10,6 +10,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh REPLICAS=5 diff --git a/tests/queries/0_stateless/01077_mutations_index_consistency.sh b/tests/queries/0_stateless/01077_mutations_index_consistency.sh index aa77906a8ad..129ef0b161c 100755 --- a/tests/queries/0_stateless/01077_mutations_index_consistency.sh +++ b/tests/queries/0_stateless/01077_mutations_index_consistency.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh b/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh index 98aa6d6e6e2..1c0206453b7 100755 --- a/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh +++ b/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_bad_alters"; diff --git a/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh index d6025652343..b3a5de8f9bc 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh REPLICAS=3 diff --git a/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh index 365bc2ae444..d5f0c987e5d 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh REPLICAS=3 diff --git a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh index 777c694e5bd..5b14c5a8543 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh REPLICAS=5 diff --git a/tests/queries/0_stateless/01085_max_distributed_connections.sh b/tests/queries/0_stateless/01085_max_distributed_connections.sh index 7b561604640..c63c671f7fc 100755 --- a/tests/queries/0_stateless/01085_max_distributed_connections.sh +++ b/tests/queries/0_stateless/01085_max_distributed_connections.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh opts=( diff --git a/tests/queries/0_stateless/01085_max_distributed_connections_http.sh b/tests/queries/0_stateless/01085_max_distributed_connections_http.sh index 23d609cfea6..d7eb75e717a 100755 --- a/tests/queries/0_stateless/01085_max_distributed_connections_http.sh +++ b/tests/queries/0_stateless/01085_max_distributed_connections_http.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh query="SELECT sleepEachRow(1) FROM remote('127.{2,3}', system.one)" diff --git a/tests/queries/0_stateless/01085_regexp_input_format.sh b/tests/queries/0_stateless/01085_regexp_input_format.sh index 4359cd5762d..5736d031c08 100755 --- a/tests/queries/0_stateless/01085_regexp_input_format.sh +++ b/tests/queries/0_stateless/01085_regexp_input_format.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS regexp"; diff --git a/tests/queries/0_stateless/01086_odbc_roundtrip.sh b/tests/queries/0_stateless/01086_odbc_roundtrip.sh index 1ce3e656f42..3c9e5c8aba9 100755 --- a/tests/queries/0_stateless/01086_odbc_roundtrip.sh +++ b/tests/queries/0_stateless/01086_odbc_roundtrip.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01086_regexp_input_format_skip_unmatched.sh b/tests/queries/0_stateless/01086_regexp_input_format_skip_unmatched.sh index 58c8c5275b8..c96aed7d3ee 100755 --- a/tests/queries/0_stateless/01086_regexp_input_format_skip_unmatched.sh +++ b/tests/queries/0_stateless/01086_regexp_input_format_skip_unmatched.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS regexp"; diff --git a/tests/queries/0_stateless/01088_benchmark_query_id.sh b/tests/queries/0_stateless/01088_benchmark_query_id.sh index 81cfa4dad84..cc3531282dd 100755 --- a/tests/queries/0_stateless/01088_benchmark_query_id.sh +++ b/tests/queries/0_stateless/01088_benchmark_query_id.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh QUERY_ID=$RANDOM diff --git a/tests/queries/0_stateless/01098_msgpack_format.sh b/tests/queries/0_stateless/01098_msgpack_format.sh index 8be55e99213..c7a1a0cff42 100755 --- a/tests/queries/0_stateless/01098_msgpack_format.sh +++ b/tests/queries/0_stateless/01098_msgpack_format.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS msgpack"; diff --git a/tests/queries/0_stateless/01098_temporary_and_external_tables.sh b/tests/queries/0_stateless/01098_temporary_and_external_tables.sh index 54834107728..bdac3c6fae3 100755 --- a/tests/queries/0_stateless/01098_temporary_and_external_tables.sh +++ b/tests/queries/0_stateless/01098_temporary_and_external_tables.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh url_without_session="https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTPS}/?" diff --git a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh index 30319e7cfea..1039f8f7d97 100755 --- a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh +++ b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # If we run sanitized binary under qemu, it will try to slowly allocate 20 TiB until OOM. diff --git a/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh b/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh index c48665b6093..287a63f858b 100755 --- a/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh +++ b/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh index 173bf44e1f1..227c4df6785 100755 --- a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh +++ b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS test_01107" diff --git a/tests/queries/0_stateless/01107_tuples_arrays_parsing_exceptions.sh b/tests/queries/0_stateless/01107_tuples_arrays_parsing_exceptions.sh index 05d4b060a8a..36a8c507605 100755 --- a/tests/queries/0_stateless/01107_tuples_arrays_parsing_exceptions.sh +++ b/tests/queries/0_stateless/01107_tuples_arrays_parsing_exceptions.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "SELECT (1, 2 2)" 2>&1 | grep -o "Syntax error" diff --git a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh index 39240ea9f8d..b0cba465c3d 100755 --- a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh +++ b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh for i in $(seq 4); do diff --git a/tests/queries/0_stateless/01114_database_atomic.sh b/tests/queries/0_stateless/01114_database_atomic.sh index 2a3a171b724..55288e1e3f6 100755 --- a/tests/queries/0_stateless/01114_database_atomic.sh +++ b/tests/queries/0_stateless/01114_database_atomic.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01146_clickhouse_local_data.sh b/tests/queries/0_stateless/01146_clickhouse_local_data.sh index 2c98b06942b..7268f051b8b 100755 --- a/tests/queries/0_stateless/01146_clickhouse_local_data.sh +++ b/tests/queries/0_stateless/01146_clickhouse_local_data.sh @@ -2,6 +2,7 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh ${CLICKHOUSE_LOCAL} --query "create table test engine Log as select 1 a" diff --git a/tests/queries/0_stateless/01150_ddl_guard_rwr.sh b/tests/queries/0_stateless/01150_ddl_guard_rwr.sh index 43804075938..6355f790e23 100755 --- a/tests/queries/0_stateless/01150_ddl_guard_rwr.sh +++ b/tests/queries/0_stateless/01150_ddl_guard_rwr.sh @@ -3,6 +3,7 @@ CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS test_01150" diff --git a/tests/queries/0_stateless/01187_set_profile_as_setting.sh b/tests/queries/0_stateless/01187_set_profile_as_setting.sh index 8247ab8870a..db9d095fe92 100755 --- a/tests/queries/0_stateless/01187_set_profile_as_setting.sh +++ b/tests/queries/0_stateless/01187_set_profile_as_setting.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -n -m -q "select value, changed from system.settings where name='readonly';" diff --git a/tests/queries/0_stateless/01192_rename_database_zookeeper.sh b/tests/queries/0_stateless/01192_rename_database_zookeeper.sh index bb84cab8977..90b9baf4ebf 100755 --- a/tests/queries/0_stateless/01192_rename_database_zookeeper.sh +++ b/tests/queries/0_stateless/01192_rename_database_zookeeper.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh # 1. init diff --git a/tests/queries/0_stateless/01193_metadata_loading.sh b/tests/queries/0_stateless/01193_metadata_loading.sh index 0ee583a7265..2e28c2c0165 100755 --- a/tests/queries/0_stateless/01193_metadata_loading.sh +++ b/tests/queries/0_stateless/01193_metadata_loading.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # it is the worst way of making performance test, nevertheless it can detect significant slowdown and some other issues, that usually found by stress test diff --git a/tests/queries/0_stateless/01194_http_query_id.sh b/tests/queries/0_stateless/01194_http_query_id.sh index 5eb766bbe2a..b1ab9eed9db 100755 --- a/tests/queries/0_stateless/01194_http_query_id.sh +++ b/tests/queries/0_stateless/01194_http_query_id.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh url="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?session_id=test_01194" diff --git a/tests/queries/0_stateless/01195_formats_diagnostic_info.sh b/tests/queries/0_stateless/01195_formats_diagnostic_info.sh index e3bd8f901e7..6c64b17f719 100755 --- a/tests/queries/0_stateless/01195_formats_diagnostic_info.sh +++ b/tests/queries/0_stateless/01195_formats_diagnostic_info.sh @@ -2,6 +2,7 @@ # shellcheck disable=SC2206 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh PARSER=(${CLICKHOUSE_LOCAL} --query 'SELECT t, s, d FROM table' --structure 't DateTime, s String, d Decimal64(10)' --input-format CSV) @@ -21,7 +22,7 @@ echo '2020-04-21 12:34:56, "Hello", 12345678,1' | "${PARSER[@]}" 2>&1| grep "ERR echo '2020-04-21 12:34:56,,123Hello' | "${PARSER[@]}" 2>&1| grep "ERROR" echo -e '2020-04-21 12:34:56, "Hello", 12345678\n\n\n\n ' | "${PARSER[@]}" 2>&1| grep "ERROR" || echo "OK" -PARSER=(${CLICKHOUSE_LOCAL} --input_format_null_as_default=0 --query 'SELECT t, s, d FROM table' --structure 't DateTime, s String, d Decimal64(10)' --input-format TSV) +PARSER=(${CLICKHOUSE_LOCAL} --input_format_null_as_default 0 --query 'SELECT t, s, d FROM table' --structure 't DateTime, s String, d Decimal64(10)' --input-format TSV) echo -e '2020-04-21 12:34:56\tHello\t12345678' | "${PARSER[@]}" 2>&1| grep "ERROR" || echo -e "\nTSV" echo -e '2020-04-21 12:34:56\tHello\t123456789' | "${PARSER[@]}" 2>&1| grep "ERROR" echo -e '2020-04-21 12:34:567\tHello\t123456789' | "${PARSER[@]}" 2>&1| grep "ERROR" diff --git a/tests/queries/0_stateless/01196_max_parser_depth.sh b/tests/queries/0_stateless/01196_max_parser_depth.sh index ecbb741121f..ae4851bf0c3 100755 --- a/tests/queries/0_stateless/01196_max_parser_depth.sh +++ b/tests/queries/0_stateless/01196_max_parser_depth.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh { printf "select "; for _ in {1..1000}; do printf "coalesce(null, "; done; printf "1"; for _ in {1..1000}; do printf ")"; done; } > "${CLICKHOUSE_TMP}"/query diff --git a/tests/queries/0_stateless/01198_client_quota_key.sh b/tests/queries/0_stateless/01198_client_quota_key.sh index 698d5b5841f..1a08b33e336 100755 --- a/tests/queries/0_stateless/01198_client_quota_key.sh +++ b/tests/queries/0_stateless/01198_client_quota_key.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --quota_key Hello --query_id test_quota_key --log_queries 1 --multiquery --query "SELECT 1; SYSTEM FLUSH LOGS; SELECT DISTINCT quota_key FROM system.query_log WHERE event_date >= yesterday() AND event_time >= now() - 300 AND query_id = 'test_quota_key'" diff --git a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.sh b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.sh index ae0c8324468..5ab0e800d39 100755 --- a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.sh +++ b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_rename_replicated" diff --git a/tests/queries/0_stateless/01232_json_as_string_format.sh b/tests/queries/0_stateless/01232_json_as_string_format.sh index dd988052edf..ed8c5d37cac 100755 --- a/tests/queries/0_stateless/01232_json_as_string_format.sh +++ b/tests/queries/0_stateless/01232_json_as_string_format.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS json_as_string"; diff --git a/tests/queries/0_stateless/01232_preparing_sets_race_condition.sh b/tests/queries/0_stateless/01232_preparing_sets_race_condition.sh index cc490a1fcd2..e42e68a6589 100755 --- a/tests/queries/0_stateless/01232_preparing_sets_race_condition.sh +++ b/tests/queries/0_stateless/01232_preparing_sets_race_condition.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -o errexit diff --git a/tests/queries/0_stateless/01238_http_memory_tracking.sh b/tests/queries/0_stateless/01238_http_memory_tracking.sh index b317a6c109b..90a7611c7c7 100755 --- a/tests/queries/0_stateless/01238_http_memory_tracking.sh +++ b/tests/queries/0_stateless/01238_http_memory_tracking.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -o pipefail diff --git a/tests/queries/0_stateless/01249_flush_interactive.sh b/tests/queries/0_stateless/01249_flush_interactive.sh index 6049c70b9df..2af75dbcbe5 100755 --- a/tests/queries/0_stateless/01249_flush_interactive.sh +++ b/tests/queries/0_stateless/01249_flush_interactive.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # A question for curious reader: diff --git a/tests/queries/0_stateless/01258_bom_tsv.sh b/tests/queries/0_stateless/01258_bom_tsv.sh index 085214f4b9d..576f16a246e 100755 --- a/tests/queries/0_stateless/01258_bom_tsv.sh +++ b/tests/queries/0_stateless/01258_bom_tsv.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # BOM can be parsed if TSV format has first column that cannot contain arbitrary binary data (such as integer) diff --git a/tests/queries/0_stateless/01268_procfs_metrics.sh b/tests/queries/0_stateless/01268_procfs_metrics.sh index 0ff3b4518d0..cad9b786667 100755 --- a/tests/queries/0_stateless/01268_procfs_metrics.sh +++ b/tests/queries/0_stateless/01268_procfs_metrics.sh @@ -5,6 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh function read_numbers_func() diff --git a/tests/queries/0_stateless/01271_http_code_parse_error.sh b/tests/queries/0_stateless/01271_http_code_parse_error.sh index 887a819edc9..5cc6f83b7be 100755 --- a/tests/queries/0_stateless/01271_http_code_parse_error.sh +++ b/tests/queries/0_stateless/01271_http_code_parse_error.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test" diff --git a/tests/queries/0_stateless/01273_arrow.sh b/tests/queries/0_stateless/01273_arrow.sh index 6d62b2b4e4a..ad8a6f0fdb9 100755 --- a/tests/queries/0_stateless/01273_arrow.sh +++ b/tests/queries/0_stateless/01273_arrow.sh @@ -3,6 +3,7 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01273_arrow_load.sh b/tests/queries/0_stateless/01273_arrow_load.sh index 7f4b88ec8e2..b2ca0e32af1 100755 --- a/tests/queries/0_stateless/01273_arrow_load.sh +++ b/tests/queries/0_stateless/01273_arrow_load.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh CB_DIR=$(dirname "$CLICKHOUSE_CLIENT_BINARY") diff --git a/tests/queries/0_stateless/01273_arrow_stream.sh b/tests/queries/0_stateless/01273_arrow_stream.sh index 3ad89b636e3..2020d4ca06f 100755 --- a/tests/queries/0_stateless/01273_arrow_stream.sh +++ b/tests/queries/0_stateless/01273_arrow_stream.sh @@ -3,6 +3,8 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01274_generate_random_nested.sh b/tests/queries/0_stateless/01274_generate_random_nested.sh index 966e1f8da28..a4add05e007 100755 --- a/tests/queries/0_stateless/01274_generate_random_nested.sh +++ b/tests/queries/0_stateless/01274_generate_random_nested.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -sS "$CLICKHOUSE_URL" -d "SELECT * FROM generateRandom('\"ParsedParams.Key1\" Array(String), \"ParsedParams.Key2\" Array(Float64), x String', 1, 10, 2) LIMIT 10" > /dev/null; diff --git a/tests/queries/0_stateless/01278_format_multiple_queries.sh b/tests/queries/0_stateless/01278_format_multiple_queries.sh index 8c06b285b6a..55dca42eaef 100755 --- a/tests/queries/0_stateless/01278_format_multiple_queries.sh +++ b/tests/queries/0_stateless/01278_format_multiple_queries.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh index f933283cddf..80af1b2c17f 100755 --- a/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh +++ b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # just in case diff --git a/tests/queries/0_stateless/01279_empty_external_table.sh b/tests/queries/0_stateless/01279_empty_external_table.sh index 16a6b5293a9..43dbcf296e6 100755 --- a/tests/queries/0_stateless/01279_empty_external_table.sh +++ b/tests/queries/0_stateless/01279_empty_external_table.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01280_ttl_where_group_by.sh b/tests/queries/0_stateless/01280_ttl_where_group_by.sh index 1d0d627e5bb..5ca79951a46 100755 --- a/tests/queries/0_stateless/01280_ttl_where_group_by.sh +++ b/tests/queries/0_stateless/01280_ttl_where_group_by.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query "drop table if exists ttl_01280_1" diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh index 5333d0b4b0b..285e2ab8dad 100755 --- a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh +++ b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh @@ -9,6 +9,7 @@ # - one users' query in background (to avoid reseting max_memory_usage_for_user) CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -o pipefail @@ -29,8 +30,8 @@ function execute_group_by() # max_memory_usage_for_user is installed to 0 once there are no more # queries for user. local opts=( - --max_memory_usage_for_user=$((150<<20)) - --max_threads=2 + "--max_memory_usage_for_user="$((150<<20)) + "--max_threads=2" ) execute_null "${opts[@]}" <<<'SELECT uniq(number) FROM numbers_mt(toUInt64(1e6)) GROUP BY number % 5e5' } diff --git a/tests/queries/0_stateless/01285_engine_join_donmikel.sh b/tests/queries/0_stateless/01285_engine_join_donmikel.sh index 2d5c769cf8d..7522ed9924b 100755 --- a/tests/queries/0_stateless/01285_engine_join_donmikel.sh +++ b/tests/queries/0_stateless/01285_engine_join_donmikel.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --multiquery --query " diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.sh b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.sh index 8f4885b29c0..560d9472504 100755 --- a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.sh +++ b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CURDIR}/01293_client_interactive_vertical_multiline.expect diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.sh b/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.sh index 873523e1cac..d1c513e0f83 100755 --- a/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.sh +++ b/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CURDIR}/01293_client_interactive_vertical_singleline.expect diff --git a/tests/queries/0_stateless/01293_optimize_final_force.sh b/tests/queries/0_stateless/01293_optimize_final_force.sh index a03bc6e311d..60d45f87385 100755 --- a/tests/queries/0_stateless/01293_optimize_final_force.sh +++ b/tests/queries/0_stateless/01293_optimize_final_force.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh for _ in {1..100}; do $CLICKHOUSE_CLIENT --multiquery --query " diff --git a/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables.sh b/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables.sh index d43a9361d71..d8f72c7837d 100755 --- a/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables.sh +++ b/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh export CURR_DATABASE="test_lazy_01294_concurrent_${CLICKHOUSE_DATABASE}" diff --git a/tests/queries/0_stateless/01300_client_save_history_when_terminated.sh b/tests/queries/0_stateless/01300_client_save_history_when_terminated.sh index 2f7c16259e3..8e0f15e57f0 100755 --- a/tests/queries/0_stateless/01300_client_save_history_when_terminated.sh +++ b/tests/queries/0_stateless/01300_client_save_history_when_terminated.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CURDIR}/01300_client_save_history_when_terminated.expect diff --git a/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh b/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh index 25dc43f9e62..65bf5de8333 100755 --- a/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh +++ b/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh function test() diff --git a/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh b/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh index 0afeeef6363..ed8a463cc62 100755 --- a/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh +++ b/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh function test() diff --git a/tests/queries/0_stateless/01304_direct_io.sh b/tests/queries/0_stateless/01304_direct_io.sh index 0126a9525ed..3ba3d020d99 100755 --- a/tests/queries/0_stateless/01304_direct_io.sh +++ b/tests/queries/0_stateless/01304_direct_io.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --multiquery --query " diff --git a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh index e3e702b7b1f..5dd3d2b38d6 100755 --- a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh +++ b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01306_benchmark_json.sh b/tests/queries/0_stateless/01306_benchmark_json.sh index 51021ee8361..fed085abd96 100755 --- a/tests/queries/0_stateless/01306_benchmark_json.sh +++ b/tests/queries/0_stateless/01306_benchmark_json.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01307_multiple_leaders_zookeeper.sh b/tests/queries/0_stateless/01307_multiple_leaders_zookeeper.sh index 3b29922b8fc..24c6199a94a 100755 --- a/tests/queries/0_stateless/01307_multiple_leaders_zookeeper.sh +++ b/tests/queries/0_stateless/01307_multiple_leaders_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01307_orc_output_format.sh b/tests/queries/0_stateless/01307_orc_output_format.sh index 9293ff6ad64..26c7db5ad1b 100755 --- a/tests/queries/0_stateless/01307_orc_output_format.sh +++ b/tests/queries/0_stateless/01307_orc_output_format.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS orc"; diff --git a/tests/queries/0_stateless/01308_orc_output_format_arrays.sh b/tests/queries/0_stateless/01308_orc_output_format_arrays.sh index 66dc04d5a3b..f24f62673ac 100755 --- a/tests/queries/0_stateless/01308_orc_output_format_arrays.sh +++ b/tests/queries/0_stateless/01308_orc_output_format_arrays.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS orc"; diff --git a/tests/queries/0_stateless/01312_skip_empty_params.sh b/tests/queries/0_stateless/01312_skip_empty_params.sh index 1b9333aab33..5920376db86 100755 --- a/tests/queries/0_stateless/01312_skip_empty_params.sh +++ b/tests/queries/0_stateless/01312_skip_empty_params.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01316_create_user_syntax_hilite.sh b/tests/queries/0_stateless/01316_create_user_syntax_hilite.sh index a7b1b0abb00..bf79b7cb7ab 100755 --- a/tests/queries/0_stateless/01316_create_user_syntax_hilite.sh +++ b/tests/queries/0_stateless/01316_create_user_syntax_hilite.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01317_no_password_in_command_line.sh b/tests/queries/0_stateless/01317_no_password_in_command_line.sh index 07ac2b88832..c9886aca31e 100755 --- a/tests/queries/0_stateless/01317_no_password_in_command_line.sh +++ b/tests/queries/0_stateless/01317_no_password_in_command_line.sh @@ -2,6 +2,7 @@ # shellcheck disable=SC2009 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01318_alter_add_constraint_format.sh b/tests/queries/0_stateless/01318_alter_add_constraint_format.sh index 7494563564b..931f2051274 100755 --- a/tests/queries/0_stateless/01318_alter_add_constraint_format.sh +++ b/tests/queries/0_stateless/01318_alter_add_constraint_format.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh b/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh index 95a8bcb5bb3..ced668e9849 100755 --- a/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh +++ b/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS mutation_table" diff --git a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh index cc6a66bd6bc..a15d8c8d2cd 100755 --- a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh +++ b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e diff --git a/tests/queries/0_stateless/01338_long_select_and_alter.sh b/tests/queries/0_stateless/01338_long_select_and_alter.sh index c4ef1cb7f39..499b62564bd 100755 --- a/tests/queries/0_stateless/01338_long_select_and_alter.sh +++ b/tests/queries/0_stateless/01338_long_select_and_alter.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt" diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh index 6a852357fa6..d990a8a1c08 100755 --- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh +++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt" diff --git a/tests/queries/0_stateless/01339_client_unrecognized_option.sh b/tests/queries/0_stateless/01339_client_unrecognized_option.sh index 13c286cd032..f590ed3807e 100755 --- a/tests/queries/0_stateless/01339_client_unrecognized_option.sh +++ b/tests/queries/0_stateless/01339_client_unrecognized_option.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT xyzgarbage 2>&1 | grep -q "Code: 552" && echo 'OK' || echo 'FAIL' diff --git a/tests/queries/0_stateless/01342_query_parameters_alias.sh b/tests/queries/0_stateless/01342_query_parameters_alias.sh index c17425ec7d7..11fbe37dabb 100755 --- a/tests/queries/0_stateless/01342_query_parameters_alias.sh +++ b/tests/queries/0_stateless/01342_query_parameters_alias.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --param_x '\N' --query 'SELECT {x:Nullable(Nothing)} as a' --format TSVWithNamesAndTypes diff --git a/tests/queries/0_stateless/01355_CSV_input_format_allow_errors.sh b/tests/queries/0_stateless/01355_CSV_input_format_allow_errors.sh index 7ae77eb9f0c..a7cc9739f71 100755 --- a/tests/queries/0_stateless/01355_CSV_input_format_allow_errors.sh +++ b/tests/queries/0_stateless/01355_CSV_input_format_allow_errors.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh SAMPLE_FILE="$CURDIR/01355_sample_data.csv" diff --git a/tests/queries/0_stateless/01358_lc_parquet.sh b/tests/queries/0_stateless/01358_lc_parquet.sh index 8732cc4eefd..3c49adc6185 100755 --- a/tests/queries/0_stateless/01358_lc_parquet.sh +++ b/tests/queries/0_stateless/01358_lc_parquet.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "drop table if exists test_lc" diff --git a/tests/queries/0_stateless/01361_fover_remote_num_tries.sh b/tests/queries/0_stateless/01361_fover_remote_num_tries.sh index 4dcf5e69415..5e49b393c7a 100755 --- a/tests/queries/0_stateless/01361_fover_remote_num_tries.sh +++ b/tests/queries/0_stateless/01361_fover_remote_num_tries.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --connections_with_failover_max_tries 10 --query "SELECT hostName() FROM remote('128.1.2.3', default.tmp)" 2>&1 | grep -o -P 'connect timed out|Network is unreachable' | wc -l diff --git a/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.sh b/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.sh index 419d6d58e85..aa7c9a94eb0 100755 --- a/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.sh +++ b/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CURDIR}/01370_client_autocomplete_word_break_characters.expect diff --git a/tests/queries/0_stateless/01375_output_format_tsv_csv_with_names.sh b/tests/queries/0_stateless/01375_output_format_tsv_csv_with_names.sh index de4486a88a5..ad9cc2c53a8 100755 --- a/tests/queries/0_stateless/01375_output_format_tsv_csv_with_names.sh +++ b/tests/queries/0_stateless/01375_output_format_tsv_csv_with_names.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh opts=( diff --git a/tests/queries/0_stateless/01375_storage_file_tsv_csv_with_names_write_prefix.sh b/tests/queries/0_stateless/01375_storage_file_tsv_csv_with_names_write_prefix.sh index d396981f873..469f7e7008b 100755 --- a/tests/queries/0_stateless/01375_storage_file_tsv_csv_with_names_write_prefix.sh +++ b/tests/queries/0_stateless/01375_storage_file_tsv_csv_with_names_write_prefix.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # zero rows diff --git a/tests/queries/0_stateless/01383_log_broken_table.sh b/tests/queries/0_stateless/01383_log_broken_table.sh index 80efa7e3908..37cd6e239e5 100755 --- a/tests/queries/0_stateless/01383_log_broken_table.sh +++ b/tests/queries/0_stateless/01383_log_broken_table.sh @@ -2,6 +2,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01393_benchmark_secure_port.sh b/tests/queries/0_stateless/01393_benchmark_secure_port.sh index 4ec220efa2e..6928041da14 100755 --- a/tests/queries/0_stateless/01393_benchmark_secure_port.sh +++ b/tests/queries/0_stateless/01393_benchmark_secure_port.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_BENCHMARK --secure -i 100 <<< 'SELECT 1' 2>&1 | grep -F 'Queries executed' | tail -n1 diff --git a/tests/queries/0_stateless/01395_limit_more_cases.sh b/tests/queries/0_stateless/01395_limit_more_cases.sh index 61c0b6ee6e0..32c854e53fb 100755 --- a/tests/queries/0_stateless/01395_limit_more_cases.sh +++ b/tests/queries/0_stateless/01395_limit_more_cases.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh SIZE=13 diff --git a/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh b/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh index 4d68a8a722e..30b2b665658 100755 --- a/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh +++ b/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh # Check that if we have one inactive replica and a huge number of INSERTs to active replicas, # the number of nodes in ZooKeeper does not grow unbounded. diff --git a/tests/queries/0_stateless/01399_http_request_headers.sh b/tests/queries/0_stateless/01399_http_request_headers.sh index d7f1c8df608..9b07f018230 100755 --- a/tests/queries/0_stateless/01399_http_request_headers.sh +++ b/tests/queries/0_stateless/01399_http_request_headers.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -H 'X-ClickHouse-User: default' -d 'SELECT 1' diff --git a/tests/queries/0_stateless/01401_FORMAT_SETTINGS.sh b/tests/queries/0_stateless/01401_FORMAT_SETTINGS.sh index aa23fcfe7aa..b70c28422c9 100755 --- a/tests/queries/0_stateless/01401_FORMAT_SETTINGS.sh +++ b/tests/queries/0_stateless/01401_FORMAT_SETTINGS.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -o pipefail diff --git a/tests/queries/0_stateless/01406_carriage_return_in_tsv_csv.sh b/tests/queries/0_stateless/01406_carriage_return_in_tsv_csv.sh index 5a1edb42a06..9c3d7726763 100755 --- a/tests/queries/0_stateless/01406_carriage_return_in_tsv_csv.sh +++ b/tests/queries/0_stateless/01406_carriage_return_in_tsv_csv.sh @@ -2,6 +2,7 @@ # shellcheck disable=SC2028 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo 'CSVWithNames' diff --git a/tests/queries/0_stateless/01412_cache_dictionary_race.sh b/tests/queries/0_stateless/01412_cache_dictionary_race.sh index 68f44e10c50..587cec52932 100755 --- a/tests/queries/0_stateless/01412_cache_dictionary_race.sh +++ b/tests/queries/0_stateless/01412_cache_dictionary_race.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh b/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh index cd59c6e46d7..ceeeed41049 100755 --- a/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh +++ b/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS replicated_mutation_table" diff --git a/tests/queries/0_stateless/01415_sticking_mutations.sh b/tests/queries/0_stateless/01415_sticking_mutations.sh index ce34cd09ca3..9bd0a6eeebf 100755 --- a/tests/queries/0_stateless/01415_sticking_mutations.sh +++ b/tests/queries/0_stateless/01415_sticking_mutations.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS sticking_mutations" diff --git a/tests/queries/0_stateless/01417_freeze_partition_verbose.sh b/tests/queries/0_stateless/01417_freeze_partition_verbose.sh index f15eb26e4b8..5294f4fe8f1 100755 --- a/tests/queries/0_stateless/01417_freeze_partition_verbose.sh +++ b/tests/queries/0_stateless/01417_freeze_partition_verbose.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh ALTER_OUT_STRUCTURE='command_type String, partition_id String, part_name String' ATTACH_OUT_STRUCTURE='old_part_name String' diff --git a/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh b/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh index d9011e2acdf..480daeefa46 100755 --- a/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh +++ b/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh ALTER_OUT_STRUCTURE='command_type String, partition_id String, part_name String' ATTACH_OUT_STRUCTURE='old_part_name String' diff --git a/tests/queries/0_stateless/01417_query_time_in_system_events.sh b/tests/queries/0_stateless/01417_query_time_in_system_events.sh index ff6d11befb0..4c0701c958e 100755 --- a/tests/queries/0_stateless/01417_query_time_in_system_events.sh +++ b/tests/queries/0_stateless/01417_query_time_in_system_events.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh DATA_BEFORE=`${CLICKHOUSE_CLIENT} --query="SELECT event,value FROM system.events WHERE event IN ('QueryTimeMicroseconds','SelectQueryTimeMicroseconds','InsertQueryTimeMicroseconds') FORMAT CSV"` diff --git a/tests/queries/0_stateless/01429_empty_arrow_and_parquet.sh b/tests/queries/0_stateless/01429_empty_arrow_and_parquet.sh index 223a5d246ae..74ac4f21c21 100755 --- a/tests/queries/0_stateless/01429_empty_arrow_and_parquet.sh +++ b/tests/queries/0_stateless/01429_empty_arrow_and_parquet.sh @@ -3,6 +3,8 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01442_merge_detach_attach.sh b/tests/queries/0_stateless/01442_merge_detach_attach.sh index a0ed8e42357..e134752a03f 100755 --- a/tests/queries/0_stateless/01442_merge_detach_attach.sh +++ b/tests/queries/0_stateless/01442_merge_detach_attach.sh @@ -3,6 +3,8 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=none/g') diff --git a/tests/queries/0_stateless/01443_merge_truncate.sh b/tests/queries/0_stateless/01443_merge_truncate.sh index 23c5a8f6c77..ffd5f225ffe 100755 --- a/tests/queries/0_stateless/01443_merge_truncate.sh +++ b/tests/queries/0_stateless/01443_merge_truncate.sh @@ -3,6 +3,7 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=none/g') diff --git a/tests/queries/0_stateless/01444_create_table_drop_database_race.sh b/tests/queries/0_stateless/01444_create_table_drop_database_race.sh index d54731fd7a3..0e2d4464bf8 100755 --- a/tests/queries/0_stateless/01444_create_table_drop_database_race.sh +++ b/tests/queries/0_stateless/01444_create_table_drop_database_race.sh @@ -3,6 +3,8 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh # This test reproduces "Directory not empty" error in DROP DATABASE query. diff --git a/tests/queries/0_stateless/01445_create_table_as_table_function.sh b/tests/queries/0_stateless/01445_create_table_as_table_function.sh index 6be015fc8a3..db2b8e100d6 100755 --- a/tests/queries/0_stateless/01445_create_table_as_table_function.sh +++ b/tests/queries/0_stateless/01445_create_table_as_table_function.sh @@ -3,6 +3,8 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query "CREATE TABLE system.columns AS numbers(10);" 2>&1 | grep -F "Code: 57" > /dev/null && echo 'OK' || echo 'FAIL' diff --git a/tests/queries/0_stateless/01446_json_strings_each_row.sh b/tests/queries/0_stateless/01446_json_strings_each_row.sh index af5bac6d2de..a2d98cd7f90 100755 --- a/tests/queries/0_stateless/01446_json_strings_each_row.sh +++ b/tests/queries/0_stateless/01446_json_strings_each_row.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo "DROP TABLE IF EXISTS test_table;" | ${CLICKHOUSE_CLIENT} diff --git a/tests/queries/0_stateless/01451_dist_logs.sh b/tests/queries/0_stateless/01451_dist_logs.sh index d192eb30251..23dee7a827d 100755 --- a/tests/queries/0_stateless/01451_dist_logs.sh +++ b/tests/queries/0_stateless/01451_dist_logs.sh @@ -4,6 +4,7 @@ CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=trace CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # triggered not for the first query diff --git a/tests/queries/0_stateless/01451_wrong_error_long_query.sh b/tests/queries/0_stateless/01451_wrong_error_long_query.sh index 00bfb285aaf..333dab193cd 100755 --- a/tests/queries/0_stateless/01451_wrong_error_long_query.sh +++ b/tests/queries/0_stateless/01451_wrong_error_long_query.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -printf "select 1 in (1, 1, %1048554s (-1))" | ${CLICKHOUSE_CURL} -ss 'http://localhost:8123/?max_query_size=1048576' --data-binary @- | grep -o "Max query size exceeded" -printf "select 1 in (1, 1, %1048554s (-1))" | ${CLICKHOUSE_CURL} -ss 'http://localhost:8123/?max_query_size=1048580' --data-binary @- +printf "select 1 in (1, 1, %1048554s (-1))" " " | ${CLICKHOUSE_CURL} -ss 'http://localhost:8123/?max_query_size=1048576' --data-binary @- | grep -o "Max query size exceeded" +printf "select 1 in (1, 1, %1048554s (-1))" " " | ${CLICKHOUSE_CURL} -ss 'http://localhost:8123/?max_query_size=1048580' --data-binary @- diff --git a/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh b/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh index 8fce5c9065b..49829b288ae 100755 --- a/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh +++ b/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mem" diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh index f81a010cf95..d8a8dde966e 100755 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh @@ -2,6 +2,7 @@ set -ue CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh function check_log diff --git a/tests/queries/0_stateless/01459_manual_write_to_replicas.sh b/tests/queries/0_stateless/01459_manual_write_to_replicas.sh index 1cf0ed56bc5..0a437d689fa 100755 --- a/tests/queries/0_stateless/01459_manual_write_to_replicas.sh +++ b/tests/queries/0_stateless/01459_manual_write_to_replicas.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh NUM_REPLICAS=10 diff --git a/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh b/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh index e033e0d5b72..a333d9be8ae 100755 --- a/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh +++ b/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh NUM_REPLICAS=10 diff --git a/tests/queries/0_stateless/01460_line_as_string_format.sh b/tests/queries/0_stateless/01460_line_as_string_format.sh index 4f94111df08..17ab4b4367b 100755 --- a/tests/queries/0_stateless/01460_line_as_string_format.sh +++ b/tests/queries/0_stateless/01460_line_as_string_format.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS line_as_string1"; @@ -18,14 +19,14 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE line_as_string1" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS line_as_string2"; $CLICKHOUSE_CLIENT --query="create table line_as_string2( - a UInt64 default 42, + a UInt64 default 42, b String materialized toString(a), c String ) engine=MergeTree() order by tuple();"; $CLICKHOUSE_CLIENT --query="INSERT INTO line_as_string2(c) values ('ClickHouse')"; -echo 'ClickHouse is a `fast` #open-source# (OLAP) 'database' "management" :system:' | $CLICKHOUSE_CLIENT --query="INSERT INTO line_as_string2(c) FORMAT LineAsString"; +echo "ClickHouse is a `fast` #open-source# (OLAP) 'database' "'"management"'" :system:" | $CLICKHOUSE_CLIENT --query="INSERT INTO line_as_string2(c) FORMAT LineAsString"; $CLICKHOUSE_CLIENT --query="SELECT * FROM line_as_string2 order by c"; $CLICKHOUSE_CLIENT --query="DROP TABLE line_as_string2" diff --git a/tests/queries/0_stateless/01472_obfuscator_uuid.sh b/tests/queries/0_stateless/01472_obfuscator_uuid.sh index 2efd4986faa..6654dcaad71 100755 --- a/tests/queries/0_stateless/01472_obfuscator_uuid.sh +++ b/tests/queries/0_stateless/01472_obfuscator_uuid.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="CREATE TABLE t_uuid(Id UUID) ENGINE=MergeTree ORDER BY (Id)" diff --git a/tests/queries/0_stateless/01474_custom_null_tsv.sh b/tests/queries/0_stateless/01474_custom_null_tsv.sh index ee9bb7900a0..9dc1c4b7777 100755 --- a/tests/queries/0_stateless/01474_custom_null_tsv.sh +++ b/tests/queries/0_stateless/01474_custom_null_tsv.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS tsv_custom_null"; diff --git a/tests/queries/0_stateless/01500_StorageFile_write_to_fd.sh b/tests/queries/0_stateless/01500_StorageFile_write_to_fd.sh index 589a578eb0b..e6fe5d17cd0 100755 --- a/tests/queries/0_stateless/01500_StorageFile_write_to_fd.sh +++ b/tests/queries/0_stateless/01500_StorageFile_write_to_fd.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # The following command will execute: diff --git a/tests/queries/0_stateless/01501_clickhouse_client_INSERT_exception.sh b/tests/queries/0_stateless/01501_clickhouse_client_INSERT_exception.sh index 4e96f393f30..2abb1818529 100755 --- a/tests/queries/0_stateless/01501_clickhouse_client_INSERT_exception.sh +++ b/tests/queries/0_stateless/01501_clickhouse_client_INSERT_exception.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS data" diff --git a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh index 06f7d38af94..869e3a1d26d 100755 --- a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh +++ b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ncpus="$(getconf _NPROCESSORS_ONLN)" diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh index 96d2e32f590..667a612ff23 100755 --- a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh +++ b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh @@ -5,6 +5,7 @@ set -e CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh b/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh index f259badea8c..645eaea743c 100755 --- a/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh +++ b/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # Regression for UAF in ThreadPool. diff --git a/tests/queries/0_stateless/01507_clickhouse_server_start_with_embedded_config.sh b/tests/queries/0_stateless/01507_clickhouse_server_start_with_embedded_config.sh index 945be0a1324..5b8f0cba639 100755 --- a/tests/queries/0_stateless/01507_clickhouse_server_start_with_embedded_config.sh +++ b/tests/queries/0_stateless/01507_clickhouse_server_start_with_embedded_config.sh @@ -4,6 +4,7 @@ CLICKHOUSE_PORT_TCP=50111 CLICKHOUSE_DATABASE=default CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo "Starting clickhouse-server" diff --git a/tests/queries/0_stateless/01508_format_regexp_raw.sh b/tests/queries/0_stateless/01508_format_regexp_raw.sh index 699fca1be61..8cf1bd73566 100755 --- a/tests/queries/0_stateless/01508_format_regexp_raw.sh +++ b/tests/queries/0_stateless/01508_format_regexp_raw.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} -n --query " diff --git a/tests/queries/0_stateless/01508_partition_pruning.sh b/tests/queries/0_stateless/01508_partition_pruning.sh index c886946c7d9..b5ec6388d5c 100755 --- a/tests/queries/0_stateless/01508_partition_pruning.sh +++ b/tests/queries/0_stateless/01508_partition_pruning.sh @@ -10,6 +10,7 @@ #------------------------------------------------------------------------------------------- CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh #export CLICKHOUSE_CLIENT="clickhouse-client --send_logs_level=none" diff --git a/tests/queries/0_stateless/01508_query_obfuscator.sh b/tests/queries/0_stateless/01508_query_obfuscator.sh index d60e42489fa..b354f0953fd 100755 --- a/tests/queries/0_stateless/01508_query_obfuscator.sh +++ b/tests/queries/0_stateless/01508_query_obfuscator.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_FORMAT --seed Hello --obfuscate <<< "SELECT 123, 'Test://2020-01-01hello1234 at 2000-01-01T01:02:03', 12e100, Gibberish_id_testCool, hello(World), avgIf(remote('127.0.0.1'))" diff --git a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh index 2af1cb214a4..4cb4734b448 100755 --- a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh +++ b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_renames0" diff --git a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh index 526859bcb58..c5ffad1c4ca 100755 --- a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh +++ b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh NUM_REPLICAS=10 diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh index eecb06bda5d..898a68d9c77 100755 --- a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh +++ b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh NUM_REPLICAS=2 diff --git a/tests/queries/0_stateless/01509_format_raw_blob.sh b/tests/queries/0_stateless/01509_format_raw_blob.sh index 68d3844d727..3d1d3fbb17b 100755 --- a/tests/queries/0_stateless/01509_format_raw_blob.sh +++ b/tests/queries/0_stateless/01509_format_raw_blob.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} -n --query " diff --git a/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh b/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh index 214c39a21cc..ca5f58512a3 100755 --- a/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh +++ b/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh @@ -3,6 +3,7 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parallel_q1" diff --git a/tests/queries/0_stateless/01510_format_regexp_raw_low_cardinality.sh b/tests/queries/0_stateless/01510_format_regexp_raw_low_cardinality.sh index 0f65280e1ce..594caca7d04 100755 --- a/tests/queries/0_stateless/01510_format_regexp_raw_low_cardinality.sh +++ b/tests/queries/0_stateless/01510_format_regexp_raw_low_cardinality.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} -n --query " diff --git a/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh b/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh index b0abd99d38c..726dcc8ee6d 100755 --- a/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh +++ b/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # _shard_num: @@ -9,10 +10,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # max_block_size to fail faster # max_memory_usage/_shard_num/repeat() will allow failure on the first shard earlier. opts=( - --max_memory_usage=1G - --max_block_size=50 - --max_threads=1 - --max_distributed_connections=2 + "--max_memory_usage=1G" + "--max_block_size=50" + "--max_threads=1" + "--max_distributed_connections=2" ) ${CLICKHOUSE_CLIENT} "${opts[@]}" -q "SELECT groupArray(repeat('a', if(_shard_num == 2, 100000, 1))), number%100000 k from remote('127.{2,3}', system.numbers) GROUP BY k LIMIT 10e6" |& { # the query should fail earlier on 127.3 and 127.2 should not even go to the memory limit exceeded error. diff --git a/tests/queries/0_stateless/01515_logtrace_function.sh b/tests/queries/0_stateless/01515_logtrace_function.sh index c536c90e74b..9953fc2ae2b 100755 --- a/tests/queries/0_stateless/01515_logtrace_function.sh +++ b/tests/queries/0_stateless/01515_logtrace_function.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=debug/g') diff --git a/tests/queries/0_stateless/01516_drop_table_stress.sh b/tests/queries/0_stateless/01516_drop_table_stress.sh index 3e2fd613a36..d72104c8c7f 100755 --- a/tests/queries/0_stateless/01516_drop_table_stress.sh +++ b/tests/queries/0_stateless/01516_drop_table_stress.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh function drop_database() diff --git a/tests/queries/0_stateless/01520_client_print_query_id.sh b/tests/queries/0_stateless/01520_client_print_query_id.sh index 21f60b49924..b32f48239c2 100755 --- a/tests/queries/0_stateless/01520_client_print_query_id.sh +++ b/tests/queries/0_stateless/01520_client_print_query_id.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CURDIR}/01520_client_print_query_id.expect diff --git a/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh b/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh index bd8cbc03095..f4681907ced 100755 --- a/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh +++ b/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo "SELECT 1;" > 01523_client_local_queries_file_parameter_tmp.sql diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.sh b/tests/queries/0_stateless/01526_client_start_and_exit.sh index d6b7b4f72ed..40927cd7599 100755 --- a/tests/queries/0_stateless/01526_client_start_and_exit.sh +++ b/tests/queries/0_stateless/01526_client_start_and_exit.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # Create a huge amount of tables, so Suggest will take a time to load diff --git a/tests/queries/0_stateless/01526_initial_query_id.sh b/tests/queries/0_stateless/01526_initial_query_id.sh index e28f9ee1e40..0cc09f733d0 100755 --- a/tests/queries/0_stateless/01526_initial_query_id.sh +++ b/tests/queries/0_stateless/01526_initial_query_id.sh @@ -2,6 +2,7 @@ set -ue CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh query_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") diff --git a/tests/queries/0_stateless/01526_max_untracked_memory.sh b/tests/queries/0_stateless/01526_max_untracked_memory.sh index 2623d175d82..b11f40b44f4 100755 --- a/tests/queries/0_stateless/01526_max_untracked_memory.sh +++ b/tests/queries/0_stateless/01526_max_untracked_memory.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh query="select randomPrintableASCII(number) from numbers(1000)" diff --git a/tests/queries/0_stateless/01526_param_uuid.sh b/tests/queries/0_stateless/01526_param_uuid.sh index 4f508ddf863..b0c68894fdb 100755 --- a/tests/queries/0_stateless/01526_param_uuid.sh +++ b/tests/queries/0_stateless/01526_param_uuid.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --param_p1='ffffffff-ffff-ffff-ffff-ffffffffffff' --query "SELECT {p1:UUID}" diff --git a/tests/queries/0_stateless/01527_clickhouse_local_optimize.sh b/tests/queries/0_stateless/01527_clickhouse_local_optimize.sh index bbbdf9c65d6..82453c00ca4 100755 --- a/tests/queries/0_stateless/01527_clickhouse_local_optimize.sh +++ b/tests/queries/0_stateless/01527_clickhouse_local_optimize.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh WORKING_FOLDER_01527="${CLICKHOUSE_TMP}/01527_clickhouse_local_optimize" diff --git a/tests/queries/0_stateless/01528_clickhouse_local_prepare_parts.sh b/tests/queries/0_stateless/01528_clickhouse_local_prepare_parts.sh index 9b09edfe27a..8684582ad45 100755 --- a/tests/queries/0_stateless/01528_clickhouse_local_prepare_parts.sh +++ b/tests/queries/0_stateless/01528_clickhouse_local_prepare_parts.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh WORKING_FOLDER_01528="${CLICKHOUSE_TMP}/01528_clickhouse_local_prepare_parts" diff --git a/tests/queries/0_stateless/01528_play.sh b/tests/queries/0_stateless/01528_play.sh index 7182f4dd6e5..09cf503676d 100755 --- a/tests/queries/0_stateless/01528_play.sh +++ b/tests/queries/0_stateless/01528_play.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/play" | grep -o '🌞' diff --git a/tests/queries/0_stateless/01529_bad_memory_tracking.sh b/tests/queries/0_stateless/01529_bad_memory_tracking.sh index f91f6ebaf80..5ad2535074a 100755 --- a/tests/queries/0_stateless/01529_bad_memory_tracking.sh +++ b/tests/queries/0_stateless/01529_bad_memory_tracking.sh @@ -3,6 +3,7 @@ CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh for _ in {1..10}; do diff --git a/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.sh b/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.sh index f341fbcdd9b..09f7e023288 100755 --- a/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.sh +++ b/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # in case when clickhouse-local can't use temp folder it will try to create diff --git a/tests/queries/0_stateless/01541_max_memory_usage_for_user.sh b/tests/queries/0_stateless/01541_max_memory_usage_for_user.sh index 64a90d871e7..c81bd1a6ce4 100755 --- a/tests/queries/0_stateless/01541_max_memory_usage_for_user.sh +++ b/tests/queries/0_stateless/01541_max_memory_usage_for_user.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # Regression for MemoryTracker drift via HTTP queries. diff --git a/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh b/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh index 582a8eb4d12..334fcc87baf 100755 --- a/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh +++ b/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01543_avro_deserialization_with_lc.sh b/tests/queries/0_stateless/01543_avro_deserialization_with_lc.sh index 0971396ec9c..f3f636dee73 100755 --- a/tests/queries/0_stateless/01543_avro_deserialization_with_lc.sh +++ b/tests/queries/0_stateless/01543_avro_deserialization_with_lc.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS test_01543 (value LowCardinality(String)) ENGINE=Memory()" diff --git a/tests/queries/0_stateless/01544_file_engine_settings.sh b/tests/queries/0_stateless/01544_file_engine_settings.sh index ff6b0d3d373..eb0a8a964d0 100755 --- a/tests/queries/0_stateless/01544_file_engine_settings.sh +++ b/tests/queries/0_stateless/01544_file_engine_settings.sh @@ -2,6 +2,7 @@ set -eu CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh the_file="$CLICKHOUSE_TMP/01544-t.csv" diff --git a/tests/queries/0_stateless/01545_system_errors.sh b/tests/queries/0_stateless/01545_system_errors.sh index 402c4e34116..63af6bb8d43 100755 --- a/tests/queries/0_stateless/01545_system_errors.sh +++ b/tests/queries/0_stateless/01545_system_errors.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh prev="$(${CLICKHOUSE_CLIENT} -q "SELECT value FROM system.errors WHERE name = 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO'")" diff --git a/tests/queries/0_stateless/01548_create_table_compound_column_format.sh b/tests/queries/0_stateless/01548_create_table_compound_column_format.sh index 6c9384e01c1..99e3aed2825 100755 --- a/tests/queries/0_stateless/01548_create_table_compound_column_format.sh +++ b/tests/queries/0_stateless/01548_create_table_compound_column_format.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo "CREATE TABLE test(a Int64, b NESTED(a Int64)) ENGINE=TinyLog" | $CLICKHOUSE_FORMAT diff --git a/tests/queries/0_stateless/01548_parallel_parsing_max_memory.sh b/tests/queries/0_stateless/01548_parallel_parsing_max_memory.sh index 884d5b6e058..d7ee2840763 100755 --- a/tests/queries/0_stateless/01548_parallel_parsing_max_memory.sh +++ b/tests/queries/0_stateless/01548_parallel_parsing_max_memory.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh yes http://foobarfoobarfoobarfoobarfoobarfoobarfoobar.com | head -c1G > 1g.csv diff --git a/tests/queries/0_stateless/01550_query_identifier_parameters.sh b/tests/queries/0_stateless/01550_query_identifier_parameters.sh index 85ca67e4e3c..ece01d83254 100755 --- a/tests/queries/0_stateless/01550_query_identifier_parameters.sh +++ b/tests/queries/0_stateless/01550_query_identifier_parameters.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --param_tbl 'numbers' --query 'select * from system.{tbl:Identifier} limit 1' diff --git a/tests/queries/0_stateless/01550_type_map_formats_input.sh b/tests/queries/0_stateless/01550_type_map_formats_input.sh index 0e167d3b0e9..75bf03a7437 100755 --- a/tests/queries/0_stateless/01550_type_map_formats_input.sh +++ b/tests/queries/0_stateless/01550_type_map_formats_input.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS map_formats_input" diff --git a/tests/queries/0_stateless/01554_row_number_after_cannot_read_all_data.sh b/tests/queries/0_stateless/01554_row_number_after_cannot_read_all_data.sh index a29b44d2f16..a52c5d811b8 100755 --- a/tests/queries/0_stateless/01554_row_number_after_cannot_read_all_data.sh +++ b/tests/queries/0_stateless/01554_row_number_after_cannot_read_all_data.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo -n -e '\x01\x00\x00\x00\x05Hello\x80' | ${CLICKHOUSE_LOCAL} --structure 'x UInt32, s String' --query "SELECT * FROM table" --input-format RowBinary 2>&1 | grep -oF '(at row 2)' diff --git a/tests/queries/0_stateless/01558_ttest_scipy.sh b/tests/queries/0_stateless/01558_ttest_scipy.sh index bd203478586..fea368f8181 100755 --- a/tests/queries/0_stateless/01558_ttest_scipy.sh +++ b/tests/queries/0_stateless/01558_ttest_scipy.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # We should have correct env vars from shell_config.sh to run this test diff --git a/tests/queries/0_stateless/01559_misplaced_codec_diagnostics.sh b/tests/queries/0_stateless/01559_misplaced_codec_diagnostics.sh index 9904b6388d6..8a3242c7036 100755 --- a/tests/queries/0_stateless/01559_misplaced_codec_diagnostics.sh +++ b/tests/queries/0_stateless/01559_misplaced_codec_diagnostics.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query "CREATE TABLE t (c CODEC(NONE)) ENGINE = Memory" 2>&1 | grep -oF 'Unknown data type family: CODEC' | uniq diff --git a/tests/queries/0_stateless/01561_clickhouse_client_stage.sh b/tests/queries/0_stateless/01561_clickhouse_client_stage.sh index afe3703f4f3..a01bc7f5065 100755 --- a/tests/queries/0_stateless/01561_clickhouse_client_stage.sh +++ b/tests/queries/0_stateless/01561_clickhouse_client_stage.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh function execute_query() diff --git a/tests/queries/0_stateless/01561_mann_whitney_scipy.sh b/tests/queries/0_stateless/01561_mann_whitney_scipy.sh index e4e9152a97d..a04b630e2f3 100755 --- a/tests/queries/0_stateless/01561_mann_whitney_scipy.sh +++ b/tests/queries/0_stateless/01561_mann_whitney_scipy.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # We should have correct env vars from shell_config.sh to run this test diff --git a/tests/queries/0_stateless/01563_distributed_query_finish.sh b/tests/queries/0_stateless/01563_distributed_query_finish.sh index 16e4ed8ebd1..a9e6a5b1fce 100755 --- a/tests/queries/0_stateless/01563_distributed_query_finish.sh +++ b/tests/queries/0_stateless/01563_distributed_query_finish.sh @@ -4,6 +4,7 @@ # (NETWORK_ERROR will be in case of connection reset) CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -nm <&1 | grep -F -c 'Syntax error' diff --git a/tests/queries/0_stateless/01599_mutation_query_params.sh b/tests/queries/0_stateless/01599_mutation_query_params.sh index 6bc12b5409e..52b0131a9c2 100755 --- a/tests/queries/0_stateless/01599_mutation_query_params.sh +++ b/tests/queries/0_stateless/01599_mutation_query_params.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01600_benchmark_query.sh b/tests/queries/0_stateless/01600_benchmark_query.sh index 6b40b4464c7..a563c87a10f 100755 --- a/tests/queries/0_stateless/01600_benchmark_query.sh +++ b/tests/queries/0_stateless/01600_benchmark_query.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_BENCHMARK --iterations 10 --query "SELECT 1" 1>/dev/null 2>"$CLICKHOUSE_TMP"/err diff --git a/tests/queries/0_stateless/01600_detach_permanently.sh b/tests/queries/0_stateless/01600_detach_permanently.sh index e897f80a33f..087545ec378 100755 --- a/tests/queries/0_stateless/01600_detach_permanently.sh +++ b/tests/queries/0_stateless/01600_detach_permanently.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ## tests with real clickhouse restart would be a bit to heavy, @@ -13,7 +14,7 @@ mkdir -p "${WORKING_FOLDER_01600}" clickhouse_local() { local query="$1" shift - ${CLICKHOUSE_LOCAL} --query "$query" $@ -- --path="${WORKING_FOLDER_01600}" + ${CLICKHOUSE_LOCAL} --query "$query" "$@" -- --path="${WORKING_FOLDER_01600}" } test_detach_attach_sequence() { diff --git a/tests/queries/0_stateless/01600_log_queries_with_extensive_info.sh b/tests/queries/0_stateless/01600_log_queries_with_extensive_info.sh index 2bc0a662cd1..039a68a76f3 100755 --- a/tests/queries/0_stateless/01600_log_queries_with_extensive_info.sh +++ b/tests/queries/0_stateless/01600_log_queries_with_extensive_info.sh @@ -2,6 +2,7 @@ set -ue CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} -q "drop database if exists test_log_queries" "--query_id=01600_log_queries_with_extensive_info_000" diff --git a/tests/queries/0_stateless/01600_quota_by_forwarded_ip.sh b/tests/queries/0_stateless/01600_quota_by_forwarded_ip.sh index 33f5434dc6c..323dd88efab 100755 --- a/tests/queries/0_stateless/01600_quota_by_forwarded_ip.sh +++ b/tests/queries/0_stateless/01600_quota_by_forwarded_ip.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01601_proxy_protocol.sh b/tests/queries/0_stateless/01601_proxy_protocol.sh index 8a431ba6dae..e8d1a7c45b2 100755 --- a/tests/queries/0_stateless/01601_proxy_protocol.sh +++ b/tests/queries/0_stateless/01601_proxy_protocol.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh printf "PROXY TCP4 255.255.255.255 255.255.255.255 65535 65535\r\n\0\21ClickHouse client\24\r\253\251\3\0\7default\0\4\1\0\1\0\0\t0.0.0.0:0\1\tmilovidov\21milovidov-desktop\vClickHouse \24\r\253\251\3\0\1\0\0\0\2\1\25SELECT 'Hello, world'\2\0\247\203\254l\325\\z|\265\254F\275\333\206\342\24\202\24\0\0\0\n\0\0\0\240\1\0\2\377\377\377\377\0\0\0" | nc "${CLICKHOUSE_HOST}" "${CLICKHOUSE_PORT_TCP_WITH_PROXY}" | head -c150 | grep --text -o -F 'Hello, world' diff --git a/tests/queries/0_stateless/01601_temporary_table_session_scope.sh b/tests/queries/0_stateless/01601_temporary_table_session_scope.sh index fbc45aec216..f6eeb50941f 100755 --- a/tests/queries/0_stateless/01601_temporary_table_session_scope.sh +++ b/tests/queries/0_stateless/01601_temporary_table_session_scope.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=DROP+TEMPORARY+TABLE+IF+EXISTS+tmptable&session_id=session_1601a" diff --git a/tests/queries/0_stateless/01602_max_distributed_connections.sh b/tests/queries/0_stateless/01602_max_distributed_connections.sh index 8c19b6f5bb7..93c6071c091 100755 --- a/tests/queries/0_stateless/01602_max_distributed_connections.sh +++ b/tests/queries/0_stateless/01602_max_distributed_connections.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh common_opts=( diff --git a/tests/queries/0_stateless/01606_git_import.sh b/tests/queries/0_stateless/01606_git_import.sh index 66b87d86653..16a0b92abe7 100755 --- a/tests/queries/0_stateless/01606_git_import.sh +++ b/tests/queries/0_stateless/01606_git_import.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # Clone some not too large repository and create a database from it. diff --git a/tests/queries/0_stateless/01607_arrays_as_nested_csv.sh b/tests/queries/0_stateless/01607_arrays_as_nested_csv.sh index 2f150e934d0..946be7fb4af 100755 --- a/tests/queries/0_stateless/01607_arrays_as_nested_csv.sh +++ b/tests/queries/0_stateless/01607_arrays_as_nested_csv.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --multiquery --query " diff --git a/tests/queries/1_stateful/00090_thread_pool_deadlock.sh b/tests/queries/1_stateful/00090_thread_pool_deadlock.sh index 63ebcd39a99..e94243bfea2 100755 --- a/tests/queries/1_stateful/00090_thread_pool_deadlock.sh +++ b/tests/queries/1_stateful/00090_thread_pool_deadlock.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh echo '1'; diff --git a/tests/queries/1_stateful/00092_obfuscator.sh b/tests/queries/1_stateful/00092_obfuscator.sh index f13fcc341be..85f476c6ae5 100755 --- a/tests/queries/1_stateful/00092_obfuscator.sh +++ b/tests/queries/1_stateful/00092_obfuscator.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --max_threads 1 --query="SELECT URL, Title, SearchPhrase FROM test.hits LIMIT 1000" > "${CLICKHOUSE_TMP}"/data.tsv diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 1b1678dee05..ddad0a5ddac 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -85,12 +85,6 @@ find $ROOT_PATH -name '.gitmodules' | while read i; do grep -F 'url = ' $i | gre # There shouldn't be any code snippets under GPL or LGPL find $ROOT_PATH/{src,base,programs} -name '*.h' -or -name '*.cpp' 2>/dev/null | xargs grep -i -F 'General Public License' && echo "There shouldn't be any code snippets under GPL or LGPL" -# Check sh tests with Shellcheck -(cd $ROOT_PATH/tests/queries/0_stateless/ && shellcheck --check-sourced --external-sources --severity info --exclude SC1071,SC2086 *.sh ../1_stateful/*.sh) - -# Check docker scripts with shellcheck -find "$ROOT_PATH/docker" -executable -type f -exec file -F' ' --mime-type {} \; | awk -F' ' '$2==" text/x-shellscript" {print $1}' | xargs shellcheck - # There shouldn't be any docker containers outside docker directory find $ROOT_PATH -not -path $ROOT_PATH'/docker*' -not -path $ROOT_PATH'/contrib*' -name Dockerfile -type f 2>/dev/null | xargs --no-run-if-empty -n1 echo "Please move Dockerfile to docker directory:" diff --git a/utils/check-style/shellcheck-run.sh b/utils/check-style/shellcheck-run.sh new file mode 100755 index 00000000000..e53563c7ff0 --- /dev/null +++ b/utils/check-style/shellcheck-run.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +ROOT_PATH=$(git rev-parse --show-toplevel) +EXCLUDE_DIRS='build/|integration/|widechar_width/|glibc-compatibility/|memcpy/|consistent-hashing/|Parsers/New' +# Check sh tests with Shellcheck +(cd $ROOT_PATH/tests/queries/0_stateless/ && shellcheck --check-sourced --external-sources --severity info --exclude SC1071,SC2086 *.sh ../1_stateful/*.sh) + +# Check docker scripts with shellcheck +find "$ROOT_PATH/docker" -executable -type f -exec file -F' ' --mime-type {} \; | awk -F' ' '$2==" text/x-shellscript" {print $1}' | grep -v "entrypoint.alpine.sh" |xargs shellcheck + From 223f122aa3e914c140a0d126a438ae68b8d5853c Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 28 Dec 2020 17:08:23 +0300 Subject: [PATCH 205/284] remove duplicate comment --- tests/queries/0_stateless/01273_arrow_stream.sh | 1 - tests/queries/0_stateless/01429_empty_arrow_and_parquet.sh | 1 - tests/queries/0_stateless/01442_merge_detach_attach.sh | 1 - .../queries/0_stateless/01444_create_table_drop_database_race.sh | 1 - .../queries/0_stateless/01445_create_table_as_table_function.sh | 1 - 5 files changed, 5 deletions(-) diff --git a/tests/queries/0_stateless/01273_arrow_stream.sh b/tests/queries/0_stateless/01273_arrow_stream.sh index 2020d4ca06f..af5931a4bce 100755 --- a/tests/queries/0_stateless/01273_arrow_stream.sh +++ b/tests/queries/0_stateless/01273_arrow_stream.sh @@ -4,7 +4,6 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh -# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01429_empty_arrow_and_parquet.sh b/tests/queries/0_stateless/01429_empty_arrow_and_parquet.sh index 74ac4f21c21..5ee8379d431 100755 --- a/tests/queries/0_stateless/01429_empty_arrow_and_parquet.sh +++ b/tests/queries/0_stateless/01429_empty_arrow_and_parquet.sh @@ -4,7 +4,6 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh -# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh diff --git a/tests/queries/0_stateless/01442_merge_detach_attach.sh b/tests/queries/0_stateless/01442_merge_detach_attach.sh index e134752a03f..c99069b4aa2 100755 --- a/tests/queries/0_stateless/01442_merge_detach_attach.sh +++ b/tests/queries/0_stateless/01442_merge_detach_attach.sh @@ -4,7 +4,6 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh -# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=none/g') diff --git a/tests/queries/0_stateless/01444_create_table_drop_database_race.sh b/tests/queries/0_stateless/01444_create_table_drop_database_race.sh index 0e2d4464bf8..aba83aac207 100755 --- a/tests/queries/0_stateless/01444_create_table_drop_database_race.sh +++ b/tests/queries/0_stateless/01444_create_table_drop_database_race.sh @@ -4,7 +4,6 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh -# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh # This test reproduces "Directory not empty" error in DROP DATABASE query. diff --git a/tests/queries/0_stateless/01445_create_table_as_table_function.sh b/tests/queries/0_stateless/01445_create_table_as_table_function.sh index db2b8e100d6..f963c700779 100755 --- a/tests/queries/0_stateless/01445_create_table_as_table_function.sh +++ b/tests/queries/0_stateless/01445_create_table_as_table_function.sh @@ -4,7 +4,6 @@ set -e CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh -# shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} --query "CREATE TABLE system.columns AS numbers(10);" 2>&1 | grep -F "Code: 57" > /dev/null && echo 'OK' || echo 'FAIL' From 5869123a86e24c5286f2fd5f46d3a28635910b8e Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 28 Dec 2020 17:58:35 +0300 Subject: [PATCH 206/284] no empty line for check-duplicate-includes --- utils/check-style/check-duplicate-includes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/check-style/check-duplicate-includes.sh b/utils/check-style/check-duplicate-includes.sh index df843ead623..64aca4d180d 100755 --- a/utils/check-style/check-duplicate-includes.sh +++ b/utils/check-style/check-duplicate-includes.sh @@ -3,4 +3,4 @@ ROOT_PATH=$(git rev-parse --show-toplevel) # Find duplicate include directives -find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | while read file; do grep -P '^#include ' $file | sort | uniq -c | grep -v -P '^\s+1\s' && echo $file; done +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | while read file; do grep -P '^#include ' $file | sort | uniq -c | grep -v -P '^\s+1\s' && echo $file; done | sed '/^[[:space:]]*$/d' From b3723e5e567aaf8f5b3565918059c486a2287d1c Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 28 Dec 2020 18:25:35 +0300 Subject: [PATCH 207/284] md5sum of test query has changed --- tests/queries/0_stateless/01509_format_raw_blob.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01509_format_raw_blob.reference b/tests/queries/0_stateless/01509_format_raw_blob.reference index dfa8f538e67..05014001bd9 100644 --- a/tests/queries/0_stateless/01509_format_raw_blob.reference +++ b/tests/queries/0_stateless/01509_format_raw_blob.reference @@ -1,2 +1,2 @@ -96b229180107fd2d23fd0a2ef9326701 - -96b229180107fd2d23fd0a2ef9326701 - +9fd46251e5574c633cbfbb9293671888 - +9fd46251e5574c633cbfbb9293671888 - From cb5986d426df3cfe73a4a6e78947a1786f7cfcd1 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 28 Dec 2020 18:25:48 +0300 Subject: [PATCH 208/284] better --- tests/queries/0_stateless/01460_line_as_string_format.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01460_line_as_string_format.sh b/tests/queries/0_stateless/01460_line_as_string_format.sh index 17ab4b4367b..4ab9cb59858 100755 --- a/tests/queries/0_stateless/01460_line_as_string_format.sh +++ b/tests/queries/0_stateless/01460_line_as_string_format.sh @@ -26,7 +26,7 @@ $CLICKHOUSE_CLIENT --query="create table line_as_string2( $CLICKHOUSE_CLIENT --query="INSERT INTO line_as_string2(c) values ('ClickHouse')"; -echo "ClickHouse is a `fast` #open-source# (OLAP) 'database' "'"management"'" :system:" | $CLICKHOUSE_CLIENT --query="INSERT INTO line_as_string2(c) FORMAT LineAsString"; +echo 'ClickHouse is a `fast` #open-source# (OLAP) database "management" :system:' | $CLICKHOUSE_CLIENT --query="INSERT INTO line_as_string2(c) FORMAT LineAsString"; $CLICKHOUSE_CLIENT --query="SELECT * FROM line_as_string2 order by c"; $CLICKHOUSE_CLIENT --query="DROP TABLE line_as_string2" From 011d2b4631e1ee4f08b2463d4c67c6d3e9ad21b9 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 29 Dec 2020 13:29:54 +0300 Subject: [PATCH 209/284] disable couple checks for now --- docker/test/performance-comparison/compare.sh | 16 ++++++++-------- src/Dictionaries/IPAddressDictionary.cpp | 2 +- utils/check-style/shellcheck-run.sh | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 674e8dd1cab..59d7cc98063 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -50,13 +50,13 @@ function configure local setup_left_server_opts=( # server options - --config-file left/config/config.xml + --config-file=left/config/config.xml -- # server *config* directives overrides --path db0 --user_files_path db0/user_files --top_level_domains_path /top_level_domains - --tcp_port "$LEFT_SERVER_PORT" + --tcp_port $LEFT_SERVER_PORT ) left/clickhouse-server "${setup_left_server_opts[@]}" &> setup-server-log.log & left_pid=$! @@ -98,13 +98,13 @@ function restart local left_server_opts=( # server options - --config-file left/config/config.xml + --config-file=left/config/config.xml -- # server *config* directives overrides --path left/db --user_files_path left/db/user_files --top_level_domains_path /top_level_domains - --tcp_port "$LEFT_SERVER_PORT" + --tcp_port $LEFT_SERVER_PORT ) left/clickhouse-server "${left_server_opts[@]}" &>> left-server-log.log & left_pid=$! @@ -113,13 +113,13 @@ function restart local right_server_opts=( # server options - --config-file right/config/config.xml + --config-file=right/config/config.xml -- # server *config* directives overrides --path right/db --user_files_path right/db/user_files --top_level_domains_path /top_level_domains - --tcp_port "$RIGHT_SERVER_PORT" + --tcp_port $RIGHT_SERVER_PORT ) right/clickhouse-server "${right_server_opts[@]}" &>> right-server-log.log & right_pid=$! @@ -1109,7 +1109,7 @@ function upload_results then echo Database for test results is not specified, will not upload them. return 0 - fi + fi # Surprisingly, clickhouse-client doesn't understand --host 127.0.0.1:9000 # so I have to extract host and port with clickhouse-local. I tried to use @@ -1117,7 +1117,7 @@ function upload_results # parse host:port. set +x # Don't show password in the log clickhouse-client \ - "$(clickhouse-local --query "with '${CHPC_DATABASE_URL}' as url select '--host ' || domain(url) || ' --port ' || toString(port(url)) format TSV")" \ + $(clickhouse-local --query "with '${CHPC_DATABASE_URL}' as url select '--host ' || domain(url) || ' --port ' || toString(port(url)) format TSV") \ --secure \ --user "${CHPC_DATABASE_USER}" \ --password "${CHPC_DATABASE_PASSWORD}" \ diff --git a/src/Dictionaries/IPAddressDictionary.cpp b/src/Dictionaries/IPAddressDictionary.cpp index 4d8492f15a2..d2bbf6ec2fa 100644 --- a/src/Dictionaries/IPAddressDictionary.cpp +++ b/src/Dictionaries/IPAddressDictionary.cpp @@ -518,7 +518,7 @@ void IPAddressDictionary::loadData() { /// We format key attribute values here instead of filling with data from key_column /// because string representation can be normalized if bits beyond mask are set. - /// Also all IPv4 will be displayed as mapped IPv6 if threre are any IPv6. + /// Also all IPv4 will be displayed as mapped IPv6 if there are any IPv6. /// It's consistent with representation in table created with `ENGINE = Dictionary` from this dictionary. char str_buffer[48]; if (has_ipv6) diff --git a/utils/check-style/shellcheck-run.sh b/utils/check-style/shellcheck-run.sh index e53563c7ff0..c0063d4b191 100755 --- a/utils/check-style/shellcheck-run.sh +++ b/utils/check-style/shellcheck-run.sh @@ -2,8 +2,8 @@ ROOT_PATH=$(git rev-parse --show-toplevel) EXCLUDE_DIRS='build/|integration/|widechar_width/|glibc-compatibility/|memcpy/|consistent-hashing/|Parsers/New' # Check sh tests with Shellcheck -(cd $ROOT_PATH/tests/queries/0_stateless/ && shellcheck --check-sourced --external-sources --severity info --exclude SC1071,SC2086 *.sh ../1_stateful/*.sh) +(cd $ROOT_PATH/tests/queries/0_stateless/ && shellcheck --check-sourced --external-sources --severity info --exclude SC1071,SC2086,SC2016 *.sh ../1_stateful/*.sh) # Check docker scripts with shellcheck -find "$ROOT_PATH/docker" -executable -type f -exec file -F' ' --mime-type {} \; | awk -F' ' '$2==" text/x-shellscript" {print $1}' | grep -v "entrypoint.alpine.sh" |xargs shellcheck +find "$ROOT_PATH/docker" -executable -type f -exec file -F' ' --mime-type {} \; | awk -F' ' '$2==" text/x-shellscript" {print $1}' | grep -v "entrypoint.alpine.sh" | grep -v "compare.sh"| xargs shellcheck From 9b4b2fa441ebd9f61d2dd712e0a0081e0ec288b2 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 29 Dec 2020 17:57:44 +0300 Subject: [PATCH 210/284] fix typo --- src/Storages/MergeTree/ReplicatedMergeTreeAltersSequence.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAltersSequence.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeAltersSequence.cpp index c2619317776..db4fe34e702 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAltersSequence.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAltersSequence.cpp @@ -26,7 +26,7 @@ void ReplicatedMergeTreeAltersSequence::addMetadataAlter( int alter_version, std::lock_guard & /*state_lock*/) { /// Data alter (mutation) always added before. See ReplicatedMergeTreeQueue::pullLogsToQueue. - /// So mutation alredy added to this sequence or doesn't exist. + /// So mutation already added to this sequence or doesn't exist. if (!queue_state.count(alter_version)) queue_state.emplace(alter_version, AlterState{.metadata_finished=false, .data_finished=true}); else From 6336fbf1dfb491f6d2692ffe70dc202aa8e0d94b Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 29 Dec 2020 20:16:57 +0300 Subject: [PATCH 211/284] fix removing of empty parts in tables with old syntax --- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- .../test_merge_tree_empty_parts/__init__.py | 0 .../configs/cleanup_thread.xml | 6 +++ .../configs/remote_servers.xml | 13 +++++++ .../test_merge_tree_empty_parts/test.py | 38 +++++++++++++++++++ 5 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_merge_tree_empty_parts/__init__.py create mode 100644 tests/integration/test_merge_tree_empty_parts/configs/cleanup_thread.xml create mode 100644 tests/integration/test_merge_tree_empty_parts/configs/remote_servers.xml create mode 100644 tests/integration/test_merge_tree_empty_parts/test.py diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index fe15d5dad29..75c405cfec6 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -6154,7 +6154,7 @@ bool StorageReplicatedMergeTree::dropPart( /// DROP_RANGE with detach will move this part together with source parts to `detached/` dir. entry.type = LogEntry::DROP_RANGE; entry.source_replica = replica_name; - entry.new_part_name = drop_part_info.getPartName(); + entry.new_part_name = getPartNamePossiblyFake(format_version, drop_part_info); entry.detach = detach; entry.create_time = time(nullptr); diff --git a/tests/integration/test_merge_tree_empty_parts/__init__.py b/tests/integration/test_merge_tree_empty_parts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_merge_tree_empty_parts/configs/cleanup_thread.xml b/tests/integration/test_merge_tree_empty_parts/configs/cleanup_thread.xml new file mode 100644 index 00000000000..943662aad67 --- /dev/null +++ b/tests/integration/test_merge_tree_empty_parts/configs/cleanup_thread.xml @@ -0,0 +1,6 @@ + + + 0 + 0 + + \ No newline at end of file diff --git a/tests/integration/test_merge_tree_empty_parts/configs/remote_servers.xml b/tests/integration/test_merge_tree_empty_parts/configs/remote_servers.xml new file mode 100644 index 00000000000..e7369160a81 --- /dev/null +++ b/tests/integration/test_merge_tree_empty_parts/configs/remote_servers.xml @@ -0,0 +1,13 @@ + + + + + true + + node1 + 9000 + + + + + diff --git a/tests/integration/test_merge_tree_empty_parts/test.py b/tests/integration/test_merge_tree_empty_parts/test.py new file mode 100644 index 00000000000..bc2679d4c92 --- /dev/null +++ b/tests/integration/test_merge_tree_empty_parts/test.py @@ -0,0 +1,38 @@ +import pytest +import helpers.client +import helpers.cluster +from helpers.test_tools import assert_eq_with_retry + + +cluster = helpers.cluster.ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/cleanup_thread.xml'], with_zookeeper=True) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + +def test_empty_parts_alter_delete(started_cluster): + node1.query("CREATE TABLE empty_parts_delete (d Date, key UInt64, value String) \ + ENGINE = ReplicatedMergeTree('/clickhouse/tables/empty_parts_delete', 'r1', d, key, 8192)") + + node1.query("INSERT INTO empty_parts_delete VALUES (toDate('2020-10-10'), 1, 'a')") + node1.query("ALTER TABLE empty_parts_delete DELETE WHERE 1 SETTINGS mutations_sync = 2") + + print(node1.query("SELECT count() FROM empty_parts_delete")) + assert_eq_with_retry(node1, "SELECT count() FROM system.parts WHERE table = 'empty_parts_delete' AND active", "0") + +def test_empty_parts_summing(started_cluster): + node1.query("CREATE TABLE empty_parts_summing (d Date, key UInt64, value Int64) \ + ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/empty_parts_summing', 'r1', d, key, 8192)") + + node1.query("INSERT INTO empty_parts_summing VALUES (toDate('2020-10-10'), 1, 1)") + node1.query("INSERT INTO empty_parts_summing VALUES (toDate('2020-10-10'), 1, -1)") + node1.query("OPTIMIZE TABLE empty_parts_summing FINAL") + + assert_eq_with_retry(node1, "SELECT count() FROM system.parts WHERE table = 'empty_parts_summing' AND active", "0") From 68fac02d35dd100bcd1ee41a4e814f5309eb407b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 29 Dec 2020 20:27:00 +0300 Subject: [PATCH 212/284] Change union_default_mode to throw exception --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b09e960da36..d8ba4f45b77 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -417,7 +417,7 @@ class IColumn; M(UInt64, multiple_joins_rewriter_version, 0, "Obsolete setting, does nothing. Will be removed after 2021-03-31", 0) \ M(Bool, enable_debug_queries, false, "Enabled debug queries, but now is obsolete", 0) \ M(Bool, allow_experimental_database_atomic, true, "Obsolete setting, does nothing. Will be removed after 2021-02-12", 0) \ - M(UnionMode, union_default_mode, UnionMode::DISTINCT, "Set default Union Mode in SelectWithUnion query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without Union Mode will throw exception.", 0) \ + M(UnionMode, union_default_mode, UnionMode::Unspecified, "Set default Union Mode in SelectWithUnion query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without Union Mode will throw exception.", 0) \ M(Bool, optimize_aggregators_of_group_by_keys, true, "Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section", 0) \ M(Bool, optimize_group_by_function_keys, true, "Eliminates functions of other keys in GROUP BY section", 0) \ From 48605e7a30fdbd8d3614d7d97614fe884f30ccba Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 29 Dec 2020 20:36:10 +0300 Subject: [PATCH 213/284] Fixed exit code of watchdog --- base/daemon/BaseDaemon.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index f25bcdb91e1..39506186732 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -962,7 +962,7 @@ void BaseDaemon::setupWatchdog() if (WIFEXITED(status)) { logger().information(fmt::format("Child process exited normally with code {}.", WEXITSTATUS(status))); - _exit(status); + _exit(WEXITSTATUS(status)); } if (WIFSIGNALED(status)) @@ -980,7 +980,7 @@ void BaseDaemon::setupWatchdog() logger().fatal(fmt::format("Child process was terminated by signal {}.", sig)); if (sig == SIGINT || sig == SIGTERM || sig == SIGQUIT) - _exit(status); + _exit(128 + sig); } } else From e81a99156fa5f65447e49a33484d654fe074faba Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Tue, 29 Dec 2020 18:51:00 +0100 Subject: [PATCH 214/284] Attempt to prevent test flap That extra SELECT is executed synchronously, after that all further actions will have a more stable timing. --- tests/integration/test_storage_kafka/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 1f31cbdbbc7..3f2bc1ca250 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -1617,6 +1617,8 @@ def test_kafka_virtual_columns2(kafka_cluster): kafka_num_consumers = 2, kafka_format = 'JSONEachRow'; + SELECT * FROM test.kafka; + CREATE MATERIALIZED VIEW test.view Engine=Log AS SELECT value, _key, _topic, _partition, _offset, toUnixTimestamp(_timestamp), toUnixTimestamp64Milli(_timestamp_ms), _headers.name, _headers.value FROM test.kafka; ''') From 9c2cbeba550dc1f566397f78eb44bd04273a4c58 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 29 Dec 2020 21:16:24 +0300 Subject: [PATCH 215/284] Fix error --- src/Interpreters/Set.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index ae2937ed796..faec94de9ac 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -107,7 +107,7 @@ void Set::setHeader(const Block & header) { std::unique_lock lock(rwlock); - if (!empty()) + if (!data.empty()) return; keys_size = header.columns(); @@ -160,7 +160,7 @@ bool Set::insertFromBlock(const Block & block) { std::unique_lock lock(rwlock); - if (empty()) + if (data.empty()) throw Exception("Method Set::setHeader must be called before Set::insertFromBlock", ErrorCodes::LOGICAL_ERROR); ColumnRawPtrs key_columns; @@ -213,7 +213,7 @@ bool Set::insertFromBlock(const Block & block) } } - return limits.check(getTotalRowCount(), getTotalByteCount(), "IN-set", ErrorCodes::SET_SIZE_LIMIT_EXCEEDED); + return limits.check(data.getTotalRowCount(), data.getTotalByteCount(), "IN-set", ErrorCodes::SET_SIZE_LIMIT_EXCEEDED); } From d584f75cd73730d2678d19e164a374821d23c0af Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 29 Dec 2020 21:45:09 +0300 Subject: [PATCH 216/284] Update test --- .../01529_union_distinct_and_setting_union_default_mode.sql | 2 ++ .../0_stateless/01556_explain_select_with_union_query.sql | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql index e29e43f64ba..6f2fe847f5d 100644 --- a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql +++ b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql @@ -1,5 +1,7 @@ SELECT 1; +SET union_default_mode='DISTINCT'; + (((((((SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1; (((((((SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1; diff --git a/tests/queries/0_stateless/01556_explain_select_with_union_query.sql b/tests/queries/0_stateless/01556_explain_select_with_union_query.sql index 16271113b5f..7abc8a36027 100644 --- a/tests/queries/0_stateless/01556_explain_select_with_union_query.sql +++ b/tests/queries/0_stateless/01556_explain_select_with_union_query.sql @@ -1,3 +1,5 @@ +SET union_default_mode = 'DISTINCT'; + EXPLAIN SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1; EXPLAIN (SELECT 1 UNION ALL SELECT 1) UNION ALL SELECT 1; EXPLAIN SELECT 1 UNION (SELECT 1 UNION ALL SELECT 1); From cf11e2b1e007dfe4d23c1651d0446ddb70af681e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 29 Dec 2020 22:31:07 +0300 Subject: [PATCH 217/284] check-marks: add decompressed size --- utils/check-marks/main.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/check-marks/main.cpp b/utils/check-marks/main.cpp index 5590616bde6..6e93b207957 100644 --- a/utils/check-marks/main.cpp +++ b/utils/check-marks/main.cpp @@ -42,9 +42,10 @@ static void checkByCompressedReadBuffer(const std::string & mrk_path, const std: out << ", has rows after " << index_granularity_rows; } - out << ".\n" << DB::flush; - bin_in.seek(offset_in_compressed_file, offset_in_decompressed_block); + out << ", decompressed size " << bin_in.available(); + + out << ".\n" << DB::flush; } } From 8c256481c9378dca786650e7f5c20f105f9e78f5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 29 Dec 2020 22:37:27 +0300 Subject: [PATCH 218/284] check-marks: fix comments and help message --- utils/check-marks/main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/check-marks/main.cpp b/utils/check-marks/main.cpp index 6e93b207957..2b244dcf0b6 100644 --- a/utils/check-marks/main.cpp +++ b/utils/check-marks/main.cpp @@ -13,7 +13,7 @@ #include -/** This program checks correctness of .mrk (marks) file for corresponding compressed .bin file. +/** This program checks correctness of .mrk/.mrk2 (marks) file for corresponding compressed .bin file. */ static void checkByCompressedReadBuffer(const std::string & mrk_path, const std::string & bin_path) @@ -62,7 +62,7 @@ int main(int argc, char ** argv) if (options.count("help") || argc != 3) { - std::cout << "Usage: " << argv[0] << " file.mrk file.bin" << std::endl; + std::cout << "Usage: " << argv[0] << " file.mrk[2] file.bin" << std::endl; std::cout << desc << std::endl; return 1; } From 8fde2d6e1b96debc5c24d3569e912773d2f740db Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 29 Dec 2020 22:40:57 +0300 Subject: [PATCH 219/284] compressor: remove dead file --- utils/compressor/main.cpp | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 utils/compressor/main.cpp diff --git a/utils/compressor/main.cpp b/utils/compressor/main.cpp deleted file mode 100644 index 087bfa116de..00000000000 --- a/utils/compressor/main.cpp +++ /dev/null @@ -1,6 +0,0 @@ -int mainEntryClickHouseCompressor(int argc, char ** argv); - -int main(int argc, char ** argv) -{ - return mainEntryClickHouseCompressor(argc, argv); -} From a71c19306256bbf88a6e516319e434bf41058b14 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 29 Dec 2020 22:12:29 +0300 Subject: [PATCH 220/284] compressor: add positional arguments support for input/output --- programs/compressor/Compressor.cpp | 36 ++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index da92efdf251..0c546ecfa06 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -68,6 +70,8 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help,h", "produce help message") + ("input", boost::program_options::value()->value_name("INPUT"), "input file") + ("output", boost::program_options::value()->value_name("OUTPUT"), "output file") ("decompress,d", "decompress") ("block-size,b", boost::program_options::value()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size") ("hc", "use LZ4HC instead of LZ4") @@ -78,12 +82,16 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) ("stat", "print block statistics of compressed data") ; + boost::program_options::positional_options_description positional_desc; + positional_desc.add("input", 1); + positional_desc.add("output", 1); + boost::program_options::variables_map options; - boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); + boost::program_options::store(boost::program_options::command_line_parser(argc, argv).options(desc).positional(positional_desc).run(), options); if (options.count("help")) { - std::cout << "Usage: " << argv[0] << " [options] < in > out" << std::endl; + std::cout << "Usage: " << argv[0] << " [options] INPUT OUTPUT" << std::endl; std::cout << desc << std::endl; return 1; } @@ -132,25 +140,35 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) codec = CompressionCodecFactory::instance().get(method_family, level); - ReadBufferFromFileDescriptor rb(STDIN_FILENO); - WriteBufferFromFileDescriptor wb(STDOUT_FILENO); + std::unique_ptr rb; + std::unique_ptr wb; + + if (options.count("input")) + rb = std::make_unique(options["input"].as()); + else + rb = std::make_unique(STDIN_FILENO); + + if (options.count("output")) + wb = std::make_unique(options["output"].as()); + else + wb = std::make_unique(STDOUT_FILENO); if (stat_mode) { /// Output statistic for compressed file. - checkAndWriteHeader(rb, wb); + checkAndWriteHeader(*rb, *wb); } else if (decompress) { /// Decompression - CompressedReadBuffer from(rb); - copyData(from, wb); + CompressedReadBuffer from(*rb); + copyData(from, *wb); } else { /// Compression - CompressedWriteBuffer to(wb, codec, block_size); - copyData(rb, to); + CompressedWriteBuffer to(*wb, codec, block_size); + copyData(*rb, to); } } catch (...) From 3362c8fc40a1bf1ee60ab1a7c067ec1edff46e75 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 29 Dec 2020 23:06:14 +0300 Subject: [PATCH 221/284] compressor: add seek support while reading compressed file Useful to debug .bin files --- programs/compressor/Compressor.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index 0c546ecfa06..ed786d91c99 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -73,6 +74,8 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) ("input", boost::program_options::value()->value_name("INPUT"), "input file") ("output", boost::program_options::value()->value_name("OUTPUT"), "output file") ("decompress,d", "decompress") + ("offset-in-compressed-file", boost::program_options::value()->default_value(0ULL), "offset to the compressed block (i.e. physical file offset)") + ("offset-in-decompressed-block", boost::program_options::value()->default_value(0ULL), "offset to the decompressed block (i.e. virtual offset)") ("block-size,b", boost::program_options::value()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size") ("hc", "use LZ4HC instead of LZ4") ("zstd", "use ZSTD instead of LZ4") @@ -161,8 +164,25 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) else if (decompress) { /// Decompression - CompressedReadBuffer from(*rb); - copyData(from, *wb); + + size_t offset_in_compressed_file = options["offset-in-compressed-file"].as(); + size_t offset_in_decompressed_block = options["offset-in-decompressed-block"].as(); + + if (offset_in_compressed_file || offset_in_decompressed_block) + { + if (!options.count("input")) + { + throw DB::Exception("--offset-in-compressed-file/--offset-in-decompressed-block requires --input", DB::ErrorCodes::BAD_ARGUMENTS); + } + CompressedReadBufferFromFile compressed_file(options["input"].as(), 0, 0, 0); + compressed_file.seek(offset_in_compressed_file, offset_in_decompressed_block); + copyData(compressed_file, *wb); + } + else + { + CompressedReadBuffer from(*rb); + copyData(from, *wb); + } } else { From 009a28189482182b5f270acb6e6e3390cc1709d8 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 29 Dec 2020 23:07:35 +0300 Subject: [PATCH 222/284] compressor: using for program_options namespace --- programs/compressor/Compressor.cpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index ed786d91c99..fa62f988f0d 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -67,30 +67,31 @@ void checkAndWriteHeader(DB::ReadBuffer & in, DB::WriteBuffer & out) int mainEntryClickHouseCompressor(int argc, char ** argv) { using namespace DB; + namespace po = boost::program_options; - boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); + po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help,h", "produce help message") - ("input", boost::program_options::value()->value_name("INPUT"), "input file") - ("output", boost::program_options::value()->value_name("OUTPUT"), "output file") + ("input", po::value()->value_name("INPUT"), "input file") + ("output", po::value()->value_name("OUTPUT"), "output file") ("decompress,d", "decompress") - ("offset-in-compressed-file", boost::program_options::value()->default_value(0ULL), "offset to the compressed block (i.e. physical file offset)") - ("offset-in-decompressed-block", boost::program_options::value()->default_value(0ULL), "offset to the decompressed block (i.e. virtual offset)") - ("block-size,b", boost::program_options::value()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size") + ("offset-in-compressed-file", po::value()->default_value(0ULL), "offset to the compressed block (i.e. physical file offset)") + ("offset-in-decompressed-block", po::value()->default_value(0ULL), "offset to the decompressed block (i.e. virtual offset)") + ("block-size,b", po::value()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size") ("hc", "use LZ4HC instead of LZ4") ("zstd", "use ZSTD instead of LZ4") - ("codec", boost::program_options::value>()->multitoken(), "use codecs combination instead of LZ4") - ("level", boost::program_options::value(), "compression level for codecs specified via flags") + ("codec", po::value>()->multitoken(), "use codecs combination instead of LZ4") + ("level", po::value(), "compression level for codecs specified via flags") ("none", "use no compression instead of LZ4") ("stat", "print block statistics of compressed data") ; - boost::program_options::positional_options_description positional_desc; + po::positional_options_description positional_desc; positional_desc.add("input", 1); positional_desc.add("output", 1); - boost::program_options::variables_map options; - boost::program_options::store(boost::program_options::command_line_parser(argc, argv).options(desc).positional(positional_desc).run(), options); + po::variables_map options; + po::store(po::command_line_parser(argc, argv).options(desc).positional(positional_desc).run(), options); if (options.count("help")) { From e02c328453a6b607983e79c35d31e10e1a30f9ca Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 29 Dec 2020 23:08:29 +0300 Subject: [PATCH 223/284] compressor: remove superfluous warning suppressions --- programs/compressor/Compressor.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index fa62f988f0d..2e88c0d0b2b 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -61,9 +61,6 @@ void checkAndWriteHeader(DB::ReadBuffer & in, DB::WriteBuffer & out) } -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wmissing-declarations" - int mainEntryClickHouseCompressor(int argc, char ** argv) { using namespace DB; From 30508c1b504285ed426620156ec3c8c813e9b790 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 29 Dec 2020 23:09:00 +0300 Subject: [PATCH 224/284] compressor: return 0 on --help --- programs/compressor/Compressor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index 2e88c0d0b2b..3751ffcd453 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -94,7 +94,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) { std::cout << "Usage: " << argv[0] << " [options] INPUT OUTPUT" << std::endl; std::cout << desc << std::endl; - return 1; + return 0; } try From e8aae12984ed8e22891b631da2f8fd24041f859c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 29 Dec 2020 23:10:51 +0300 Subject: [PATCH 225/284] Cover clickhouse-compressor in tests --- .../01621_clickhouse_compressor.reference | 0 .../01621_clickhouse_compressor.sh | 31 +++++++++++++++++++ tests/queries/shell_config.sh | 1 + 3 files changed, 32 insertions(+) create mode 100644 tests/queries/0_stateless/01621_clickhouse_compressor.reference create mode 100755 tests/queries/0_stateless/01621_clickhouse_compressor.sh diff --git a/tests/queries/0_stateless/01621_clickhouse_compressor.reference b/tests/queries/0_stateless/01621_clickhouse_compressor.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01621_clickhouse_compressor.sh b/tests/queries/0_stateless/01621_clickhouse_compressor.sh new file mode 100755 index 00000000000..e00270e5db9 --- /dev/null +++ b/tests/queries/0_stateless/01621_clickhouse_compressor.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +set -e + +TEMP_DIR="$(mktemp -d /tmp/clickhouse.test..XXXXXX)" +cd "${TEMP_DIR:?}" + +function cleanup() +{ + rm -fr "${TEMP_DIR:?}" +} +trap cleanup EXIT + +# This is random garbage, so compression ratio will be very low. +tr -cd 'a-z0-9' < /dev/urandom | head -c1M > input + +# stdin/stdout streams +$CLICKHOUSE_COMPRESSOR < input > output +diff -q <($CLICKHOUSE_COMPRESSOR --decompress < output) input + +# positional arguments, and that fact that input/output will be overwritten +$CLICKHOUSE_COMPRESSOR input output +diff -q <($CLICKHOUSE_COMPRESSOR --decompress output) input + +# --offset-in-decompressed-block +diff -q <($CLICKHOUSE_COMPRESSOR --decompress --offset-in-decompressed-block 10 output) <(tail -c+$((10+1)) input) + +# TODO: --offset-in-compressed-file using some .bin file (via clickhouse-local + check-marks) diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index 5e9c78d26d9..0ca2cee3c77 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -22,6 +22,7 @@ export CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT:="$CLICKHOUSE_CLIENT_BINARY ${CLICK [ -x "${CLICKHOUSE_BINARY}" ] && CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY} local"} export CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY}-local"} export CLICKHOUSE_OBFUSCATOR=${CLICKHOUSE_OBFUSCATOR:="${CLICKHOUSE_BINARY}-obfuscator"} +export CLICKHOUSE_COMPRESSOR=${CLICKHOUSE_COMPRESSOR:="${CLICKHOUSE_BINARY}-compressor"} export CLICKHOUSE_BENCHMARK=${CLICKHOUSE_BENCHMARK:="${CLICKHOUSE_BINARY}-benchmark ${CLICKHOUSE_BENCHMARK_OPT0:-}"} export CLICKHOUSE_GIT_IMPORT=${CLICKHOUSE_GIT_IMPORT="${CLICKHOUSE_BINARY}-git-import"} From c84d02c658150efb21191c79852066c241a4b583 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Tue, 29 Dec 2020 23:54:11 +0300 Subject: [PATCH 226/284] Update union.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Изменил возможность записывать подзапросы с union в круглых скобках. --- docs/en/sql-reference/statements/select/union.md | 2 +- docs/ru/sql-reference/statements/select/union.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/select/union.md b/docs/en/sql-reference/statements/select/union.md index ade4a19ab3b..cae7aa07abb 100644 --- a/docs/en/sql-reference/statements/select/union.md +++ b/docs/en/sql-reference/statements/select/union.md @@ -27,7 +27,7 @@ Result columns are matched by their index (order inside `SELECT`). If column nam Type casting is performed for unions. For example, if two queries being combined have the same field with non-`Nullable` and `Nullable` types from a compatible type, the resulting `UNION` has a `Nullable` type field. -Queries that are parts of `UNION` can’t be enclosed in round brackets. [ORDER BY](../../../sql-reference/statements/select/order-by.md) and [LIMIT](../../../sql-reference/statements/select/limit.md) are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with `UNION` in a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause. +Queries that are parts of `UNION` can be enclosed in round brackets. [ORDER BY](../../../sql-reference/statements/select/order-by.md) and [LIMIT](../../../sql-reference/statements/select/limit.md) are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with `UNION` in a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause. If you use `UNION` without explicitly specifying `UNION ALL` or `UNION DISTINCT`, you can specify the union mode using the [union_default_mode](../../../operations/settings/settings.md#union-default-mode) setting. The setting values can be `ALL`, `DISTINCT` or an empty string. However, if you use `UNION` with `union_default_mode` setting to empty string, it will throw an exception. The following examples demonstrate the results of queries with different values setting. diff --git a/docs/ru/sql-reference/statements/select/union.md b/docs/ru/sql-reference/statements/select/union.md index ca9616cfd25..e0a79978fdd 100644 --- a/docs/ru/sql-reference/statements/select/union.md +++ b/docs/ru/sql-reference/statements/select/union.md @@ -27,7 +27,7 @@ SELECT CounterID, 2 AS table, sum(Sign) AS c При объединении выполняет приведение типов. Например, если два запроса имеют одно и то же поле с не-`Nullable` и `Nullable` совместимыми типами, полученные в результате `UNION` данные будут иметь `Nullable` тип. -Запросы, которые являются частью `UNION`, не могут быть заключены в круглые скобки. [ORDER BY](order-by.md) и [LIMIT](limit.md) применяются к отдельным запросам, а не к конечному результату. Если вам нужно применить преобразование к конечному результату, вы можете разместить все объединенные с помощью `UNION` запросы в подзапрос в секции [FROM](from.md). +Запросы, которые являются частью `UNION`, могут быть заключены в круглые скобки. [ORDER BY](order-by.md) и [LIMIT](limit.md) применяются к отдельным запросам, а не к конечному результату. Если вам нужно применить преобразование к конечному результату, вы можете разместить все объединенные с помощью `UNION` запросы в подзапрос в секции [FROM](from.md). Если используете `UNION` без явного указания `UNION ALL` или `UNION DISTINCT`, то вы можете указать режим объединения с помощью настройки [union_default_mode](../../../operations/settings/settings.md#union-default-mode), значениями которой могут быть `ALL`, `DISTINCT` или пустая строка. Однако если вы используете `UNION` с настройкой `union_default_mode`, значением которой является пустая строка, то будет сгенерировано исключение. В следующих примерах продемонстрированы результаты запросов при разных значениях настройки. From 107360fc77654fd6031c5359c45617627da40ef0 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 30 Dec 2020 00:04:39 +0300 Subject: [PATCH 227/284] add tests --- ...rallel_formatting_csv_and_friends.reference | 8 ++++++++ ...0159_parallel_formatting_csv_and_friends.sh | 18 ++++++++++++++++++ .../00159_parallel_formatting_http.reference | 12 ++++++++++++ .../00159_parallel_formatting_http.sh | 16 ++++++++++++++++ ...allel_formatting_json_and_friends.reference | 12 ++++++++++++ ...159_parallel_formatting_json_and_friends.sh | 18 ++++++++++++++++++ ...rallel_formatting_tsv_and_friends.reference | 12 ++++++++++++ ...0159_parallel_formatting_tsv_and_friends.sh | 18 ++++++++++++++++++ 8 files changed, 114 insertions(+) create mode 100644 tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.reference create mode 100755 tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.sh create mode 100644 tests/queries/1_stateful/00159_parallel_formatting_http.reference create mode 100755 tests/queries/1_stateful/00159_parallel_formatting_http.sh create mode 100644 tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.reference create mode 100755 tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh create mode 100644 tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference create mode 100755 tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh diff --git a/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.reference b/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.reference new file mode 100644 index 00000000000..04107d74341 --- /dev/null +++ b/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.reference @@ -0,0 +1,8 @@ +CSV, false +ea1c740f03f5dcc43a3044528ad0a98f - +CSV, true +ea1c740f03f5dcc43a3044528ad0a98f - +CSVWithNames, false +e986f353467c87b07e7143d7bff2daff - +CSVWithNames, true +e986f353467c87b07e7143d7bff2daff - diff --git a/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.sh b/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.sh new file mode 100755 index 00000000000..6296d722a97 --- /dev/null +++ b/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +FORMATS=('CSV' 'CSVWithNames') + +for format in "${FORMATS[@]}" +do + echo "$format, false"; + $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ + "SELECT ClientEventTime as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum + + echo "$format, true"; + $CLICKHOUSE_CLIENT --output_format_parallel_formatting=true -q \ + "SELECT ClientEventTime as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum +done + diff --git a/tests/queries/1_stateful/00159_parallel_formatting_http.reference b/tests/queries/1_stateful/00159_parallel_formatting_http.reference new file mode 100644 index 00000000000..499a0b8a7c7 --- /dev/null +++ b/tests/queries/1_stateful/00159_parallel_formatting_http.reference @@ -0,0 +1,12 @@ +TSV, false +8a984bbbfb127c430f67173f5371c6cb - +TSV, true +8a984bbbfb127c430f67173f5371c6cb - +CSV, false +ea1c740f03f5dcc43a3044528ad0a98f - +CSV, true +ea1c740f03f5dcc43a3044528ad0a98f - +JSONCompactEachRow, false +ba1081a754a06ef6563840b2d8d4d327 - +JSONCompactEachRow, true +ba1081a754a06ef6563840b2d8d4d327 - diff --git a/tests/queries/1_stateful/00159_parallel_formatting_http.sh b/tests/queries/1_stateful/00159_parallel_formatting_http.sh new file mode 100755 index 00000000000..4c5fb1f066f --- /dev/null +++ b/tests/queries/1_stateful/00159_parallel_formatting_http.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + + +FORMATS=('TSV' 'CSV' 'JSONCompactEachRow') + +for format in "${FORMATS[@]}" +do + echo "$format, false"; + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=SELECT+ClientEventTime+as+a,MobilePhoneModel+as+b,ClientIP6+as+c+FROM+test.hits+ORDER+BY+a,b,c+Format+$format&output_format_parallel_formatting=false" -d' ' | md5sum + + echo "$format, true"; + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=SELECT+ClientEventTime+as+a,MobilePhoneModel+as+b,ClientIP6+as+c+FROM+test.hits+ORDER+BY+a,b,c+Format+$format&output_format_parallel_formatting=true" -d' ' | md5sum +done diff --git a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.reference b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.reference new file mode 100644 index 00000000000..96353f350ec --- /dev/null +++ b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.reference @@ -0,0 +1,12 @@ +JSONEachRow, false +7251839681e559f5a92db107571bb357 - +JSONEachRow, true +7251839681e559f5a92db107571bb357 - +JSONCompactEachRow, false +ba1081a754a06ef6563840b2d8d4d327 - +JSONCompactEachRow, true +ba1081a754a06ef6563840b2d8d4d327 - +JSONCompactStringsEachRowWithNamesAndTypes, false +902e53f621d5336aa7f702a5d6b64b42 - +JSONCompactStringsEachRowWithNamesAndTypes, true +902e53f621d5336aa7f702a5d6b64b42 - diff --git a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh new file mode 100755 index 00000000000..64a7b83fb86 --- /dev/null +++ b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + + +FORMATS=('JSONEachRow' 'JSONCompactEachRow' 'JSONCompactStringsEachRowWithNamesAndTypes') + +for format in "${FORMATS[@]}" +do + echo "$format, false"; + $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ + "SELECT ClientEventTime as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum + + echo "$format, true"; + $CLICKHOUSE_CLIENT --output_format_parallel_formatting=true -q \ + "SELECT ClientEventTime as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum +done diff --git a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference new file mode 100644 index 00000000000..04d6db3b4af --- /dev/null +++ b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference @@ -0,0 +1,12 @@ +TSV, false +8a984bbbfb127c430f67173f5371c6cb - +TSV, true +8a984bbbfb127c430f67173f5371c6cb - +TSVWithNames, false +ead321ed96754ff1aa39d112bc28c43d - +TSVWithNames, true +ead321ed96754ff1aa39d112bc28c43d - +TSKV, false +1735308ecea5c269846f36a55d5b335f - +TSKV, true +1735308ecea5c269846f36a55d5b335f - diff --git a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh new file mode 100755 index 00000000000..afdda8d1453 --- /dev/null +++ b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + + +FORMATS=('TSV' 'TSVWithNames' 'TSKV') + +for format in "${FORMATS[@]}" +do + echo "$format, false"; + $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ + "SELECT ClientEventTime as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum + + echo "$format, true"; + $CLICKHOUSE_CLIENT --output_format_parallel_formatting=true -q \ + "SELECT ClientEventTime as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum +done From 74904a8e0f6f4362afa1ee987c9c94116dbfc170 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Wed, 30 Dec 2020 01:25:09 +0300 Subject: [PATCH 228/284] Fix --- docs/en/sql-reference/table-functions/mysql.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/table-functions/mysql.md b/docs/en/sql-reference/table-functions/mysql.md index 7be328a5475..2338fc6db89 100644 --- a/docs/en/sql-reference/table-functions/mysql.md +++ b/docs/en/sql-reference/table-functions/mysql.md @@ -25,13 +25,18 @@ mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_ - `password` — User password. -- `replace_query` — Flag that converts `INSERT INTO` queries to `REPLACE INTO`. If `replace_query=1`, the query is replaced. +- `replace_query` — Flag that converts `INSERT INTO` queries to `REPLACE INTO`. Possible values: + - `0` - The query is executed as `INSERT INTO`. + - `1` - The query is executed as `REPLACE INTO`. -- `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` expression that is added to the `INSERT` query. +- `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` expression that is added to the `INSERT` query. Can be specified only with `replace_query = 0` (if you simultaneously pass `replace_query = 1` and `on_duplicate_clause`, ClickHouse generates an exception). - Example: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, where `on_duplicate_clause` is `UPDATE c2 = c2 + 1`. See the MySQL documentation to find which `on_duplicate_clause` you can use with the `ON DUPLICATE KEY` clause. + Example: + ```sql + INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1` + ``` - To specify `on_duplicate_clause` you need to pass `0` to the `replace_query` parameter. If you simultaneously pass `replace_query = 1` and `on_duplicate_clause`, ClickHouse generates an exception. + `on_duplicate_clause` here is `UPDATE c2 = c2 + 1`. See the MySQL documentation to find which `on_duplicate_clause` you can use with the `ON DUPLICATE KEY` clause. Simple `WHERE` clauses such as `=, !=, >, >=, <, <=` are currently executed on the MySQL server. @@ -53,9 +58,9 @@ mysql> CREATE TABLE `test`.`test` ( -> `float_nullable` FLOAT NULL DEFAULT NULL, -> PRIMARY KEY (`int_id`)); -mysql> insert into test (`int_id`, `float`) VALUES (1,2); +mysql> INSERT INTO test (`int_id`, `float`) VALUES (1,2); -mysql> select * from test; +mysql> SELECT * FROM test; +--------+--------------+-------+----------------+ | int_id | int_nullable | float | float_nullable | +--------+--------------+-------+----------------+ From ffd73082ba647b8a00c6417669556bd93a85423f Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 30 Dec 2020 01:34:26 +0300 Subject: [PATCH 229/284] fix memory and add comment --- src/IO/BufferWithOwnMemory.h | 2 +- src/IO/WriteBuffer.h | 1 + .../Impl/ParallelFormattingOutputFormat.h | 33 ++++++++++++++++++- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/IO/BufferWithOwnMemory.h b/src/IO/BufferWithOwnMemory.h index 104c15a4acc..782eea84ed7 100644 --- a/src/IO/BufferWithOwnMemory.h +++ b/src/IO/BufferWithOwnMemory.h @@ -174,7 +174,7 @@ public: private: void nextImpl() override final { - const size_t prev_size = memory.size(); + const size_t prev_size = Base::position() - memory.data(); memory.resize(2 * prev_size + 1); Base::set(memory.data() + prev_size, memory.size() - prev_size, 0); } diff --git a/src/IO/WriteBuffer.h b/src/IO/WriteBuffer.h index 53d0dbe09eb..c513b22b0a5 100644 --- a/src/IO/WriteBuffer.h +++ b/src/IO/WriteBuffer.h @@ -28,6 +28,7 @@ class WriteBuffer : public BufferBase { public: using BufferBase::set; + using BufferBase::position; WriteBuffer(Position ptr, size_t size) : BufferBase(ptr, size, 0) {} void set(Position ptr, size_t size) { BufferBase::set(ptr, size, 0); } diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 6f4b65a072a..69a8132c50c 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -21,6 +21,37 @@ namespace DB { +/** + * ORDER-PRESERVING parallel formatting of data formats. + * The idea is similar to ParallelParsingInputFormat. + * You add several Chunks through consume() method, each Chunk is formatted by some thread + * in ThreadPool into a temporary buffer. (Formatting is being done in parallel.) + * Then, another thread add temporary buffers into a "real" WriteBuffer. + * + * Formatters + * | | | | | | | | | | + * v v v v v v v v v v + * |---|---|---|---|---|---|---|---|---|---| + * | 1 | 2 | 3 | 4 | 5 | . | . | . | . | N | <-- Processing units + * |---|---|---|---|---|---|---|---|---|---| + * ^ ^ + * | | + * Collector addChunk + * + * There is a container of ProcessingUnits - internal entity, storing a Chunk to format, + * a continuous memory buffer to store the formatted Chunk and some flags for synchronization needs. + * Each ProcessingUnits has a unique number - the place in the container. + * So, Chunk is added through addChunk method, which waits until current ProcessingUnit would be ready to insert + * (ProcessingUnitStatus = READY_TO_INSERT), changes status to READY_TO_PARSE and spawns a new task in ThreadPool to parse it. + * The other thread, we call it Collector waits until a ProcessingUnit which it points to would be READY_TO_READ. + * Then it adds a temporary buffer to a real WriteBuffer. + * Both Collector and a thread which adds Chunks have unit_number - a pointer to ProcessingUnit which they are aim to work with. + * + * Note, that collector_unit_number is always less or equal to current_unit_number, that's why the formatting is order-preserving. + * + * To stop the execution, a fake Chunk is added (ProcessingUnitType = FINALIZE) and finalize() + * function is blocked until the Collector thread is done. +*/ class ParallelFormattingOutputFormat : public IOutputFormat { public: @@ -288,7 +319,7 @@ private: } } - + /// This function is executed in ThreadPool and the only purpose of it is to format one Chunk into a continuous buffer in memory. void formatterThreadFunction(size_t current_unit_number) { setThreadName("Formatter"); From dad2b0c35af858fb01ee11a248f9b0c3d238c1ad Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Wed, 30 Dec 2020 01:38:30 +0300 Subject: [PATCH 230/284] Small syntax fix --- docs/en/sql-reference/table-functions/mysql.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/table-functions/mysql.md b/docs/en/sql-reference/table-functions/mysql.md index 2338fc6db89..167d712e783 100644 --- a/docs/en/sql-reference/table-functions/mysql.md +++ b/docs/en/sql-reference/table-functions/mysql.md @@ -71,7 +71,7 @@ mysql> SELECT * FROM test; Selecting data from ClickHouse: ``` sql -SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123') +SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123'); ``` ``` text From eed9e8732c3bee4ae527af6c59c8c41cfb2570cb Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Wed, 30 Dec 2020 01:40:32 +0300 Subject: [PATCH 231/284] Another one --- docs/en/sql-reference/table-functions/mysql.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/table-functions/mysql.md b/docs/en/sql-reference/table-functions/mysql.md index 167d712e783..ed859e5ca79 100644 --- a/docs/en/sql-reference/table-functions/mysql.md +++ b/docs/en/sql-reference/table-functions/mysql.md @@ -33,7 +33,7 @@ mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_ Example: ```sql - INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1` + INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1; ``` `on_duplicate_clause` here is `UPDATE c2 = c2 + 1`. See the MySQL documentation to find which `on_duplicate_clause` you can use with the `ON DUPLICATE KEY` clause. From 8d63689b17d4cfd3c6b69f9478ec6fff0810fd13 Mon Sep 17 00:00:00 2001 From: ana-uvarova Date: Wed, 30 Dec 2020 02:53:44 +0300 Subject: [PATCH 232/284] content fix --- .../server-configuration-parameters/settings.md | 16 +++++++++++++++- docs/en/operations/system-tables/metric_log.md | 2 +- .../server-configuration-parameters/settings.md | 16 +++++++++++++++- docs/ru/operations/system-tables/metric_log.md | 2 +- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 78402e8e7a8..6fba98892f7 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -569,7 +569,11 @@ For more information, see the MergeTreeSettings.h header file. ## metric_log {#metric_log} -To turn on metrics history collection on [`system.metric_log`](../../operations/system-tables/metric_log.md), create `/etc/clickhouse-server/config.d/metric_log.xml` with following content: +It is enabled by default. If it`s not, you can do this manually. + +**Enabling** + +To manually turn on metrics history collection [`system.metric_log`](../../operations/system-tables/metric_log.md), create `/etc/clickhouse-server/config.d/metric_log.xml` with the following content: ``` xml @@ -582,6 +586,16 @@ To turn on metrics history collection on [`system.metric_log`](../../operations/ ``` +**Disabling** + +To disable `metric_log` setting, you should create the following file `/etc/clickhouse-server/config.d/disable_metric_log.xml` with the following content: + +``` xml + + + +``` + ## replicated_merge_tree {#server_configuration_parameters-replicated_merge_tree} Fine tuning for tables in the [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/mergetree.md). diff --git a/docs/en/operations/system-tables/metric_log.md b/docs/en/operations/system-tables/metric_log.md index 8cef5dc0931..1f72c9a7358 100644 --- a/docs/en/operations/system-tables/metric_log.md +++ b/docs/en/operations/system-tables/metric_log.md @@ -42,7 +42,7 @@ CurrentMetric_DistributedFilesToInsert: 0 **See also** -- [metric_log setting](../../operations/server-configuration-parameters/settings.md#metric_log) — What you should add. +- [metric_log setting](../../operations/server-configuration-parameters/settings.md#metric_log) — Enabling and disabling the setting. - [system.asynchronous_metrics](../../operations/system-tables/asynchronous_metrics.md) — Contains periodically calculated metrics. - [system.events](../../operations/system-tables/events.md#system_tables-events) — Contains a number of events that occurred. - [system.metrics](../../operations/system-tables/metrics.md) — Contains instantly calculated metrics. diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 25197b57c14..c858cf9e67b 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -558,7 +558,11 @@ ClickHouse проверяет условия для `min_part_size` и `min_part ## metric_log {#metric_log} -Чтобы включить сбор истории метрик в таблице [`system.metric_log`](../../operations/system-tables/metric_log.md), создайте `/etc/clickhouse-server/config.d/metric_log.xml` следующего содержания: +Эта настройка включена по умолчанию. Если это не так, вы можете включить ее сами. + +**Включение** + +Чтобы вручную включить сбор истории метрик в таблице [`system.metric_log`](../../operations/system-tables/metric_log.md), создайте `/etc/clickhouse-server/config.d/metric_log.xml` следующего содержания: ``` xml @@ -571,6 +575,16 @@ ClickHouse проверяет условия для `min_part_size` и `min_part ``` +**Выключение** + +Чтобы отключить настройку `metric_log` , создайте файл `/etc/clickhouse-server/config.d/disable_metric_log.xml` следующего содержания: + +``` xml + + + +``` + ## replicated\_merge\_tree {#server_configuration_parameters-replicated_merge_tree} Тонкая настройка таблиц в [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/mergetree.md). diff --git a/docs/ru/operations/system-tables/metric_log.md b/docs/ru/operations/system-tables/metric_log.md index 2bd23dd2c61..2458c93da59 100644 --- a/docs/ru/operations/system-tables/metric_log.md +++ b/docs/ru/operations/system-tables/metric_log.md @@ -42,7 +42,7 @@ CurrentMetric_ReplicatedChecks: 0 **Смотрите также** -- [Настройка metric_log](../../operations/server-configuration-parameters/settings.md#metric_log) — то, что вы должны добавить для записи истории. +- [Настройка metric_log](../../operations/server-configuration-parameters/settings.md#metric_log) — как включить и выключить запись истории. - [system.asynchronous_metrics](#system_tables-asynchronous_metrics) — таблица с периодически вычисляемыми метриками. - [system.events](#system_tables-events) — таблица с количеством произошедших событий. - [system.metrics](#system_tables-metrics) — таблица с мгновенно вычисляемыми метриками. From c5f92e50969f07c3e9d546c8e144836e596fbdc2 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 30 Dec 2020 06:07:30 +0300 Subject: [PATCH 233/284] better formatfactory --- programs/client/Client.cpp | 10 +- programs/obfuscator/Obfuscator.cpp | 2 +- programs/odbc-bridge/MainHandler.cpp | 2 +- .../ExecutableDictionarySource.cpp | 4 +- src/Dictionaries/HTTPDictionarySource.cpp | 4 +- src/Formats/FormatFactory.cpp | 73 ++++-- src/Formats/FormatFactory.h | 14 +- src/Interpreters/Context.cpp | 15 +- src/Interpreters/Context.h | 8 +- src/Interpreters/executeQuery.cpp | 7 +- .../Impl/ParallelFormattingOutputFormat.cpp | 201 ++++++++++++++++ .../Impl/ParallelFormattingOutputFormat.h | 217 ++---------------- src/Server/GRPCServer.cpp | 8 +- src/Storages/HDFS/StorageHDFS.cpp | 2 +- src/Storages/Kafka/KafkaBlockOutputStream.cpp | 2 +- .../RabbitMQ/RabbitMQBlockOutputStream.cpp | 2 +- src/Storages/StorageFile.cpp | 4 +- src/Storages/StorageMySQL.cpp | 2 +- src/Storages/StorageS3.cpp | 2 +- src/Storages/StorageURL.cpp | 2 +- src/Storages/tests/gtest_storage_log.cpp | 2 +- 21 files changed, 322 insertions(+), 261 deletions(-) create mode 100644 src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 328855adcc0..f01b4bf6d81 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1934,11 +1934,11 @@ private: if (has_vertical_output_suffix) current_format = "Vertical"; - if (!is_interactive && !need_render_progress) - block_out_stream = context.getOutputFormatParallelIfPossible(current_format, *out_buf, block); - - if (!block_out_stream) - block_out_stream = context.getOutputFormat(current_format, *out_buf, block); + /// It is not clear how to write progress with parallel formatting. It may increase code complexity significantly. + if (!need_render_progress) + block_out_stream = context.getOutputStreamParallelIfPossible(current_format, *out_buf, block); + else + block_out_stream = context.getOutputStream(current_format, *out_buf, block); block_out_stream->writePrefix(); } diff --git a/programs/obfuscator/Obfuscator.cpp b/programs/obfuscator/Obfuscator.cpp index d0d7f201c68..950db4e4f05 100644 --- a/programs/obfuscator/Obfuscator.cpp +++ b/programs/obfuscator/Obfuscator.cpp @@ -1180,7 +1180,7 @@ try file_in.seek(0, SEEK_SET); BlockInputStreamPtr input = context.getInputFormat(input_format, file_in, header, max_block_size); - BlockOutputStreamPtr output = context.getOutputFormat(output_format, file_out, header); + BlockOutputStreamPtr output = context.getOutputStream(output_format, file_out, header); if (processed_rows + source_rows > limit) input = std::make_shared(input, limit - processed_rows, 0); diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index c456c354dda..64cb7bc0b46 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -176,7 +176,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne std::string query = params.get("query"); LOG_TRACE(log, "Query: {}", query); - BlockOutputStreamPtr writer = FormatFactory::instance().getOutput(format, out, *sample_block, context); + BlockOutputStreamPtr writer = FormatFactory::instance().getOutputStream(format, out, *sample_block, context); auto pool = getPool(connection_string); ODBCBlockInputStream inp(pool->get(), query, *sample_block, max_block_size); copyData(inp, *writer); diff --git a/src/Dictionaries/ExecutableDictionarySource.cpp b/src/Dictionaries/ExecutableDictionarySource.cpp index 48699cc1104..f2abe10f970 100644 --- a/src/Dictionaries/ExecutableDictionarySource.cpp +++ b/src/Dictionaries/ExecutableDictionarySource.cpp @@ -184,7 +184,7 @@ BlockInputStreamPtr ExecutableDictionarySource::loadIds(const std::vector & id ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = [&](std::ostream & ostr) { WriteBufferFromOStream out_buffer(ostr); - auto output_stream = context.getOutputFormat(format, out_buffer, sample_block); + auto output_stream = context.getOutputStream(format, out_buffer, sample_block); formatIDs(output_stream, ids); }; @@ -153,7 +153,7 @@ BlockInputStreamPtr HTTPDictionarySource::loadKeys(const Columns & key_columns, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = [&](std::ostream & ostr) { WriteBufferFromOStream out_buffer(ostr); - auto output_stream = context.getOutputFormat(format, out_buffer, sample_block); + auto output_stream = context.getOutputStream(format, out_buffer, sample_block); formatKeys(dict_struct, output_stream, key_columns, requested_rows); }; diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index cfcbd9fb1c1..05e56e2aa4a 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -202,28 +202,20 @@ InputFormatPtr FormatFactory::getInput( return format; } -BlockOutputStreamPtr FormatFactory::getOutputParallelIfPossible(const String & name, +BlockOutputStreamPtr FormatFactory::getOutputStreamParallelIfPossible(const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback, const std::optional & _format_settings) const { - auto format_settings = _format_settings - ? *_format_settings : getFormatSettings(context); - - if (!getCreators(name).output_processor_creator) - { - throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); - } + const auto & output_getter = getCreators(name).output_processor_creator; const Settings & settings = context.getSettingsRef(); bool parallel_formatting = settings.output_format_parallel_formatting; - if (parallel_formatting && getCreators(name).supports_parallel_formatting && !settings.allow_experimental_live_view && !settings.output_format_json_array_of_rows) + if (output_getter && parallel_formatting && getCreators(name).supports_parallel_formatting + && !settings.output_format_json_array_of_rows && !settings.allow_experimental_live_view) { - const auto & output_getter = getCreators(name).output_processor_creator; - - /** TODO: Materialization is needed, because formats can use the functions `IDataType`, - * which only work with full columns. - */ + auto format_settings = _format_settings + ? *_format_settings : getFormatSettings(context); auto formatter_creator = [output_getter, sample, callback, format_settings] (WriteBuffer & output) -> OutputFormatPtr @@ -239,12 +231,11 @@ BlockOutputStreamPtr FormatFactory::getOutputParallelIfPossible(const String & n return std::make_shared(std::make_shared(format), sample); } - - return nullptr; + return getOutputStream(name, buf, sample, context, callback, _format_settings); } -BlockOutputStreamPtr FormatFactory::getOutput(const String & name, +BlockOutputStreamPtr FormatFactory::getOutputStream(const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback, const std::optional & _format_settings) const { @@ -253,10 +244,19 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, if (!getCreators(name).output_processor_creator) { - throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); + const auto & output_getter = getCreators(name).output_creator; + if (!output_getter) + throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); + + /** Materialization is needed, because formats can use the functions `IDataType`, + * which only work with full columns. + */ + return std::make_shared( + output_getter(buf, sample, std::move(callback), format_settings), + sample); } - auto format = getOutputFormat(name, buf, sample, context, std::move(callback), format_settings); + auto format = getOutputFormat(name, buf, sample, context, std::move(callback), _format_settings); return std::make_shared(std::make_shared(format), sample); } @@ -294,6 +294,39 @@ InputFormatPtr FormatFactory::getInputFormat( return format; } +OutputFormatPtr FormatFactory::getOutputFormatParallelIfPossible( + const String & name, WriteBuffer & buf, const Block & sample, + const Context & context, WriteCallback callback, + const std::optional & _format_settings) const +{ + const auto & output_getter = getCreators(name).output_processor_creator; + if (!output_getter) + throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); + + RowOutputFormatParams params; + params.callback = std::move(callback); + + auto format_settings = _format_settings + ? *_format_settings : getFormatSettings(context); + + const Settings & settings = context.getSettingsRef(); + + if (settings.output_format_parallel_formatting && getCreators(name).supports_parallel_formatting + && !settings.output_format_json_array_of_rows && !settings.allow_experimental_live_view) + { + auto formatter_creator = [output_getter, sample, callback, format_settings] + (WriteBuffer & output) -> OutputFormatPtr + { return output_getter(output, sample, {std::move(callback)}, format_settings);}; + + ParallelFormattingOutputFormat::Params builder{buf, sample, formatter_creator, settings.max_threads}; + + return std::make_shared(builder); + } + + return getOutputFormat(name, buf, sample, context, callback, _format_settings); +} + + OutputFormatPtr FormatFactory::getOutputFormat( const String & name, WriteBuffer & buf, const Block & sample, @@ -302,7 +335,7 @@ OutputFormatPtr FormatFactory::getOutputFormat( { const auto & output_getter = getCreators(name).output_processor_creator; if (!output_getter) - throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); + throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); RowOutputFormatParams params; params.callback = std::move(callback); diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index 89859152135..c4d14cc930c 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -116,12 +116,14 @@ public: UInt64 max_block_size, const std::optional & format_settings = std::nullopt) const; - /// Checks all preconditions. Returns nullptr of parallel formatting cannot be done. - BlockOutputStreamPtr getOutputParallelIfPossible(const String & name, WriteBuffer & buf, + /// Checks all preconditions. Returns ordinary stream if parallel formatting cannot be done. + /// Currenly used only in Client. Don't use it something else! Better look at getOutputFormatParallelIfPossible. + BlockOutputStreamPtr getOutputStreamParallelIfPossible(const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback = {}, const std::optional & format_settings = std::nullopt) const; - BlockOutputStreamPtr getOutput(const String & name, WriteBuffer & buf, + /// Currenly used only in Client. Don't use it something else! Better look at getOutputFormat. + BlockOutputStreamPtr getOutputStream(const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback = {}, const std::optional & format_settings = std::nullopt) const; @@ -133,6 +135,12 @@ public: UInt64 max_block_size, const std::optional & format_settings = std::nullopt) const; + /// Checks all preconditions. Returns ordinary format if parallel formatting cannot be done. + OutputFormatPtr getOutputFormatParallelIfPossible( + const String & name, WriteBuffer & buf, const Block & sample, + const Context & context, WriteCallback callback = {}, + const std::optional & format_settings = std::nullopt) const; + OutputFormatPtr getOutputFormat( const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback = {}, diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index c3f841a487a..a5d75867889 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -2089,17 +2089,22 @@ BlockInputStreamPtr Context::getInputFormat(const String & name, ReadBuffer & bu return std::make_shared(FormatFactory::instance().getInput(name, buf, sample, *this, max_block_size)); } -BlockOutputStreamPtr Context::getOutputFormatParallelIfPossible(const String & name, WriteBuffer & buf, const Block & sample) const +BlockOutputStreamPtr Context::getOutputStreamParallelIfPossible(const String & name, WriteBuffer & buf, const Block & sample) const { - return FormatFactory::instance().getOutputParallelIfPossible(name, buf, sample, *this); + return FormatFactory::instance().getOutputStreamParallelIfPossible(name, buf, sample, *this); } -BlockOutputStreamPtr Context::getOutputFormat(const String & name, WriteBuffer & buf, const Block & sample) const +BlockOutputStreamPtr Context::getOutputStream(const String & name, WriteBuffer & buf, const Block & sample) const { - return FormatFactory::instance().getOutput(name, buf, sample, *this); + return FormatFactory::instance().getOutputStream(name, buf, sample, *this); } -OutputFormatPtr Context::getOutputFormatProcessor(const String & name, WriteBuffer & buf, const Block & sample) const +OutputFormatPtr Context::getOutputFormatParallelIfPossible(const String & name, WriteBuffer & buf, const Block & sample) const +{ + return FormatFactory::instance().getOutputFormatParallelIfPossible(name, buf, sample, *this); +} + +OutputFormatPtr Context::getOutputFormat(const String & name, WriteBuffer & buf, const Block & sample) const { return FormatFactory::instance().getOutputFormat(name, buf, sample, *this); } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 3bc21a3c1e2..79140f0d209 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -426,10 +426,12 @@ public: /// I/O formats. BlockInputStreamPtr getInputFormat(const String & name, ReadBuffer & buf, const Block & sample, UInt64 max_block_size) const; - BlockOutputStreamPtr getOutputFormatParallelIfPossible(const String & name, WriteBuffer & buf, const Block & sample) const; - BlockOutputStreamPtr getOutputFormat(const String & name, WriteBuffer & buf, const Block & sample) const; + /// Don't use streams. Better look at getOutputFormat... + BlockOutputStreamPtr getOutputStreamParallelIfPossible(const String & name, WriteBuffer & buf, const Block & sample) const; + BlockOutputStreamPtr getOutputStream(const String & name, WriteBuffer & buf, const Block & sample) const; - OutputFormatPtr getOutputFormatProcessor(const String & name, WriteBuffer & buf, const Block & sample) const; + OutputFormatPtr getOutputFormatParallelIfPossible(const String & name, WriteBuffer & buf, const Block & sample) const; + OutputFormatPtr getOutputFormat(const String & name, WriteBuffer & buf, const Block & sample) const; InterserverIOHandler & getInterserverIOHandler(); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index f7a930b66fe..6b376947e65 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -974,10 +974,7 @@ void executeQuery( ? getIdentifierName(ast_query_with_output->format) : context.getDefaultFormat(); - BlockOutputStreamPtr out; - out = context.getOutputFormatParallelIfPossible(format_name, *out_buf, streams.in->getHeader()); - if (!out) - out = context.getOutputFormat(format_name, *out_buf, streams.in->getHeader()); + auto out = context.getOutputStream(format_name, *out_buf, streams.in->getHeader()); /// Save previous progress callback if any. TODO Do it more conveniently. auto previous_progress_callback = context.getProgressCallback(); @@ -1022,7 +1019,7 @@ void executeQuery( return std::make_shared(header); }); - auto out = context.getOutputFormatProcessor(format_name, *out_buf, pipeline.getHeader()); + auto out = context.getOutputFormat(format_name, *out_buf, pipeline.getHeader()); out->setAutoFlush(); /// Save previous progress callback if any. TODO Do it more conveniently. diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp new file mode 100644 index 00000000000..eda2665119a --- /dev/null +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp @@ -0,0 +1,201 @@ +#include + +#include + +namespace DB +{ + void ParallelFormattingOutputFormat::finalize() + { + need_flush = true; + IOutputFormat::finalized = true; + /// Don't throw any background_exception here, because we want to finalize the execution. + /// Exception will be checked after main thread is finished. + addChunk(Chunk{}, ProcessingUnitType::FINALIZE, /*can_throw_exception*/ false); + collector_finished.wait(); + + if (collector_thread.joinable()) + collector_thread.join(); + + { + std::unique_lock lock(mutex); + if (background_exception) + std::rethrow_exception(background_exception); + } + } + + void ParallelFormattingOutputFormat::addChunk(Chunk chunk, ProcessingUnitType type, bool can_throw_exception) + { + { + std::unique_lock lock(mutex); + if (background_exception && can_throw_exception) + std::rethrow_exception(background_exception); + } + + const auto current_unit_number = writer_unit_number % processing_units.size(); + auto & unit = processing_units[current_unit_number]; + + { + std::unique_lock lock(mutex); + writer_condvar.wait(lock, + [&]{ return unit.status == READY_TO_INSERT || emergency_stop; }); + } + + if (emergency_stop) + return; + + assert(unit.status == READY_TO_INSERT); + unit.chunk = std::move(chunk); + /// Resize memory without deallocation. + unit.segment.resize(0); + unit.status = READY_TO_FORMAT; + unit.type = type; + + scheduleFormatterThreadForUnitWithNumber(current_unit_number); + + ++writer_unit_number; + } + + + void ParallelFormattingOutputFormat::finishAndWait() + { + emergency_stop = true; + + { + std::unique_lock lock(mutex); + collector_condvar.notify_all(); + writer_condvar.notify_all(); + } + + if (collector_thread.joinable()) + collector_thread.join(); + + try + { + pool.wait(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + + + void ParallelFormattingOutputFormat::collectorThreadFunction() + { + setThreadName("Collector"); + + try + { + while (!emergency_stop) + { + const auto current_unit_number = collector_unit_number % processing_units.size(); + auto & unit = processing_units[current_unit_number]; + + { + std::unique_lock lock(mutex); + collector_condvar.wait(lock, + [&]{ return unit.status == READY_TO_READ || emergency_stop; }); + } + + if (emergency_stop) + break; + + assert(unit.status == READY_TO_READ); + + /// Use this copy to after notification to stop the execution. + auto copy_if_unit_type = unit.type; + + /// Do main work here. + out.write(unit.segment.data(), unit.actual_memory_size); + + if (need_flush.exchange(false) || auto_flush) + IOutputFormat::flush(); + + ++collector_unit_number; + + { + /// Notify other threads. + std::lock_guard lock(mutex); + unit.status = READY_TO_INSERT; + writer_condvar.notify_all(); + } + /// We can exit only after writing last piece of to out buffer. + if (copy_if_unit_type == ProcessingUnitType::FINALIZE) + { + break; + } + } + collector_finished.set(); + } + catch (...) + { + collector_finished.set(); + onBackgroundException(); + } + } + + + void ParallelFormattingOutputFormat::formatterThreadFunction(size_t current_unit_number) + { + setThreadName("Formatter"); + + try + { + auto & unit = processing_units[current_unit_number]; + assert(unit.status = READY_TO_FORMAT); + + /// We want to preallocate memory buffer (increase capacity) + /// and put the pointer at the beginning of the buffer + unit.segment.resize(DBMS_DEFAULT_BUFFER_SIZE); + /// The second invocation won't release memory, only set size equals to 0. + unit.segment.resize(0); + + unit.actual_memory_size = 0; + BufferWithOutsideMemory out_buffer(unit.segment); + + auto formatter = internal_formatter_creator(out_buffer); + + switch (unit.type) + { + case ProcessingUnitType::START : + { + formatter->doWritePrefix(); + break; + } + case ProcessingUnitType::PLAIN : + { + formatter->consume(std::move(unit.chunk)); + break; + } + case ProcessingUnitType::TOTALS : + { + formatter->consumeTotals(std::move(unit.chunk)); + break; + } + case ProcessingUnitType::EXTREMES : + { + formatter->consumeExtremes(std::move(unit.chunk)); + break; + } + case ProcessingUnitType::FINALIZE : + { + formatter->doWriteSuffix(); + break; + } + } + /// Flush all the data to handmade buffer. + formatter->flush(); + unit.actual_memory_size = out_buffer.getActualSize(); + + { + std::lock_guard lock(mutex); + unit.status = READY_TO_READ; + collector_condvar.notify_all(); + } + } + catch (...) + { + onBackgroundException(); + } + } +} diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 69a8132c50c..da89488bd28 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -1,22 +1,19 @@ #pragma once #include + #include -#include -#include - -#include -#include -#include - -#include #include -#include -#include -#include - #include #include +#include +#include +#include +#include +#include + +#include +#include namespace DB { @@ -79,7 +76,7 @@ public: /// and n threads for formatting. processing_units.resize(params.max_threads_for_parallel_formatting + 2); collector_thread = ThreadFromGlobalPool([&] { collectorThreadFunction(); }); - LOG_TRACE(&Poco::Logger::get("ParallelFormattingOutputFormat"), "Parallel formatting is being used."); + LOG_TRACE(&Poco::Logger::get("ParallelFormattingOutputFormat"), "Parallel formatting is being used"); } ~ParallelFormattingOutputFormat() override @@ -120,24 +117,7 @@ protected: addChunk(std::move(extremes), ProcessingUnitType::EXTREMES, /*can_throw_exception*/ true); } - void finalize() override - { - need_flush = true; - IOutputFormat::finalized = true; - /// Don't throw any background_exception here, because we want to finalize the execution. - /// Exception will be checked after main thread is finished. - addChunk(Chunk{}, ProcessingUnitType::FINALIZE, /*can_throw_exception*/ false); - collector_finished.wait(); - - if (collector_thread.joinable()) - collector_thread.join(); - - { - std::unique_lock lock(mutex); - if (background_exception) - std::rethrow_exception(background_exception); - } - } + void finalize() override; private: InternalFormatterCreator internal_formatter_creator; @@ -160,37 +140,7 @@ private: FINALIZE }; - void addChunk(Chunk chunk, ProcessingUnitType type, bool can_throw_exception) - { - { - std::unique_lock lock(mutex); - if (background_exception && can_throw_exception) - std::rethrow_exception(background_exception); - } - - const auto current_unit_number = writer_unit_number % processing_units.size(); - auto & unit = processing_units[current_unit_number]; - - { - std::unique_lock lock(mutex); - writer_condvar.wait(lock, - [&]{ return unit.status == READY_TO_INSERT || emergency_stop; }); - } - - if (emergency_stop) - return; - - assert(unit.status == READY_TO_INSERT); - unit.chunk = std::move(chunk); - /// Resize memory without deallocation. - unit.segment.resize(0); - unit.status = READY_TO_FORMAT; - unit.type = type; - - scheduleFormatterThreadForUnitWithNumber(current_unit_number); - - ++writer_unit_number; - } + void addChunk(Chunk chunk, ProcessingUnitType type, bool can_throw_exception); struct ProcessingUnit { @@ -224,29 +174,7 @@ private: std::condition_variable collector_condvar; std::condition_variable writer_condvar; - void finishAndWait() - { - emergency_stop = true; - - { - std::unique_lock lock(mutex); - collector_condvar.notify_all(); - writer_condvar.notify_all(); - } - - if (collector_thread.joinable()) - collector_thread.join(); - - try - { - pool.wait(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } - + void finishAndWait(); void onBackgroundException() { @@ -265,124 +193,11 @@ private: pool.scheduleOrThrowOnError([this, ticket_number] { formatterThreadFunction(ticket_number); }); } - void collectorThreadFunction() - { - setThreadName("Collector"); - - try - { - while (!emergency_stop) - { - const auto current_unit_number = collector_unit_number % processing_units.size(); - auto & unit = processing_units[current_unit_number]; - - { - std::unique_lock lock(mutex); - collector_condvar.wait(lock, - [&]{ return unit.status == READY_TO_READ || emergency_stop; }); - } - - if (emergency_stop) - break; - - assert(unit.status == READY_TO_READ); - - /// Use this copy to after notification to stop the execution. - auto copy_if_unit_type = unit.type; - - /// Do main work here. - out.write(unit.segment.data(), unit.actual_memory_size); - - if (need_flush.exchange(false) || auto_flush) - IOutputFormat::flush(); - - ++collector_unit_number; - - { - /// Notify other threads. - std::lock_guard lock(mutex); - unit.status = READY_TO_INSERT; - writer_condvar.notify_all(); - } - /// We can exit only after writing last piece of to out buffer. - if (copy_if_unit_type == ProcessingUnitType::FINALIZE) - { - break; - } - } - collector_finished.set(); - } - catch (...) - { - collector_finished.set(); - onBackgroundException(); - } - } + /// Collects all temporary buffers into main WriteBuffer. + void collectorThreadFunction(); /// This function is executed in ThreadPool and the only purpose of it is to format one Chunk into a continuous buffer in memory. - void formatterThreadFunction(size_t current_unit_number) - { - setThreadName("Formatter"); - - try - { - auto & unit = processing_units[current_unit_number]; - assert(unit.status = READY_TO_FORMAT); - - /// We want to preallocate memory buffer (increase capacity) - /// and put the pointer at the beginning of the buffer - /// FIXME: Implement reserve() method in Memory. - unit.segment.resize(DBMS_DEFAULT_BUFFER_SIZE); - unit.segment.resize(0); - - unit.actual_memory_size = 0; - BufferWithOutsideMemory out_buffer(unit.segment); - - auto formatter = internal_formatter_creator(out_buffer); - - switch (unit.type) - { - case ProcessingUnitType::START : - { - formatter->doWritePrefix(); - break; - } - case ProcessingUnitType::PLAIN : - { - formatter->consume(std::move(unit.chunk)); - break; - } - case ProcessingUnitType::TOTALS : - { - formatter->consumeTotals(std::move(unit.chunk)); - break; - } - case ProcessingUnitType::EXTREMES : - { - formatter->consumeExtremes(std::move(unit.chunk)); - break; - } - case ProcessingUnitType::FINALIZE : - { - formatter->doWriteSuffix(); - break; - } - } - /// Flush all the data to handmade buffer. - formatter->flush(); - unit.actual_memory_size = out_buffer.getActualSize(); - - { - std::lock_guard lock(mutex); - unit.status = READY_TO_READ; - collector_condvar.notify_all(); - } - } - catch (...) - { - onBackgroundException(); - } - } + void formatterThreadFunction(size_t current_unit_number); }; } diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index bdf6d3ba5d7..475bfc81801 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -994,7 +994,7 @@ namespace AsynchronousBlockInputStream async_in(io.in); write_buffer.emplace(*result.mutable_output()); - block_output_stream = query_context->getOutputFormat(output_format, *write_buffer, async_in.getHeader()); + block_output_stream = query_context->getOutputStream(output_format, *write_buffer, async_in.getHeader()); Stopwatch after_send_progress; /// Unless the input() function is used we are not going to receive input data anymore. @@ -1066,7 +1066,7 @@ namespace auto executor = std::make_shared(io.pipeline); write_buffer.emplace(*result.mutable_output()); - block_output_stream = query_context->getOutputFormat(output_format, *write_buffer, executor->getHeader()); + block_output_stream = query_context->getOutputStream(output_format, *write_buffer, executor->getHeader()); block_output_stream->writePrefix(); Stopwatch after_send_progress; @@ -1321,7 +1321,7 @@ namespace return; WriteBufferFromString buf{*result.mutable_totals()}; - auto stream = query_context->getOutputFormat(output_format, buf, totals); + auto stream = query_context->getOutputStream(output_format, buf, totals); stream->writePrefix(); stream->write(totals); stream->writeSuffix(); @@ -1333,7 +1333,7 @@ namespace return; WriteBufferFromString buf{*result.mutable_extremes()}; - auto stream = query_context->getOutputFormat(output_format, buf, extremes); + auto stream = query_context->getOutputStream(output_format, buf, extremes); stream->writePrefix(); stream->write(extremes); stream->writeSuffix(); diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 3cc7ac6e082..fe165e8a8a8 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -184,7 +184,7 @@ public: : sample_block(sample_block_) { write_buf = wrapWriteBufferWithCompressionMethod(std::make_unique(uri, context.getGlobalContext().getConfigRef()), compression_method, 3); - writer = FormatFactory::instance().getOutput(format, *write_buf, sample_block, context); + writer = FormatFactory::instance().getOutputStream(format, *write_buf, sample_block, context); } Block getHeader() const override diff --git a/src/Storages/Kafka/KafkaBlockOutputStream.cpp b/src/Storages/Kafka/KafkaBlockOutputStream.cpp index dc5b5283cdc..e1742741670 100644 --- a/src/Storages/Kafka/KafkaBlockOutputStream.cpp +++ b/src/Storages/Kafka/KafkaBlockOutputStream.cpp @@ -35,7 +35,7 @@ void KafkaBlockOutputStream::writePrefix() auto format_settings = getFormatSettings(*context); format_settings.protobuf.allow_many_rows_no_delimiters = true; - child = FormatFactory::instance().getOutput(storage.getFormatName(), *buffer, + child = FormatFactory::instance().getOutputStream(storage.getFormatName(), *buffer, getHeader(), *context, [this](const Columns & columns, size_t row) { diff --git a/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp b/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp index b3bd57bdd0b..289f0c61b7d 100644 --- a/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp +++ b/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp @@ -45,7 +45,7 @@ void RabbitMQBlockOutputStream::writePrefix() auto format_settings = getFormatSettings(context); format_settings.protobuf.allow_many_rows_no_delimiters = true; - child = FormatFactory::instance().getOutput(storage.getFormatName(), *buffer, + child = FormatFactory::instance().getOutputStream(storage.getFormatName(), *buffer, getHeader(), context, [this](const Columns & /* columns */, size_t /* rows */) { diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 1469b535b51..45073ccd35a 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -481,12 +481,12 @@ public: write_buf = wrapWriteBufferWithCompressionMethod(std::move(naked_buffer), compression_method, 3); - writer = FormatFactory::instance().getOutputParallelIfPossible(storage.format_name, + writer = FormatFactory::instance().getOutputStreamParallelIfPossible(storage.format_name, *write_buf, metadata_snapshot->getSampleBlock(), context, {}, format_settings); if (!writer) - writer = FormatFactory::instance().getOutput(storage.format_name, + writer = FormatFactory::instance().getOutputStream(storage.format_name, *write_buf, metadata_snapshot->getSampleBlock(), context, {}, format_settings); } diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index defac1b57cf..caac7c5d95e 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -147,7 +147,7 @@ public: sqlbuf << backQuoteMySQL(remote_database_name) << "." << backQuoteMySQL(remote_table_name); sqlbuf << " (" << dumpNamesWithBackQuote(block) << ") VALUES "; - auto writer = FormatFactory::instance().getOutput("Values", sqlbuf, metadata_snapshot->getSampleBlock(), storage.global_context); + auto writer = FormatFactory::instance().getOutputStream("Values", sqlbuf, metadata_snapshot->getSampleBlock(), storage.global_context); writer->write(block); if (!storage.on_duplicate_clause.empty()) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 380c98264e0..5d7fc0cdaa9 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -155,7 +155,7 @@ namespace { write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique(client, bucket, key, min_upload_part_size, max_single_part_upload_size), compression_method, 3); - writer = FormatFactory::instance().getOutput(format, *write_buf, sample_block, context); + writer = FormatFactory::instance().getOutputStream(format, *write_buf, sample_block, context); } Block getHeader() const override diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 8c69d9c0179..ddbab99c3f0 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -156,7 +156,7 @@ StorageURLBlockOutputStream::StorageURLBlockOutputStream(const Poco::URI & uri, write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique(uri, Poco::Net::HTTPRequest::HTTP_POST, timeouts), compression_method, 3); - writer = FormatFactory::instance().getOutput(format, *write_buf, sample_block, + writer = FormatFactory::instance().getOutputStream(format, *write_buf, sample_block, context, {} /* write callback */, format_settings); } diff --git a/src/Storages/tests/gtest_storage_log.cpp b/src/Storages/tests/gtest_storage_log.cpp index 0fb418e8413..cbb894c7420 100644 --- a/src/Storages/tests/gtest_storage_log.cpp +++ b/src/Storages/tests/gtest_storage_log.cpp @@ -134,7 +134,7 @@ std::string readData(DB::StoragePtr & table, const DB::Context & context) tryRegisterFormats(); WriteBufferFromOwnString out_buf; - BlockOutputStreamPtr output = FormatFactory::instance().getOutput("Values", out_buf, sample, context); + BlockOutputStreamPtr output = FormatFactory::instance().getOutputStream("Values", out_buf, sample, context); copyData(*in, *output); From a2e5f7693acafe901a3e2dee35513c66f61478bf Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Wed, 30 Dec 2020 11:20:38 +0800 Subject: [PATCH 234/284] Fix condition error --- base/common/DateLUTImpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 1d82f668a49..3698276d0e1 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -674,7 +674,7 @@ public: return DayNum(0); // TODO (nemkov, DateTime64 phase 2): implement creating real date for year outside of LUT range. // The day after 2106-02-07 will not stored fully as struct Values, so just overflow it as 0 - if (unlikely(year == DATE_LUT_MAX_YEAR && (month > 2 || day_of_month > 7))) + if (unlikely(year == DATE_LUT_MAX_YEAR && (month > 2 || (month == 2 && day_of_month > 7)))) return DayNum(0); return DayNum(years_months_lut[(year - DATE_LUT_MIN_YEAR) * 12 + month - 1] + day_of_month - 1); From 60b4a36c4ad682111b95da57810ddd22a94236ee Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 30 Dec 2020 07:50:58 +0300 Subject: [PATCH 235/284] arcadia fix + live view fix + cleanup --- src/Formats/FormatFactory.cpp | 12 +++++++----- src/Processors/Formats/IInputFormat.h | 7 +++++++ .../Formats/Impl/JSONEachRowRowInputFormat.cpp | 4 ++-- .../Formats/Impl/JSONEachRowRowInputFormat.h | 1 + .../Formats/Impl/ParallelFormattingOutputFormat.h | 6 +++--- .../Formats/Impl/ParallelParsingInputFormat.cpp | 1 + .../Formats/Impl/ParallelParsingInputFormat.h | 13 ++++++++++--- src/Processors/ya.make | 1 + src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/StorageFile.cpp | 1 - .../01246_insert_into_watch_live_view.py | 7 ------- tests/queries/0_stateless/helpers/client.py | 2 +- 12 files changed, 34 insertions(+), 23 deletions(-) diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 05e56e2aa4a..ffa8ccfcced 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -165,6 +165,9 @@ InputFormatPtr FormatFactory::getInput( if (settings.max_memory_usage && settings.min_chunk_bytes_for_parallel_parsing * settings.max_threads * 2 > settings.max_memory_usage) parallel_parsing = false; + if (settings.max_memory_usage_for_user && settings.min_chunk_bytes_for_parallel_parsing * settings.max_threads * 2 > settings.max_memory_usage_for_user) + parallel_parsing = false; + if (parallel_parsing && name == "JSONEachRow") { /// FIXME ParallelParsingBlockInputStream doesn't support formats with non-trivial readPrefix() and readSuffix() @@ -211,8 +214,8 @@ BlockOutputStreamPtr FormatFactory::getOutputStreamParallelIfPossible(const Stri const Settings & settings = context.getSettingsRef(); bool parallel_formatting = settings.output_format_parallel_formatting; - if (output_getter && parallel_formatting && getCreators(name).supports_parallel_formatting - && !settings.output_format_json_array_of_rows && !settings.allow_experimental_live_view) + if (output_getter && parallel_formatting && getCreators(name).supports_parallel_formatting + && !settings.output_format_json_array_of_rows) { auto format_settings = _format_settings ? *_format_settings : getFormatSettings(context); @@ -311,8 +314,8 @@ OutputFormatPtr FormatFactory::getOutputFormatParallelIfPossible( const Settings & settings = context.getSettingsRef(); - if (settings.output_format_parallel_formatting && getCreators(name).supports_parallel_formatting - && !settings.output_format_json_array_of_rows && !settings.allow_experimental_live_view) + if (settings.output_format_parallel_formatting && getCreators(name).supports_parallel_formatting + && !settings.output_format_json_array_of_rows) { auto formatter_creator = [output_getter, sample, callback, format_settings] (WriteBuffer & output) -> OutputFormatPtr @@ -327,7 +330,6 @@ OutputFormatPtr FormatFactory::getOutputFormatParallelIfPossible( } - OutputFormatPtr FormatFactory::getOutputFormat( const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback, diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index 00cb38405cf..e1537aff6c5 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -38,6 +38,13 @@ public: static const BlockMissingValues none; return none; } + + size_t getCurrentUnitNumber() const { return current_unit_number; } + void setCurrentUnitNumber(size_t current_unit_number_) { current_unit_number = current_unit_number_; } + +private: + /// Number of currently parsed chunk (if parallel parsing is enabled) + size_t current_unit_number = 0; }; } diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp index bd22718b57e..8a707ae6554 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp @@ -245,7 +245,7 @@ bool JSONEachRowRowInputFormat::readRow(MutableColumns & columns, RowReadExtensi /// then seeking to next ;, or \n would trigger reading of an extra row at the end. /// Semicolon is added for convenience as it could be used at end of INSERT query. - bool is_first_row = getTotalRows() == 1; + bool is_first_row = getCurrentUnitNumber() == 0 && getTotalRows() == 1; if (!in.eof()) { /// There may be optional ',' (but not before the first row) @@ -253,8 +253,8 @@ bool JSONEachRowRowInputFormat::readRow(MutableColumns & columns, RowReadExtensi ++in.position(); else if (!data_in_square_brackets && *in.position() == ';') { - return allow_new_rows = false; /// ';' means the end of query (but it cannot be before ']') + return allow_new_rows = false; } else if (data_in_square_brackets && *in.position() == ']') { diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h index 4d6c92077b2..29a6ce6ecb8 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h @@ -5,6 +5,7 @@ #include #include + namespace DB { diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index da89488bd28..7e7c44a8aae 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -34,7 +34,7 @@ namespace DB * ^ ^ * | | * Collector addChunk - * + * * There is a container of ProcessingUnits - internal entity, storing a Chunk to format, * a continuous memory buffer to store the formatted Chunk and some flags for synchronization needs. * Each ProcessingUnits has a unique number - the place in the container. @@ -43,9 +43,9 @@ namespace DB * The other thread, we call it Collector waits until a ProcessingUnit which it points to would be READY_TO_READ. * Then it adds a temporary buffer to a real WriteBuffer. * Both Collector and a thread which adds Chunks have unit_number - a pointer to ProcessingUnit which they are aim to work with. - * + * * Note, that collector_unit_number is always less or equal to current_unit_number, that's why the formatting is order-preserving. - * + * * To stop the execution, a fake Chunk is added (ProcessingUnitType = FINALIZE) and finalize() * function is blocked until the Collector thread is done. */ diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index cdefd00ac69..b2e6dc7708e 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -83,6 +83,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa ReadBuffer read_buffer(unit.segment.data(), unit.segment.size(), 0); InputFormatPtr input_format = internal_parser_creator(read_buffer); + input_format->setCurrentUnitNumber(current_ticket_number); InternalParser parser(input_format); unit.chunk_ext.chunk.clear(); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index 4efe8b465de..802f05b3d8b 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -27,6 +27,16 @@ class Context; * The number of chunks equals to the number or parser threads. * The size of chunk is equal to min_chunk_bytes_for_parallel_parsing setting. * + * Parsers + * | | | | | | | | | | + * v v v v v v v v v v + * |---|---|---|---|---|---|---|---|---|---| + * | 1 | 2 | 3 | 4 | 5 | . | . | . | . | N | <-- Processing units + * |---|---|---|---|---|---|---|---|---|---| + * ^ ^ + * | | + * readImpl Segmentator + * * This stream has three kinds of threads: one segmentator, multiple parsers, * and one reader thread -- that is, the one from which readImpl() is called. * They operate one after another on parts of data called "processing units". @@ -78,9 +88,6 @@ public: , file_segmentation_engine(params.file_segmentation_engine) , format_name(params.format_name) , min_chunk_bytes(params.min_chunk_bytes) - // Subtract one thread that we use for segmentation and one for - // reading. After that, must have at least two threads left for - // parsing. See the assertion below. , pool(params.max_threads) { // One unit for each thread, including segmentator and reader, plus a diff --git a/src/Processors/ya.make b/src/Processors/ya.make index 6e99589bf02..2eb27be8899 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -45,6 +45,7 @@ SRCS( Formats/Impl/MySQLOutputFormat.cpp Formats/Impl/NullFormat.cpp Formats/Impl/ODBCDriver2BlockOutputFormat.cpp + Formats/Impl/ParallelFormattingOutputFormat.cpp Formats/Impl/ParallelParsingInputFormat.cpp Formats/Impl/PostgreSQLOutputFormat.cpp Formats/Impl/PrettyBlockOutputFormat.cpp diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 4a40b9e4abd..df7aa71cabb 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -61,6 +61,7 @@ #include #include + namespace ProfileEvents { extern const Event RejectedInserts; @@ -2892,7 +2893,6 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, const Context ReadBufferFromMemory right_paren_buf(")", 1); ConcatReadBuffer buf({&left_paren_buf, &fields_buf, &right_paren_buf}); - auto input_format = FormatFactory::instance().getInput("Values", buf, metadata_snapshot->getPartitionKey().sample_block, context, context.getSettingsRef().max_block_size); auto input_stream = std::make_shared(input_format); diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 45073ccd35a..793d57000d9 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -326,7 +326,6 @@ public: method = chooseCompressionMethod(current_path, storage->compression_method); } - read_buf = wrapReadBufferWithCompressionMethod(std::move(nested_buffer), method); auto format = FormatFactory::instance().getInput( storage->format_name, *read_buf, metadata_snapshot->getSampleBlock(), context, max_block_size, storage->format_settings); diff --git a/tests/queries/0_stateless/01246_insert_into_watch_live_view.py b/tests/queries/0_stateless/01246_insert_into_watch_live_view.py index f533573bfc7..b762e2d172f 100755 --- a/tests/queries/0_stateless/01246_insert_into_watch_live_view.py +++ b/tests/queries/0_stateless/01246_insert_into_watch_live_view.py @@ -17,13 +17,6 @@ with client(name='client1>', log=log) as client1, client(name='client2>', log=lo client1.expect(prompt) client2.expect(prompt) client3.expect(prompt) - - client1.send('SET output_format_parallel_formatting=0') - client1.expect(prompt) - client2.send('SET output_format_parallel_formatting=0') - client2.expect(prompt) - client3.send('SET output_format_parallel_formatting=0') - client3.expect(prompt) client1.send('SET allow_experimental_live_view = 1') client1.expect(prompt) diff --git a/tests/queries/0_stateless/helpers/client.py b/tests/queries/0_stateless/helpers/client.py index 086d920d0b7..3da91a4ad20 100644 --- a/tests/queries/0_stateless/helpers/client.py +++ b/tests/queries/0_stateless/helpers/client.py @@ -15,7 +15,7 @@ class client(object): def __init__(self, command=None, name='', log=None): self.client = uexpect.spawn(['/bin/bash','--noediting']) if command is None: - command = os.environ.get('CLICKHOUSE_BINARY', 'clickhouse') + ' client' + command = os.environ.get('CLICKHOUSE_BINARY', 'clickhouse') + ' client --progress' self.client.command = command self.client.eol('\r') self.client.logger(log, prefix=name) From 2f04cb5ebe1fb7daa504d59079e4cc211b30349f Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 30 Dec 2020 08:31:45 +0300 Subject: [PATCH 236/284] abort() instead of std::terminate() + cleanup --- src/Common/ThreadPool.h | 14 ++++---------- src/Storages/HDFS/StorageHDFS.cpp | 1 - src/Storages/StorageURL.cpp | 1 - .../01246_insert_into_watch_live_view.py | 2 +- tests/queries/0_stateless/helpers/client.py | 2 +- 5 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h index cceadc4ea2c..f5cd887ed44 100644 --- a/src/Common/ThreadPool.h +++ b/src/Common/ThreadPool.h @@ -187,7 +187,7 @@ public: ThreadFromGlobalPool & operator=(ThreadFromGlobalPool && rhs) { if (joinable()) - std::terminate(); + abort(); state = std::move(rhs.state); return *this; } @@ -195,19 +195,13 @@ public: ~ThreadFromGlobalPool() { if (joinable()) - { - std::cerr << StackTrace().toString() << std::endl; - std::terminate(); - } + abort(); } void join() { if (!joinable()) - { - std::cerr << StackTrace().toString() << std::endl; - std::terminate(); - } + abort(); state->wait(); state.reset(); @@ -216,7 +210,7 @@ public: void detach() { if (!joinable()) - std::terminate(); + abort(); state.reset(); } diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index fe165e8a8a8..f7afd4a497d 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -122,7 +122,6 @@ public: current_path = uri + path; auto compression = chooseCompressionMethod(path, compression_method); - auto read_buf = wrapReadBufferWithCompressionMethod(std::make_unique(current_path, context.getGlobalContext().getConfigRef()), compression); auto input_format = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size); auto input_stream = std::make_shared(input_format); diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index ddbab99c3f0..ca984f9ece9 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -106,7 +106,6 @@ namespace context.getRemoteHostFilter()), compression_method); - auto input_format = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size, format_settings); reader = std::make_shared(input_format); reader = std::make_shared(reader, columns, context); diff --git a/tests/queries/0_stateless/01246_insert_into_watch_live_view.py b/tests/queries/0_stateless/01246_insert_into_watch_live_view.py index b762e2d172f..0f7c6965b7b 100755 --- a/tests/queries/0_stateless/01246_insert_into_watch_live_view.py +++ b/tests/queries/0_stateless/01246_insert_into_watch_live_view.py @@ -17,7 +17,7 @@ with client(name='client1>', log=log) as client1, client(name='client2>', log=lo client1.expect(prompt) client2.expect(prompt) client3.expect(prompt) - + client1.send('SET allow_experimental_live_view = 1') client1.expect(prompt) client3.send('SET allow_experimental_live_view = 1') diff --git a/tests/queries/0_stateless/helpers/client.py b/tests/queries/0_stateless/helpers/client.py index 3da91a4ad20..086d920d0b7 100644 --- a/tests/queries/0_stateless/helpers/client.py +++ b/tests/queries/0_stateless/helpers/client.py @@ -15,7 +15,7 @@ class client(object): def __init__(self, command=None, name='', log=None): self.client = uexpect.spawn(['/bin/bash','--noediting']) if command is None: - command = os.environ.get('CLICKHOUSE_BINARY', 'clickhouse') + ' client --progress' + command = os.environ.get('CLICKHOUSE_BINARY', 'clickhouse') + ' client' self.client.command = command self.client.eol('\r') self.client.logger(log, prefix=name) From a1d3c0173d421c059d6c69859743c9fb664f96b9 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 30 Dec 2020 08:34:49 +0300 Subject: [PATCH 237/284] better --- src/Formats/JSONEachRowUtils.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Formats/JSONEachRowUtils.cpp b/src/Formats/JSONEachRowUtils.cpp index 76d14276878..6017f3983c6 100644 --- a/src/Formats/JSONEachRowUtils.cpp +++ b/src/Formats/JSONEachRowUtils.cpp @@ -9,7 +9,6 @@ std::pair fileSegmentationEngineJSONEachRowImpl(ReadBuffer & in, D skipWhitespaceIfAny(in); char * pos = in.position(); - size_t balance = 0; bool quotes = false; size_t number_of_rows = 0; From ec0f70b8ef1547b2449d1816fe759ef9baaf96ae Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 30 Dec 2020 08:39:08 +0300 Subject: [PATCH 238/284] cleanup --- src/Storages/StorageFile.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 793d57000d9..85888ee4b6a 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -483,11 +483,6 @@ public: writer = FormatFactory::instance().getOutputStreamParallelIfPossible(storage.format_name, *write_buf, metadata_snapshot->getSampleBlock(), context, {}, format_settings); - - if (!writer) - writer = FormatFactory::instance().getOutputStream(storage.format_name, - *write_buf, metadata_snapshot->getSampleBlock(), context, - {}, format_settings); } Block getHeader() const override { return metadata_snapshot->getSampleBlock(); } From c256c84c659c436ec36e93827dbefea74c69dc8e Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Wed, 30 Dec 2020 09:44:59 +0300 Subject: [PATCH 239/284] Trying to fix syntax bug --- .../en/sql-reference/table-functions/mysql.md | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/docs/en/sql-reference/table-functions/mysql.md b/docs/en/sql-reference/table-functions/mysql.md index ed859e5ca79..304f4c3ffe7 100644 --- a/docs/en/sql-reference/table-functions/mysql.md +++ b/docs/en/sql-reference/table-functions/mysql.md @@ -32,7 +32,8 @@ mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_ - `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` expression that is added to the `INSERT` query. Can be specified only with `replace_query = 0` (if you simultaneously pass `replace_query = 1` and `on_duplicate_clause`, ClickHouse generates an exception). Example: - ```sql + + ``` sql INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1; ``` @@ -53,19 +54,17 @@ Table in MySQL: ``` text mysql> CREATE TABLE `test`.`test` ( -> `int_id` INT NOT NULL AUTO_INCREMENT, - -> `int_nullable` INT NULL DEFAULT NULL, -> `float` FLOAT NOT NULL, - -> `float_nullable` FLOAT NULL DEFAULT NULL, -> PRIMARY KEY (`int_id`)); mysql> INSERT INTO test (`int_id`, `float`) VALUES (1,2); mysql> SELECT * FROM test; -+--------+--------------+-------+----------------+ -| int_id | int_nullable | float | float_nullable | -+--------+--------------+-------+----------------+ -| 1 | NULL | 2 | NULL | -+--------+--------------+-------+----------------+ ++--------+-------+ +| int_id | float | ++--------+-------+ +| 1 | 2 | ++--------+-------+ ``` Selecting data from ClickHouse: @@ -75,9 +74,9 @@ SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123'); ``` ``` text -┌─int_id─┬─int_nullable─┬─float─┬─float_nullable─┐ -│ 1 │ ᴺᵁᴸᴸ │ 2 │ ᴺᵁᴸᴸ │ -└────────┴──────────────┴───────┴────────────────┘ +┌─int_id─┬─float─┐ +│ 1 │ 2 │ +└────────┴───────┘ ``` Replacing and inserting: @@ -88,11 +87,11 @@ INSERT INTO mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 0, 'UPDATE SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123'); ``` -```text -┌─int_id─┬─int_nullable─┬─float─┬─float_nullable─┐ -│ 1 │ ᴺᵁᴸᴸ │ 3 │ ᴺᵁᴸᴸ │ -│ 2 │ ᴺᵁᴸᴸ │ 4 │ ᴺᵁᴸᴸ │ -└────────┴──────────────┴───────┴────────────────┘ +``` text +┌─int_id─┬─float─┐ +│ 1 │ 3 │ +│ 2 │ 4 │ +└────────┴───────┘ ``` **See Also** From ecfa92bceb18e6c6362b1355da1813c087bba834 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Wed, 30 Dec 2020 09:53:32 +0300 Subject: [PATCH 240/284] Still trying --- docs/en/sql-reference/table-functions/mysql.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/en/sql-reference/table-functions/mysql.md b/docs/en/sql-reference/table-functions/mysql.md index 304f4c3ffe7..0af9ae415c2 100644 --- a/docs/en/sql-reference/table-functions/mysql.md +++ b/docs/en/sql-reference/table-functions/mysql.md @@ -31,11 +31,7 @@ mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_ - `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` expression that is added to the `INSERT` query. Can be specified only with `replace_query = 0` (if you simultaneously pass `replace_query = 1` and `on_duplicate_clause`, ClickHouse generates an exception). - Example: - - ``` sql - INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1; - ``` + Example: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1;` `on_duplicate_clause` here is `UPDATE c2 = c2 + 1`. See the MySQL documentation to find which `on_duplicate_clause` you can use with the `ON DUPLICATE KEY` clause. From 203af8b7ec38c938185410c877e08e1a83e88061 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 30 Dec 2020 16:52:37 +0300 Subject: [PATCH 241/284] Fix error --- src/Interpreters/HashJoin.cpp | 18 ++++++++++++++---- src/Interpreters/HashJoin.h | 4 ++++ src/Storages/StorageMerge.cpp | 1 - 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 5d7fb213134..8f74a7be493 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -360,6 +360,17 @@ void HashJoin::init(Type type_) size_t HashJoin::getTotalRowCount() const { std::shared_lock lock(data->rwlock); + return getTotalRowCountLocked(); +} + +size_t HashJoin::getTotalByteCount() const +{ + std::shared_lock lock(data->rwlock); + return getTotalByteCountLocked(); +} + +size_t HashJoin::getTotalRowCountLocked() const +{ size_t res = 0; if (data->type == Type::CROSS) @@ -375,9 +386,8 @@ size_t HashJoin::getTotalRowCount() const return res; } -size_t HashJoin::getTotalByteCount() const +size_t HashJoin::getTotalByteCountLocked() const { - std::shared_lock lock(data->rwlock); size_t res = 0; if (data->type == Type::CROSS) @@ -594,8 +604,8 @@ bool HashJoin::addJoinedBlock(const Block & source_block, bool check_limits) return true; /// TODO: Do not calculate them every time - total_rows = getTotalRowCount(); - total_bytes = getTotalByteCount(); + total_rows = getTotalRowCountLocked(); + total_bytes = getTotalByteCountLocked(); } return table_join->sizeLimits().check(total_rows, total_bytes, "JOIN", ErrorCodes::SET_SIZE_LIMIT_EXCEEDED); diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index ceddc9e4c7c..37bc9a9d345 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -393,6 +393,10 @@ private: ColumnWithTypeAndName joinGetImpl(const Block & block, const Block & block_with_columns_to_add, const Maps & maps_) const; static Type chooseMethod(const ColumnRawPtrs & key_columns, Sizes & key_sizes); + + /// Call with already locked rwlock. + size_t getTotalRowCountLocked() const; + size_t getTotalByteCountLocked() const; }; } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 63d28eaff77..18534d36e71 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -542,7 +542,6 @@ void StorageMerge::convertingSourceStream( + "\n" + header.dumpStructure(), ErrorCodes::LOGICAL_ERROR); } } - } } From b94a65471576e4d1addb8176d83c40e6d1a39cd1 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 30 Dec 2020 16:55:31 +0300 Subject: [PATCH 242/284] build fix --- src/Core/Settings.h | 2 +- src/Formats/FormatFactory.cpp | 3 --- .../Formats/Impl/ParallelParsingInputFormat.cpp | 8 ++++---- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 9fd729fc787..e916b059367 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -121,7 +121,7 @@ class IColumn; \ M(Bool, input_format_parallel_parsing, true, "Enable parallel parsing for some data formats.", 0) \ M(UInt64, min_chunk_bytes_for_parallel_parsing, (10 * 1024 * 1024), "The minimum chunk size in bytes, which each thread will parse in parallel.", 0) \ - M(Bool, output_format_parallel_formatting, true, "Enable parallel formatting for all data formats.", 0) \ + M(Bool, output_format_parallel_formatting, true, "Enable parallel formatting for some data formats.", 0) \ \ M(UInt64, merge_tree_min_rows_for_concurrent_read, (20 * 8192), "If at least as many lines are read from one file, the reading can be parallelized.", 0) \ M(UInt64, merge_tree_min_bytes_for_concurrent_read, (24 * 10 * 1024 * 1024), "If at least as many bytes are read from one file, the reading can be parallelized.", 0) \ diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index ffa8ccfcced..e78190d2976 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -306,9 +306,6 @@ OutputFormatPtr FormatFactory::getOutputFormatParallelIfPossible( if (!output_getter) throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); - RowOutputFormatParams params; - params.callback = std::move(callback); - auto format_settings = _format_settings ? *_format_settings : getFormatSettings(context); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index b2e6dc7708e..57a0508e31a 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -67,8 +67,8 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa if (thread_group) CurrentThread::attachTo(thread_group); - const auto current_unit_number = current_ticket_number % processing_units.size(); - auto & unit = processing_units[current_unit_number]; + const auto parser_unit_number = current_ticket_number % processing_units.size(); + auto & unit = processing_units[parser_unit_number]; try { @@ -151,8 +151,8 @@ Chunk ParallelParsingInputFormat::generate() return {}; } - const auto current_unit_number = reader_ticket_number % processing_units.size(); - auto & unit = processing_units[current_unit_number]; + const auto inserter_unit_number = reader_ticket_number % processing_units.size(); + auto & unit = processing_units[inserter_unit_number]; if (!next_block_in_current_unit.has_value()) { From 83003162c8282d58c9d419a6fe0dd3c58631904d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 30 Dec 2020 16:56:34 +0300 Subject: [PATCH 243/284] Update settings.md --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 8e6c11d77a3..43ac1e1eb2e 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2480,7 +2480,7 @@ Possible values: - `'ALL'` — ClickHouse outputs all rows as a result of combining queries including duplicate rows. - `''` — Clickhouse generates an exception when used with `UNION`. -Default value: `'DISTINCT'`. +Default value: `''`. See examples in [UNION](../../sql-reference/statements/select/union.md). From 278dda04a6252d0bb4a1fe269ea89960aeed4404 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 30 Dec 2020 16:59:04 +0300 Subject: [PATCH 244/284] Update union.md --- docs/en/sql-reference/statements/select/union.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/union.md b/docs/en/sql-reference/statements/select/union.md index cae7aa07abb..cf18ff7a4a2 100644 --- a/docs/en/sql-reference/statements/select/union.md +++ b/docs/en/sql-reference/statements/select/union.md @@ -6,7 +6,7 @@ toc_title: UNION You can use `UNION` with explicitly specifying `UNION ALL` or `UNION DISTINCT`. -By default, `UNION` has the same behavior as `UNION DISTINCT`. The difference between `UNION ALL` and `UNION DISTINCT` is that `UNION DISTINCT` will do a distinct transform for union result, it is equivalent to `SELECT DISTINCT` from a subquery containing `UNION ALL`. +If you don't specify `ALL` or `DISTINCT`, it will depend on the `union_default_mode` setting. The difference between `UNION ALL` and `UNION DISTINCT` is that `UNION DISTINCT` will do a distinct transform for union result, it is equivalent to `SELECT DISTINCT` from a subquery containing `UNION ALL`. You can use `UNION` to combine any number of `SELECT` queries by extending their results. Example: From 0793617553e3a653d1e0f66b6aab494ccc1f6150 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 30 Dec 2020 16:59:25 +0300 Subject: [PATCH 245/284] Update settings.md --- docs/ru/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 692d0d59059..03edec14d05 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2334,7 +2334,7 @@ SELECT number FROM numbers(3) FORMAT JSONEachRow; - `'ALL'` — ClickHouse выводит все строки в результате объединения результатов запросов, включая повторяющиеся строки. - `''` — Clickhouse генерирует исключение при использовании с `UNION`. -Значение по умолчанию: `'DISTINCT'`. +Значение по умолчанию: `''`. Смотрите примеры в разделе [UNION](../../sql-reference/statements/select/union.md). From b3660d77832615ffff0adbe6bb4ee1a770ad09b3 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 30 Dec 2020 17:00:54 +0300 Subject: [PATCH 246/284] Update union.md --- docs/ru/sql-reference/statements/select/union.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/sql-reference/statements/select/union.md b/docs/ru/sql-reference/statements/select/union.md index e0a79978fdd..8f1dc11c802 100644 --- a/docs/ru/sql-reference/statements/select/union.md +++ b/docs/ru/sql-reference/statements/select/union.md @@ -6,9 +6,9 @@ toc_title: UNION Вы можете использовать `UNION` в двух режимах: `UNION ALL` или `UNION DISTINCT`. -По умолчанию, `UNION` ведет себя так же, как и `UNION DISTINCT`. Разница между `UNION ALL` и `UNION DISTINCT` в том, что `UNION DISTINCT` выполняет явное преобразование для результата объединения. Это равнозначно выражению `SELECT DISTINCT` из подзапроса, содержащего `UNION ALL`. +Если `UNION` используется без указания `ALL` или `DISTINCT`, то его поведение определяется настройкой `union_default_mode`. Разница между `UNION ALL` и `UNION DISTINCT` в том, что `UNION DISTINCT` выполняет явное преобразование для результата объединения. Это равнозначно выражению `SELECT DISTINCT` из подзапроса, содержащего `UNION ALL`. -Чтобы объединить любое количество `SELECT` запросов путем расширения их результатов, вы можете использовать `UNION`. Пример: +Чтобы объединить любое количество `SELECT` запросов путем объединения их результатов, вы можете использовать `UNION`. Пример: ``` sql SELECT CounterID, 1 AS table, toInt64(count()) AS c From 61ee873328c624837bc378185f09a78bb15c28a9 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 30 Dec 2020 17:04:34 +0300 Subject: [PATCH 247/284] Update docs/en/operations/settings/settings.md Co-authored-by: Azat Khuzhin --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 01e15f772ac..2304c91ed31 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -726,7 +726,7 @@ log_queries=1 ## log_queries_min_query_duration_ms {#settings-log-queries-min-query-duration-ms} -Minimum query execution time to navigate to the following tables: +If enabled (non-zero), queries faster then the value of this setting will not be logged (you can think about this as a `long_query_time` for [MySQL Slow Query Log](https://dev.mysql.com/doc/refman/5.7/en/slow-query-log.html)), and this basically means that you will not find them in the following tables: - `system.query_log` - `system.query_thread_log` From 41639652f6cb58f5dd38d7d847207bab02e06951 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 30 Dec 2020 17:07:01 +0300 Subject: [PATCH 248/284] Update settings.md --- docs/ru/operations/settings/settings.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index e7355935687..75f0f74db8e 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -693,18 +693,18 @@ log_queries=1 ## log_queries_min_query_duration_ms {#settings-log-queries-min-query-duration-ms} -Минимальное время выполнения запроса для перехода к следующим таблицам: +Минимальное время выполнения запроса для логгирования в системные таблицы: - `system.query_log` - `system.query_thread_log` -В лог попадут только запросы следующего типа: +В случае ненулевого порога `log_queries_min_query_duration_ms`, в лог будут записываться лишь события об окончании выполнения запроса: - `QUERY_FINISH` - `EXCEPTION_WHILE_PROCESSING` - Тип: milliseconds -- Значение по умолчанию: 0 (для любого запроса) +- Значение по умолчанию: 0 (логгировать все запросы) ## log_queries_min_type {#settings-log-queries-min-type} From ca4a795be39bf15c0a8d1805f2139322a96702e7 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 30 Dec 2020 17:07:45 +0300 Subject: [PATCH 249/284] Update date.md --- docs/ru/sql-reference/data-types/date.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/data-types/date.md b/docs/ru/sql-reference/data-types/date.md index e2162f22ff5..490bc5c28b4 100644 --- a/docs/ru/sql-reference/data-types/date.md +++ b/docs/ru/sql-reference/data-types/date.md @@ -11,7 +11,7 @@ toc_title: Date ## Примеры {#examples} -**1.** Создание таблицы со столбцом типа `DateTime` и добавление в неё данных: +**1.** Создание таблицы и добавление в неё данных: ``` sql CREATE TABLE dt From f80d3c293ae1e0fed16b8b37235a5589a3a498fa Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 30 Dec 2020 17:09:43 +0300 Subject: [PATCH 250/284] Update hash-functions.md --- docs/ru/sql-reference/functions/hash-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md index 5dfdff91c31..f7820889ea9 100644 --- a/docs/ru/sql-reference/functions/hash-functions.md +++ b/docs/ru/sql-reference/functions/hash-functions.md @@ -157,7 +157,7 @@ SELECT groupBitXor(cityHash64(*)) FROM table ## farmHash64 {#farmhash64} -Создает 64-битное значение [FarmHash](https://github.com/google/farmhash) или значение Fingerprint. `farmFingerprint64` предпочтительнее для стабильных и перемещаемых значений. +Создает 64-битное значение [FarmHash](https://github.com/google/farmhash), независимое от платформы (архитектуры сервера), что важно, если значения сохраняются или используются для разбиения данных на группы. ``` sql farmFingerprint64(par1, ...) From 736cc6959ae13cbcf2ebf8bd6382f1ca875bb2b6 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 30 Dec 2020 17:24:39 +0300 Subject: [PATCH 251/284] remove kafka flaky fix --- tests/integration/test_storage_kafka/test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 3f2bc1ca250..1f31cbdbbc7 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -1617,8 +1617,6 @@ def test_kafka_virtual_columns2(kafka_cluster): kafka_num_consumers = 2, kafka_format = 'JSONEachRow'; - SELECT * FROM test.kafka; - CREATE MATERIALIZED VIEW test.view Engine=Log AS SELECT value, _key, _topic, _partition, _offset, toUnixTimestamp(_timestamp), toUnixTimestamp64Milli(_timestamp_ms), _headers.name, _headers.value FROM test.kafka; ''') From 797c969c4d1a831a04231250cee2ac6cc690544f Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 30 Dec 2020 17:25:55 +0300 Subject: [PATCH 252/284] Revert "Add some extra tests to copier" --- .../configs/conf.d/big_clusters.xml | 71 -------- .../test_cluster_copier/task_many_to_one.xml | 128 -------------- .../test_cluster_copier/test_big_cluster.py | 159 ------------------ .../{test_trivial.py => trivial_test.py} | 33 ++-- 4 files changed, 11 insertions(+), 380 deletions(-) delete mode 100644 tests/integration/test_cluster_copier/configs/conf.d/big_clusters.xml delete mode 100644 tests/integration/test_cluster_copier/task_many_to_one.xml delete mode 100644 tests/integration/test_cluster_copier/test_big_cluster.py rename tests/integration/test_cluster_copier/{test_trivial.py => trivial_test.py} (80%) diff --git a/tests/integration/test_cluster_copier/configs/conf.d/big_clusters.xml b/tests/integration/test_cluster_copier/configs/conf.d/big_clusters.xml deleted file mode 100644 index da29534cf33..00000000000 --- a/tests/integration/test_cluster_copier/configs/conf.d/big_clusters.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - - - - true - - s0_0_0 - 9000 - - - s0_0_1 - 9000 - - - - true - - s0_1_0 - 9000 - - - s0_1_1 - 9000 - - - - true - - s0_2_0 - 9000 - - - s0_2_1 - 9000 - - - - true - - s0_3_0 - 9000 - - - s0_3_1 - 9000 - - - - true - - s0_4_0 - 9000 - - - s0_4_1 - 9000 - - - - - - - - s1_0_0 - 9000 - - - - - \ No newline at end of file diff --git a/tests/integration/test_cluster_copier/task_many_to_one.xml b/tests/integration/test_cluster_copier/task_many_to_one.xml deleted file mode 100644 index 038ef5df3ae..00000000000 --- a/tests/integration/test_cluster_copier/task_many_to_one.xml +++ /dev/null @@ -1,128 +0,0 @@ - - - - - - true - - s0_0_0 - 9000 - - - s0_0_1 - 9000 - - - - true - - s0_1_0 - 9000 - - - s0_1_1 - 9000 - - - - true - - s0_2_0 - 9000 - - - s0_2_1 - 9000 - - - - true - - s0_3_0 - 9000 - - - s0_3_1 - 9000 - - - - true - - s0_4_0 - 9000 - - - s0_4_1 - 9000 - - - - - - - - s1_0_0 - 9000 - - - - - - - 3 - - - - 1 - - - - - 0 - - - - - 3 - - 1 - - - - - - - - big_source - default - big_all - - - big_destination - default - big - - - - ENGINE = MergeTree() - ORDER BY (first_id, second_id, toSecond(datetime)) - PARTITION BY toSecond(datetime) - - - - cityHash64(first_id, second_id) - - - diff --git a/tests/integration/test_cluster_copier/test_big_cluster.py b/tests/integration/test_cluster_copier/test_big_cluster.py deleted file mode 100644 index 2424f0149e4..00000000000 --- a/tests/integration/test_cluster_copier/test_big_cluster.py +++ /dev/null @@ -1,159 +0,0 @@ -import os -import random -import sys -import time -from contextlib import contextmanager - -import docker -import kazoo -import pytest -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV - -CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR)) - -COPYING_FAIL_PROBABILITY = 0.2 -MOVING_FAIL_PROBABILITY = 0.2 - -cluster = ClickHouseCluster(__file__) - - -def check_all_hosts_sucesfully_executed(tsv_content, num_hosts): - M = TSV.toMat(tsv_content) - hosts = [(l[0], l[1]) for l in M] # (host, port) - codes = [l[2] for l in M] - - assert len(hosts) == num_hosts and len(set(hosts)) == num_hosts, "\n" + tsv_content - assert len(set(codes)) == 1, "\n" + tsv_content - assert codes[0] == "0", "\n" + tsv_content - - -def ddl_check_query(instance, query, num_hosts=3): - contents = instance.query(query) - check_all_hosts_sucesfully_executed(contents, num_hosts) - return contents - - -@pytest.fixture(scope="module") -def started_cluster(): - global cluster - try: - clusters_schema = { - "0": { - "0": ["0", "1"], - "1": ["0", "1"], - "2": ["0", "1"], - "3": ["0", "1"], - "4": ["0", "1"] - }, - "1": { - "0": ["0"] - } - } - - for cluster_name, shards in clusters_schema.items(): - for shard_name, replicas in shards.items(): - for replica_name in replicas: - name = "s{}_{}_{}".format(cluster_name, shard_name, replica_name) - cluster.add_instance(name, - main_configs=["configs/conf.d/query_log.xml", "configs/conf.d/ddl.xml", - "configs/conf.d/big_clusters.xml"], - user_configs=["configs/users.xml"], - macros={"cluster": cluster_name, "shard": shard_name, "replica": replica_name}, - with_zookeeper=True) - - cluster.start() - yield cluster - - finally: - cluster.shutdown() - - -class Task_many_to_one: - def __init__(self, cluster): - self.cluster = cluster - self.zk_task_path = "/clickhouse-copier/task_many_to_one" - self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_many_to_one.xml'), 'r').read() - self.rows = 1000 - - def start(self): - instance = cluster.instances['s0_0_0'] - - for cluster_num in ["big_source", "big_destination"]: - instance.query("DROP DATABASE IF EXISTS default ON CLUSTER {}".format(cluster_num)) - instance.query("CREATE DATABASE IF NOT EXISTS default ON CLUSTER {}".format(cluster_num)) - - instance.query("CREATE TABLE big ON CLUSTER big_source (first_id UUID DEFAULT generateUUIDv4(), second_id UInt64, datetime DateTime DEFAULT now()) " + - "ENGINE=ReplicatedMergeTree " + - "PARTITION BY toSecond(datetime) " + - "ORDER BY (first_id, second_id, toSecond(datetime))") - instance.query("CREATE TABLE big_all ON CLUSTER big_source (first_id UUID, second_id UInt64, datetime DateTime) ENGINE=Distributed(big_source, default, big, rand() % 5)") - instance.query("INSERT INTO big_all SELECT generateUUIDv4(), number, now() FROM system.numbers LIMIT 1002", - settings={"insert_distributed_sync": 1}) - - def check(self): - assert TSV(self.cluster.instances['s0_0_0'].query("SELECT count() FROM big_all")) == TSV("1002\n") - assert TSV(self.cluster.instances['s1_0_0'].query("SELECT count() FROM big")) == TSV("1002\n") - -def execute_task(task, cmd_options): - task.start() - - zk = cluster.get_kazoo_client('zoo1') - print("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) - - try: - zk.delete("/clickhouse-copier", recursive=True) - except kazoo.exceptions.NoNodeError: - print("No node /clickhouse-copier. It is Ok in first test.") - - zk_task_path = task.zk_task_path - zk.ensure_path(zk_task_path) - zk.create(zk_task_path + "/description", task.copier_task_config.encode()) - - # Run cluster-copier processes on each node - docker_api = docker.from_env().api - copiers_exec_ids = [] - - cmd = ['/usr/bin/clickhouse', 'copier', - '--config', '/etc/clickhouse-server/config-copier.xml', - '--task-path', zk_task_path, - '--base-dir', '/var/log/clickhouse-server/copier'] - cmd += cmd_options - - copiers = random.sample(list(cluster.instances.keys()), 3) - - for instance_name in copiers: - instance = cluster.instances[instance_name] - container = instance.get_docker_handle() - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs/config-copier.xml"), - "/etc/clickhouse-server/config-copier.xml") - print("Copied copier config to {}".format(instance.name)) - exec_id = docker_api.exec_create(container.id, cmd, stderr=True) - output = docker_api.exec_start(exec_id).decode('utf8') - print(output) - copiers_exec_ids.append(exec_id) - print("Copier for {} ({}) has started".format(instance.name, instance.ip_address)) - - # Wait for copiers stopping and check their return codes - for exec_id, instance_name in zip(copiers_exec_ids, copiers): - instance = cluster.instances[instance_name] - while True: - res = docker_api.exec_inspect(exec_id) - if not res['Running']: - break - time.sleep(0.5) - - assert res['ExitCode'] == 0, "Instance: {} ({}). Info: {}".format(instance.name, instance.ip_address, repr(res)) - - try: - task.check() - finally: - zk.delete(zk_task_path, recursive=True) - - -# Tests - -def test_copy_simple(started_cluster): - execute_task(Task_many_to_one(started_cluster), []) - \ No newline at end of file diff --git a/tests/integration/test_cluster_copier/test_trivial.py b/tests/integration/test_cluster_copier/trivial_test.py similarity index 80% rename from tests/integration/test_cluster_copier/test_trivial.py rename to tests/integration/test_cluster_copier/trivial_test.py index 71b76915cfa..8a43440ac90 100644 --- a/tests/integration/test_cluster_copier/test_trivial.py +++ b/tests/integration/test_cluster_copier/trivial_test.py @@ -5,7 +5,6 @@ from contextlib import contextmanager import docker import pytest -import random CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR)) @@ -86,14 +85,9 @@ def execute_task(task, cmd_options): zk = cluster.get_kazoo_client('zoo1') print("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) - try: - zk.delete("/clickhouse-copier", recursive=True) - except kazoo.exceptions.NoNodeError: - print("No node /clickhouse-copier. It is Ok in first test.") - zk_task_path = task.zk_task_path zk.ensure_path(zk_task_path) - zk.create(zk_task_path + "/description", task.copier_task_config.encode()) + zk.create(zk_task_path + "/description", task.copier_task_config) # Run cluster-copier processes on each node docker_api = docker.from_env().api @@ -105,28 +99,23 @@ def execute_task(task, cmd_options): '--base-dir', '/var/log/clickhouse-server/copier'] cmd += cmd_options - copiers = list(cluster.instances.keys()) + print(cmd) - for instance_name in copiers: - instance = cluster.instances[instance_name] + for instance_name, instance in cluster.instances.items(): container = instance.get_docker_handle() - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs/config-copier.xml"), - "/etc/clickhouse-server/config-copier.xml") - print("Copied copier config to {}".format(instance.name)) exec_id = docker_api.exec_create(container.id, cmd, stderr=True) - output = docker_api.exec_start(exec_id).decode('utf8') - print(output) + docker_api.exec_start(exec_id, detach=True) + copiers_exec_ids.append(exec_id) print("Copier for {} ({}) has started".format(instance.name, instance.ip_address)) # Wait for copiers stopping and check their return codes - for exec_id, instance_name in zip(copiers_exec_ids, copiers): - instance = cluster.instances[instance_name] + for exec_id, instance in zip(copiers_exec_ids, iter(cluster.instances.values())): while True: res = docker_api.exec_inspect(exec_id) if not res['Running']: break - time.sleep(0.5) + time.sleep(1) assert res['ExitCode'] == 0, "Instance: {} ({}). Info: {}".format(instance.name, instance.ip_address, repr(res)) @@ -163,10 +152,10 @@ def test_trivial_copy(started_cluster, use_sample_offset): ) def test_trivial_copy_with_copy_fault(started_cluster, use_sample_offset): if use_sample_offset: - execute_task(TaskTrivial(started_cluster, use_sample_offset), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY), + execute_task(TaskTrivial(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY), '--experimental-use-sample-offset', '1']) else: - execute_task(TaskTrivial(started_cluster, use_sample_offset), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) + execute_task(TaskTrivial(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) @pytest.mark.parametrize( @@ -178,10 +167,10 @@ def test_trivial_copy_with_copy_fault(started_cluster, use_sample_offset): ) def test_trivial_copy_with_move_fault(started_cluster, use_sample_offset): if use_sample_offset: - execute_task(TaskTrivial(started_cluster, use_sample_offset), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY), + execute_task(TaskTrivial(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY), '--experimental-use-sample-offset', '1']) else: - execute_task(TaskTrivial(started_cluster, use_sample_offset), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) + execute_task(TaskTrivial(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) if __name__ == '__main__': From ae72f96111e2b99449e6f5a9e9e0677b9756fffc Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 30 Dec 2020 22:38:11 +0800 Subject: [PATCH 253/284] Fix SimpleAggregateFunction in SummingMergeTree --- .../Algorithms/SummingSortedAlgorithm.cpp | 48 +++++++++++++++---- .../Algorithms/SummingSortedAlgorithm.h | 8 ++++ ...e_function_in_summing_merge_tree.reference | 1 + ...gregate_function_in_summing_merge_tree.sql | 9 ++++ 4 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.reference create mode 100644 tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.sql diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp index f558a6f0677..61065aad5c3 100644 --- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp @@ -4,9 +4,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -44,13 +46,20 @@ struct SummingSortedAlgorithm::AggregateDescription /// In case when column has type AggregateFunction: /// use the aggregate function from itself instead of 'function' above. bool is_agg_func_type = false; + bool is_simple_agg_func_type = false; void init(const char * function_name, const DataTypes & argument_types) { AggregateFunctionProperties properties; - function = AggregateFunctionFactory::instance().get(function_name, argument_types, {}, properties); + init(AggregateFunctionFactory::instance().get(function_name, argument_types, {}, properties)); + } + + void init(AggregateFunctionPtr function_, bool is_simple_agg_func_type_ = false) + { + function = std::move(function_); add_function = function->getAddressOfAddFunction(); state.reset(function->sizeOfData(), function->alignOfData()); + is_simple_agg_func_type = is_simple_agg_func_type_; } void createState() @@ -222,9 +231,10 @@ static SummingSortedAlgorithm::ColumnsDefinition defineColumns( else { bool is_agg_func = WhichDataType(column.type).isAggregateFunction(); + const auto * simple = dynamic_cast(column.type->getCustomName()); /// There are special const columns for example after prewhere sections. - if ((!column.type->isSummable() && !is_agg_func) || isColumnConst(*column.column)) + if ((!column.type->isSummable() && !is_agg_func && !simple) || isColumnConst(*column.column)) { def.column_numbers_not_to_aggregate.push_back(i); continue; @@ -246,7 +256,14 @@ static SummingSortedAlgorithm::ColumnsDefinition defineColumns( desc.is_agg_func_type = is_agg_func; desc.column_numbers = {i}; - if (!is_agg_func) + if (simple) + { + // simple aggregate function + desc.init(simple->getFunction(), true); + if (desc.function->allocatesMemoryInArena()) + def.allocates_memory_in_arena = true; + } + else if (!is_agg_func) { desc.init("sumWithOverflow", {column.type}); } @@ -354,7 +371,7 @@ static MutableColumns getMergedDataColumns( for (const auto & desc : def.columns_to_aggregate) { // Wrap aggregated columns in a tuple to match function signature - if (!desc.is_agg_func_type && isTuple(desc.function->getReturnType())) + if (!desc.is_agg_func_type && !desc.is_simple_agg_func_type && isTuple(desc.function->getReturnType())) { size_t tuple_size = desc.column_numbers.size(); MutableColumns tuple_columns(tuple_size); @@ -399,7 +416,7 @@ static void postprocessChunk( auto column = std::move(columns[next_column]); ++next_column; - if (!desc.is_agg_func_type && isTuple(desc.function->getReturnType())) + if (!desc.is_agg_func_type && !desc.is_simple_agg_func_type && isTuple(desc.function->getReturnType())) { /// Unpack tuple into block. size_t tuple_size = desc.column_numbers.size(); @@ -455,6 +472,13 @@ SummingSortedAlgorithm::SummingMergedData::SummingMergedData( { current_row.resize(def.column_names.size()); initAggregateDescription(); + + /// Just to make startGroup() simpler. + if (def.allocates_memory_in_arena) + { + arena = std::make_unique(); + arena_size = arena->size(); + } } void SummingSortedAlgorithm::SummingMergedData::startGroup(ColumnRawPtrs & raw_columns, size_t row) @@ -467,6 +491,12 @@ void SummingSortedAlgorithm::SummingMergedData::startGroup(ColumnRawPtrs & raw_c for (auto & desc : def.columns_to_aggregate) desc.createState(); + if (def.allocates_memory_in_arena && arena->size() > arena_size) + { + arena = std::make_unique(); + arena_size = arena->size(); + } + if (def.maps_to_sum.empty()) { /// We have only columns_to_aggregate. The status of current row will be determined @@ -505,10 +535,10 @@ void SummingSortedAlgorithm::SummingMergedData::finishGroup() { try { - desc.function->insertResultInto(desc.state.data(), *desc.merged_column, nullptr); + desc.function->insertResultInto(desc.state.data(), *desc.merged_column, arena.get()); /// Update zero status of current row - if (desc.column_numbers.size() == 1) + if (!desc.is_simple_agg_func_type && desc.column_numbers.size() == 1) { // Flag row as non-empty if at least one column number if non-zero current_row_is_zero = current_row_is_zero @@ -586,7 +616,7 @@ void SummingSortedAlgorithm::SummingMergedData::addRowImpl(ColumnRawPtrs & raw_c if (desc.column_numbers.size() == 1) { auto & col = raw_columns[desc.column_numbers[0]]; - desc.add_function(desc.function.get(), desc.state.data(), &col, row, nullptr); + desc.add_function(desc.function.get(), desc.state.data(), &col, row, arena.get()); } else { @@ -595,7 +625,7 @@ void SummingSortedAlgorithm::SummingMergedData::addRowImpl(ColumnRawPtrs & raw_c for (size_t i = 0; i < desc.column_numbers.size(); ++i) column_ptrs[i] = raw_columns[desc.column_numbers[i]]; - desc.add_function(desc.function.get(), desc.state.data(), column_ptrs.data(), row, nullptr); + desc.add_function(desc.function.get(), desc.state.data(), column_ptrs.data(), row, arena.get()); } } } diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h index 2a455ad4cea..df3749b5e88 100644 --- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h @@ -50,6 +50,9 @@ public: /// Names of columns from header. Names column_names; + + /// Does SimpleAggregateFunction allocates memory in arena? + bool allocates_memory_in_arena = false; }; /// Specialization for SummingSortedTransform. Inserts only data for non-aggregated columns. @@ -73,6 +76,11 @@ public: private: ColumnsDefinition & def; + /// Memory pool for SimpleAggregateFunction + /// (only when allocates_memory_in_arena == true). + std::unique_ptr arena; + size_t arena_size = 0; + bool is_group_started = false; Row current_row; diff --git a/tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.reference b/tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.reference new file mode 100644 index 00000000000..dbc5d8aae43 --- /dev/null +++ b/tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.reference @@ -0,0 +1 @@ +([25],[1]) ([25],[1]) diff --git a/tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.sql b/tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.sql new file mode 100644 index 00000000000..764cebbc9a9 --- /dev/null +++ b/tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.sql @@ -0,0 +1,9 @@ +drop table if exists test_smt; + +create table test_smt (id UInt32, sMap SimpleAggregateFunction(sumMap, Tuple(Array(UInt8), Array(Int64))), aMap AggregateFunction(sumMap, Tuple(Array(UInt8), Array(Int64)))) engine SummingMergeTree partition by tuple() order by id; + +insert into test_smt select id, sumMap(k), sumMapState(k) from (select 2 as id, arrayJoin([([0], [1]), ([0, 25], [-1, toInt64(1)])]) as k) group by id, rowNumberInAllBlocks(); + +select sumMap(sMap), sumMapMerge(aMap) from test_smt; + +drop table if exists test_smt; From aa67966932a0caf8df6fc1f8b1106dc399e6dc62 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 30 Dec 2020 17:45:09 +0300 Subject: [PATCH 254/284] Update Compressor.cpp --- programs/compressor/Compressor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index 3751ffcd453..d4b706abf4d 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -92,6 +92,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) if (options.count("help")) { + std::cout << "Usage: " << argv[0] << " [options] < INPUT > OUTPUT" << std::endl; std::cout << "Usage: " << argv[0] << " [options] INPUT OUTPUT" << std::endl; std::cout << desc << std::endl; return 0; From 3620c8b4d47cd71c000b7bf88c026dc5fff4f2a1 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 30 Dec 2020 17:49:11 +0300 Subject: [PATCH 255/284] Update CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0209eaee249..e63304241d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,7 +76,7 @@ if (ENABLE_CHECK_HEAVY_BUILDS) if (SANITIZE STREQUAL "memory" OR COMPILER_GCC) set (RLIMIT_DATA 10000000000) endif() - set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS}--data=${RLIMIT_DATA}--cpu=600) + set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=600) endif () if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None") From 76c08d97ea1ac9d6845e799b622443d0ba11998c Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Wed, 30 Dec 2020 16:10:46 +0100 Subject: [PATCH 256/284] Update build.md Bit better. #18562. llvm auto installation also install lld. --- docs/en/development/build.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/en/development/build.md b/docs/en/development/build.md index d6e06be3e38..282f326da61 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -23,9 +23,29 @@ $ sudo apt-get install git cmake python ninja-build Or cmake3 instead of cmake on older systems. +### Install clang-11 (recommended) {#install-clang-11} + +The easiest way to do on Ubuntu/Debian is using automatic installation script (check [official webpage](https://apt.llvm.org/)) + +```bash +sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" +``` + +For other Linux distribution - check the packages avaliability (there are [prebuild packages](https://releases.llvm.org/download.html) for CentOS 8 Stream, SuSE, RedHad on arm and others) or build clang [from sources](https://clang.llvm.org/get_started.html). + +#### Use clang-11 for Builds {#use-gcc-10-for-builds} + +``` bash +$ export CC=clang-11 +$ export CXX=clang++-11 +$ LINKER_NAME=lld" +``` + ### Install GCC 10 {#install-gcc-10} -There are several ways to do this. +We recommend building ClickHouse with clang-11, GCC-10 is also supported but in some cases it produce less optimal code. + +If you want to use GCC-10 there are several ways to install it. #### Install from Repository {#install-from-repository} @@ -49,7 +69,7 @@ $ sudo apt-get install gcc-10 g++-10 See [utils/ci/build-gcc-from-sources.sh](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh) -### Use GCC 10 for Builds {#use-gcc-10-for-builds} +#### Use GCC 10 for Builds {#use-gcc-10-for-builds} ``` bash $ export CC=gcc-10 From f6bdb67909b56738e9a4cae0bafada03b03f0c21 Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Wed, 30 Dec 2020 16:12:12 +0100 Subject: [PATCH 257/284] Update build.md --- docs/en/development/build.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 282f326da61..68bea5cbfcc 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -38,7 +38,6 @@ For other Linux distribution - check the packages avaliability (there are [prebu ``` bash $ export CC=clang-11 $ export CXX=clang++-11 -$ LINKER_NAME=lld" ``` ### Install GCC 10 {#install-gcc-10} From d4ec52f73256ce4b47f902106789e9aa1c79746a Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 30 Dec 2020 18:21:58 +0300 Subject: [PATCH 258/284] shellchek --- src/Formats/FormatFactory.h | 4 ++-- src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp | 2 +- src/Processors/Formats/Impl/ParallelParsingInputFormat.h | 2 +- .../1_stateful/00159_parallel_formatting_csv_and_friends.sh | 1 + tests/queries/1_stateful/00159_parallel_formatting_http.sh | 1 + .../1_stateful/00159_parallel_formatting_json_and_friends.sh | 1 + .../1_stateful/00159_parallel_formatting_tsv_and_friends.sh | 1 + 7 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index c4d14cc930c..4fa7e9a0c01 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -117,12 +117,12 @@ public: const std::optional & format_settings = std::nullopt) const; /// Checks all preconditions. Returns ordinary stream if parallel formatting cannot be done. - /// Currenly used only in Client. Don't use it something else! Better look at getOutputFormatParallelIfPossible. + /// Currently used only in Client. Don't use it something else! Better look at getOutputFormatParallelIfPossible. BlockOutputStreamPtr getOutputStreamParallelIfPossible(const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback = {}, const std::optional & format_settings = std::nullopt) const; - /// Currenly used only in Client. Don't use it something else! Better look at getOutputFormat. + /// Currently used only in Client. Don't use it something else! Better look at getOutputFormat. BlockOutputStreamPtr getOutputStream(const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback = {}, const std::optional & format_settings = std::nullopt) const; diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 57a0508e31a..932e5304bbd 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -90,7 +90,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa unit.chunk_ext.block_missing_values.clear(); // We don't know how many blocks will be. So we have to read them all - // until an empty block occured. + // until an empty block occurred. Chunk chunk; while (!parsing_finished && (chunk = parser.getChunk()) != Chunk()) { diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index 802f05b3d8b..9dda2dfe55d 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -125,7 +125,7 @@ protected: /* * The format parsers themselves are not being cancelled here, so we'll * have to wait until they process the current block. Given that the - * chunk size is on the order of megabytes, this should't be too long. + * chunk size is on the order of megabytes, this shouldn't be too long. * We can't call IInputFormat->cancel here, because the parser object is * local to the parser thread, and we don't want to introduce any * synchronization between parser threads and the other threads to get diff --git a/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.sh b/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.sh index 6296d722a97..dc14928afa6 100755 --- a/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.sh +++ b/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh FORMATS=('CSV' 'CSVWithNames') diff --git a/tests/queries/1_stateful/00159_parallel_formatting_http.sh b/tests/queries/1_stateful/00159_parallel_formatting_http.sh index 4c5fb1f066f..8fd8c15b7c7 100755 --- a/tests/queries/1_stateful/00159_parallel_formatting_http.sh +++ b/tests/queries/1_stateful/00159_parallel_formatting_http.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh index 64a7b83fb86..e02515d5c16 100755 --- a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh +++ b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh diff --git a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh index afdda8d1453..a81dfdc33b4 100755 --- a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh +++ b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh From 218aab185686e4457c4073cd7cece540f3e8efc8 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 30 Dec 2020 18:53:33 +0300 Subject: [PATCH 259/284] shellcheck --- tests/queries/0_stateless/01621_clickhouse_compressor.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01621_clickhouse_compressor.sh b/tests/queries/0_stateless/01621_clickhouse_compressor.sh index e00270e5db9..2746f31c9da 100755 --- a/tests/queries/0_stateless/01621_clickhouse_compressor.sh +++ b/tests/queries/0_stateless/01621_clickhouse_compressor.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -e From c8978d4ef9868a16c063ea085919d19e0ba1e627 Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Wed, 30 Dec 2020 17:34:21 +0100 Subject: [PATCH 260/284] Update build.md --- docs/en/development/build.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 68bea5cbfcc..487e6ba1806 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -25,7 +25,7 @@ Or cmake3 instead of cmake on older systems. ### Install clang-11 (recommended) {#install-clang-11} -The easiest way to do on Ubuntu/Debian is using automatic installation script (check [official webpage](https://apt.llvm.org/)) +On Ubuntu/Debian you can use automatic installation script (check [official webpage](https://apt.llvm.org/)) ```bash sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" From 96a1d6d3caa0ca3d75ecb30f4a2edddb06ddd987 Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Wed, 30 Dec 2020 17:38:20 +0100 Subject: [PATCH 261/284] Update build.md --- docs/en/development/build.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 487e6ba1806..080011e683a 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -25,13 +25,13 @@ Or cmake3 instead of cmake on older systems. ### Install clang-11 (recommended) {#install-clang-11} -On Ubuntu/Debian you can use automatic installation script (check [official webpage](https://apt.llvm.org/)) +On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/)) ```bash sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ``` -For other Linux distribution - check the packages avaliability (there are [prebuild packages](https://releases.llvm.org/download.html) for CentOS 8 Stream, SuSE, RedHad on arm and others) or build clang [from sources](https://clang.llvm.org/get_started.html). +For other Linux distribution - check the availability of the [prebuild packages](https://releases.llvm.org/download.html) or build clang [from sources](https://clang.llvm.org/get_started.html). #### Use clang-11 for Builds {#use-gcc-10-for-builds} @@ -42,7 +42,7 @@ $ export CXX=clang++-11 ### Install GCC 10 {#install-gcc-10} -We recommend building ClickHouse with clang-11, GCC-10 is also supported but in some cases it produce less optimal code. +We recommend building ClickHouse with clang-11, GCC-10 also supported, but it produces less optimal code. If you want to use GCC-10 there are several ways to install it. From b0a6e4e82f959a368bd989ab9f72c15d5b88fadc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 30 Dec 2020 20:55:10 +0300 Subject: [PATCH 262/284] Fix shellcheck --- tests/queries/0_stateless/01621_clickhouse_compressor.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01621_clickhouse_compressor.sh b/tests/queries/0_stateless/01621_clickhouse_compressor.sh index e00270e5db9..a9a7c87368e 100755 --- a/tests/queries/0_stateless/01621_clickhouse_compressor.sh +++ b/tests/queries/0_stateless/01621_clickhouse_compressor.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +# shellcheck source=../shell_config.sh CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh From 60a318a139be778e74824d530ac223cb5a14883f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 30 Dec 2020 21:16:50 +0300 Subject: [PATCH 263/284] Fix broken test #17902 --- .../00377_shard_group_uniq_array_of_string_array.sql | 4 ++-- .../01293_client_interactive_vertical_multiline.expect | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/00377_shard_group_uniq_array_of_string_array.sql b/tests/queries/0_stateless/00377_shard_group_uniq_array_of_string_array.sql index 76f717d6683..41ae2507d20 100644 --- a/tests/queries/0_stateless/00377_shard_group_uniq_array_of_string_array.sql +++ b/tests/queries/0_stateless/00377_shard_group_uniq_array_of_string_array.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS group_uniq_arr_str; CREATE TABLE group_uniq_arr_str ENGINE = Memory AS - SELECT hex(intHash32(g)) as id, if(c == 0, [hex(v)], if(c == 1, emptyArrayString(), [hex(v), hex(v)])) as v FROM - (SELECT intDiv(number%1000000, 100) as v, intDiv(number%100, 10) as g, number%10 as c FROM system.numbers WHERE c < 3 LIMIT 10000000); + SELECT hex(intHash32(g)) as id, if(c == 0, [hex(v)], if(c == 1, emptyArrayString(), [hex(v), hex(v)])) as v FROM + (SELECT intDiv(number%1000000, 100) as v, intDiv(number%100, 10) as g, number%10 as c FROM system.numbers WHERE c < 3 LIMIT 10000000); SELECT length(groupUniqArray(v)) FROM group_uniq_arr_str GROUP BY id ORDER BY id; SELECT length(groupUniqArray(v)) FROM remote('127.0.0.{2,3,4,5}', currentDatabase(), 'group_uniq_arr_str') GROUP BY id ORDER BY id; diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect index 15671b3870a..c7b3725b7ae 100755 --- a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect +++ b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect @@ -34,7 +34,7 @@ send -- "" expect eof set timeout 60 -spawn bash -c "$env(CLICKHOUSE_CLIENT_BINARY) $env(CLICKHOUSE_CLIENT_OPT)" +spawn bash -c "$env(CLICKHOUSE_CLIENT_BINARY) $env(CLICKHOUSE_CLIENT_OPT) --multiline" match_max 100000 expect ":) " From f0cd43ddaf9f3365927502af2ad7ec8c4077108e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 30 Dec 2020 21:31:06 +0300 Subject: [PATCH 264/284] Add two exceptions for TSan --- tests/queries/skip_list.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 14c5cd5a55e..54fb572454e 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -19,7 +19,9 @@ "01193_metadata_loading", "01473_event_time_microseconds", "01526_max_untracked_memory", /// requires TraceCollector, does not available under sanitizers - "01474_executable_dictionary" /// informational stderr from sanitizer at start + "01474_executable_dictionary", /// informational stderr from sanitizer at start + "functions_bad_arguments", /// Too long for TSan + "01603_read_with_backoff_bug" /// Too long for TSan ], "address-sanitizer": [ "00877", From f76410be3e75371dbaeac0b1b3db135c9e0f3eb2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 30 Dec 2020 21:46:02 +0300 Subject: [PATCH 265/284] More correct words about parser --- src/Interpreters/executeQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index a2ae96e8199..b8040fedefc 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -162,7 +162,7 @@ static void logQuery(const String & query, const Context & context, bool interna client_info.current_address.toString(), (current_user != "default" ? ", user: " + current_user : ""), (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()), - (context.getSettingsRef().use_antlr_parser ? "new" : "old"), + (context.getSettingsRef().use_antlr_parser ? "experimental" : "production"), joinLines(query)); if (client_info.client_trace_context.trace_id) From fa48e062ebe8af0c999b9f06ddf391e0c40319c1 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 30 Dec 2020 21:51:41 +0300 Subject: [PATCH 266/284] fix build --- src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 932e5304bbd..d1660b53019 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -21,8 +21,8 @@ void ParallelParsingInputFormat::segmentatorThreadFunction(ThreadGroupStatusPtr { while (!parsing_finished) { - const auto current_unit_number = segmentator_ticket_number % processing_units.size(); - auto & unit = processing_units[current_unit_number]; + const auto segmentator_unit_number = segmentator_ticket_number % processing_units.size(); + auto & unit = processing_units[segmentator_unit_number]; { std::unique_lock lock(mutex); From a530bad6ead42d3fc274b919faa6f59c91498772 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 30 Dec 2020 21:55:11 +0300 Subject: [PATCH 267/284] Remove useless support for symbolic port names --- src/Common/DNSResolver.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Common/DNSResolver.cpp b/src/Common/DNSResolver.cpp index 9059d2838bb..f54d3e72df5 100644 --- a/src/Common/DNSResolver.cpp +++ b/src/Common/DNSResolver.cpp @@ -80,13 +80,7 @@ static void splitHostAndPort(const std::string & host_and_port, std::string & ou out_port = static_cast(port); } else - { - struct servent * se = getservbyname(port_str.c_str(), nullptr); - if (se) - out_port = ntohs(static_cast(se->s_port)); - else - throw Exception("Service not found", ErrorCodes::BAD_ARGUMENTS); - } + throw Exception("Port must be numeric", ErrorCodes::BAD_ARGUMENTS); } static DNSResolver::IPAddresses resolveIPAddressImpl(const std::string & host) From 611caf177886a4b5b37fbd274f945e9db5a62865 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 30 Dec 2020 22:12:24 +0300 Subject: [PATCH 268/284] Send fatal logs in all tests --- programs/client/Client.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index a8c4f070bea..2cdb1881685 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -948,7 +948,7 @@ private: { /// disable logs if expects errors TestHint test_hint(test_mode, all_queries_text); if (test_hint.clientError() || test_hint.serverError()) - processTextAsSingleQuery("SET send_logs_level = 'none'"); + processTextAsSingleQuery("SET send_logs_level = 'fatal'"); // Echo all queries if asked; makes for a more readable reference // file. From dc320ec68f991b51ff899046add900aa4ec4291a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 30 Dec 2020 23:35:16 +0300 Subject: [PATCH 269/284] Remove fa and tr docs --- docs/fa/commercial/cloud.md | 23 - docs/fa/commercial/index.md | 9 - docs/fa/commercial/support.md | 23 - docs/fa/development/architecture.md | 204 -- docs/fa/development/browse-code.md | 14 - docs/fa/development/build-cross-arm.md | 45 - docs/fa/development/build-cross-osx.md | 67 - docs/fa/development/build-osx.md | 95 - docs/fa/development/build.md | 142 -- docs/fa/development/contrib.md | 43 - docs/fa/development/developer-instruction.md | 289 --- docs/fa/development/index.md | 12 - docs/fa/development/style.md | 842 -------- docs/fa/development/tests.md | 1 - docs/fa/engines/database-engines/index.md | 22 - docs/fa/engines/database-engines/lazy.md | 18 - docs/fa/engines/database-engines/mysql.md | 135 -- docs/fa/engines/index.md | 8 - docs/fa/engines/table-engines/index.md | 86 - .../table-engines/integrations/hdfs.md | 123 -- .../table-engines/integrations/index.md | 8 - .../table-engines/integrations/jdbc.md | 90 - .../table-engines/integrations/kafka.md | 180 -- .../table-engines/integrations/mysql.md | 105 - .../table-engines/integrations/odbc.md | 132 -- .../engines/table-engines/log-family/index.md | 49 - .../engines/table-engines/log-family/log.md | 16 - .../table-engines/log-family/stripelog.md | 95 - .../table-engines/log-family/tinylog.md | 16 - .../mergetree-family/aggregatingmergetree.md | 105 - .../mergetree-family/collapsingmergetree.md | 306 --- .../custom-partitioning-key.md | 128 -- .../mergetree-family/graphitemergetree.md | 174 -- .../table-engines/mergetree-family/index.md | 9 - .../mergetree-family/mergetree.md | 654 ------ .../mergetree-family/replacingmergetree.md | 69 - .../mergetree-family/replication.md | 218 -- .../mergetree-family/summingmergetree.md | 141 -- .../versionedcollapsingmergetree.md | 239 -- .../engines/table-engines/special/buffer.md | 71 - .../table-engines/special/dictionary.md | 97 - .../table-engines/special/distributed.md | 152 -- .../table-engines/special/external-data.md | 68 - docs/fa/engines/table-engines/special/file.md | 90 - .../engines/table-engines/special/generate.md | 61 - .../fa/engines/table-engines/special/index.md | 8 - docs/fa/engines/table-engines/special/join.md | 111 - .../table-engines/special/materializedview.md | 12 - .../engines/table-engines/special/memory.md | 19 - .../fa/engines/table-engines/special/merge.md | 70 - docs/fa/engines/table-engines/special/null.md | 14 - docs/fa/engines/table-engines/special/set.md | 19 - docs/fa/engines/table-engines/special/url.md | 82 - docs/fa/engines/table-engines/special/view.md | 12 - docs/fa/faq/general.md | 60 - docs/fa/faq/index.md | 8 - .../example-datasets/amplab-benchmark.md | 131 -- .../example-datasets/criteo.md | 83 - .../getting-started/example-datasets/index.md | 23 - .../example-datasets/metrica.md | 71 - .../example-datasets/nyc-taxi.md | 391 ---- .../example-datasets/ontime.md | 412 ---- .../example-datasets/star-schema.md | 371 ---- .../example-datasets/wikistat.md | 35 - docs/fa/getting-started/index.md | 17 - docs/fa/getting-started/install.md | 183 -- docs/fa/getting-started/playground.md | 48 - docs/fa/getting-started/tutorial.md | 666 ------ docs/fa/guides/apply-catboost-model.md | 241 --- docs/fa/guides/index.md | 16 - docs/fa/images/column-oriented.gif | Bin 43771 -> 0 bytes docs/fa/images/logo.svg | 1 - docs/fa/images/row-oriented.gif | Bin 39281 -> 0 bytes docs/fa/index.md | 98 - docs/fa/interfaces/cli.md | 149 -- docs/fa/interfaces/cpp.md | 13 - docs/fa/interfaces/formats.md | 1213 ----------- docs/fa/interfaces/http.md | 617 ------ docs/fa/interfaces/index.md | 29 - docs/fa/interfaces/jdbc.md | 15 - docs/fa/interfaces/mysql.md | 49 - docs/fa/interfaces/odbc.md | 12 - docs/fa/interfaces/tcp.md | 13 - .../third-party/client-libraries.md | 62 - docs/fa/interfaces/third-party/gui.md | 156 -- docs/fa/interfaces/third-party/index.md | 8 - .../fa/interfaces/third-party/integrations.md | 110 - docs/fa/interfaces/third-party/proxy.md | 46 - docs/fa/introduction/adopters.md | 86 - docs/fa/introduction/distinctive-features.md | 73 - docs/fa/introduction/history.md | 56 - docs/fa/introduction/index.md | 8 - docs/fa/introduction/performance.md | 29 - docs/fa/operations/access-rights.md | 144 -- docs/fa/operations/backup.md | 42 - docs/fa/operations/configuration-files.md | 58 - docs/fa/operations/index.md | 28 - docs/fa/operations/monitoring.md | 46 - .../optimizing-performance/index.md | 9 - .../sampling-query-profiler.md | 65 - docs/fa/operations/performance-test.md | 82 - docs/fa/operations/quotas.md | 112 - docs/fa/operations/requirements.md | 61 - .../server-configuration-parameters/index.md | 20 - .../settings.md | 907 -------- .../settings/constraints-on-settings.md | 76 - docs/fa/operations/settings/index.md | 33 - .../settings/permissions-for-queries.md | 62 - .../operations/settings/query-complexity.md | 302 --- .../operations/settings/settings-profiles.md | 82 - docs/fa/operations/settings/settings-users.md | 164 -- docs/fa/operations/settings/settings.md | 1254 ----------- docs/fa/operations/system-tables.md | 1168 ---------- docs/fa/operations/tips.md | 252 --- docs/fa/operations/troubleshooting.md | 146 -- docs/fa/operations/update.md | 21 - .../utilities/clickhouse-benchmark.md | 156 -- .../operations/utilities/clickhouse-copier.md | 176 -- .../operations/utilities/clickhouse-local.md | 81 - docs/fa/operations/utilities/index.md | 15 - .../aggregate-functions/combinators.md | 245 --- .../aggregate-functions/index.md | 62 - .../parametric-functions.md | 499 ----- .../aggregate-functions/reference.md | 1914 ----------------- docs/fa/sql-reference/ansi.md | 180 -- .../data-types/aggregatefunction.md | 71 - docs/fa/sql-reference/data-types/array.md | 77 - docs/fa/sql-reference/data-types/boolean.md | 12 - docs/fa/sql-reference/data-types/date.md | 15 - docs/fa/sql-reference/data-types/datetime.md | 129 -- .../fa/sql-reference/data-types/datetime64.md | 104 - docs/fa/sql-reference/data-types/decimal.md | 109 - .../sql-reference/data-types/domains/index.md | 33 - .../sql-reference/data-types/domains/ipv4.md | 84 - .../sql-reference/data-types/domains/ipv6.md | 86 - docs/fa/sql-reference/data-types/enum.md | 132 -- .../sql-reference/data-types/fixedstring.md | 63 - docs/fa/sql-reference/data-types/float.md | 87 - docs/fa/sql-reference/data-types/index.md | 15 - docs/fa/sql-reference/data-types/int-uint.md | 26 - .../nested-data-structures/index.md | 13 - .../nested-data-structures/nested.md | 106 - docs/fa/sql-reference/data-types/nullable.md | 46 - .../data-types/simpleaggregatefunction.md | 38 - .../special-data-types/expression.md | 12 - .../data-types/special-data-types/index.md | 15 - .../data-types/special-data-types/interval.md | 85 - .../data-types/special-data-types/nothing.md | 26 - .../data-types/special-data-types/set.md | 12 - docs/fa/sql-reference/data-types/string.md | 20 - docs/fa/sql-reference/data-types/tuple.md | 52 - docs/fa/sql-reference/data-types/uuid.md | 77 - .../external-dicts-dict-hierarchical.md | 71 - .../external-dicts-dict-layout.md | 397 ---- .../external-dicts-dict-lifetime.md | 92 - .../external-dicts-dict-sources.md | 631 ------ .../external-dicts-dict-structure.md | 176 -- .../external-dicts-dict.md | 54 - .../external-dictionaries/external-dicts.md | 62 - .../external-dictionaries/index.md | 9 - docs/fa/sql-reference/dictionaries/index.md | 22 - .../dictionaries/internal-dicts.md | 56 - .../functions/arithmetic-functions.md | 87 - .../functions/array-functions.md | 1061 --------- docs/fa/sql-reference/functions/array-join.md | 37 - .../sql-reference/functions/bit-functions.md | 255 --- .../functions/bitmap-functions.md | 496 ----- .../functions/comparison-functions.md | 37 - .../functions/conditional-functions.md | 207 -- .../functions/date-time-functions.md | 451 ---- .../functions/encoding-functions.md | 175 -- .../functions/ext-dict-functions.md | 206 -- .../functions/functions-for-nulls.md | 313 --- docs/fa/sql-reference/functions/geo.md | 511 ----- .../sql-reference/functions/hash-functions.md | 484 ----- .../functions/higher-order-functions.md | 264 --- .../sql-reference/functions/in-functions.md | 27 - docs/fa/sql-reference/functions/index.md | 74 - .../sql-reference/functions/introspection.md | 310 --- .../functions/ip-address-functions.md | 249 --- .../sql-reference/functions/json-functions.md | 297 --- .../functions/logical-functions.md | 22 - .../functions/machine-learning-functions.md | 21 - .../sql-reference/functions/math-functions.md | 116 - .../functions/other-functions.md | 1205 ----------- .../functions/random-functions.md | 66 - .../functions/rounding-functions.md | 190 -- .../functions/splitting-merging-functions.md | 117 - .../functions/string-functions.md | 489 ----- .../functions/string-replace-functions.md | 95 - .../functions/string-search-functions.md | 384 ---- .../functions/type-conversion-functions.md | 534 ----- .../sql-reference/functions/url-functions.md | 210 -- .../sql-reference/functions/uuid-functions.md | 122 -- .../functions/ym-dict-functions.md | 157 -- docs/fa/sql-reference/index.md | 20 - docs/fa/sql-reference/operators/in.md | 204 -- docs/fa/sql-reference/operators/index.md | 277 --- docs/fa/sql-reference/statements/alter.md | 602 ------ docs/fa/sql-reference/statements/create.md | 502 ----- docs/fa/sql-reference/statements/grant.md | 476 ---- docs/fa/sql-reference/statements/index.md | 8 - .../sql-reference/statements/insert-into.md | 80 - docs/fa/sql-reference/statements/misc.md | 358 --- docs/fa/sql-reference/statements/revoke.md | 50 - .../statements/select/array-join.md | 282 --- .../statements/select/distinct.md | 63 - .../sql-reference/statements/select/format.md | 18 - .../sql-reference/statements/select/from.md | 44 - .../statements/select/group-by.md | 132 -- .../sql-reference/statements/select/having.md | 14 - .../sql-reference/statements/select/index.md | 158 -- .../statements/select/into-outfile.md | 14 - .../sql-reference/statements/select/join.md | 188 -- .../statements/select/limit-by.md | 71 - .../sql-reference/statements/select/limit.md | 14 - .../statements/select/order-by.md | 72 - .../statements/select/prewhere.md | 22 - .../sql-reference/statements/select/sample.md | 113 - .../sql-reference/statements/select/union.md | 35 - .../sql-reference/statements/select/where.md | 15 - .../sql-reference/statements/select/with.md | 80 - docs/fa/sql-reference/statements/show.md | 169 -- docs/fa/sql-reference/statements/system.md | 113 - docs/fa/sql-reference/syntax.md | 187 -- docs/fa/sql-reference/table-functions/file.md | 121 -- .../sql-reference/table-functions/generate.md | 44 - docs/fa/sql-reference/table-functions/hdfs.md | 104 - .../fa/sql-reference/table-functions/index.md | 38 - .../fa/sql-reference/table-functions/input.md | 47 - docs/fa/sql-reference/table-functions/jdbc.md | 29 - .../fa/sql-reference/table-functions/merge.md | 14 - .../fa/sql-reference/table-functions/mysql.md | 86 - .../sql-reference/table-functions/numbers.md | 30 - docs/fa/sql-reference/table-functions/odbc.md | 108 - .../sql-reference/table-functions/remote.md | 85 - docs/fa/sql-reference/table-functions/url.md | 26 - docs/fa/whats-new/changelog/2017.md | 1 - docs/fa/whats-new/changelog/2018.md | 1 - docs/fa/whats-new/changelog/2019.md | 1 - docs/fa/whats-new/changelog/index.md | 1 - docs/fa/whats-new/index.md | 8 - docs/fa/whats-new/roadmap.md | 19 - docs/fa/whats-new/security-changelog.md | 77 - docs/tr/commercial/cloud.md | 23 - docs/tr/commercial/index.md | 9 - docs/tr/commercial/support.md | 1 - docs/tr/development/architecture.md | 203 -- docs/tr/development/browse-code.md | 14 - docs/tr/development/build-cross-arm.md | 43 - docs/tr/development/build-cross-osx.md | 64 - docs/tr/development/build-osx.md | 93 - docs/tr/development/build.md | 141 -- docs/tr/development/contrib.md | 41 - docs/tr/development/developer-instruction.md | 287 --- docs/tr/development/index.md | 12 - docs/tr/development/style.md | 841 -------- docs/tr/development/tests.md | 1 - docs/tr/engines/database-engines/index.md | 21 - docs/tr/engines/database-engines/lazy.md | 18 - docs/tr/engines/database-engines/mysql.md | 135 -- docs/tr/engines/index.md | 8 - docs/tr/engines/table-engines/index.md | 85 - .../table-engines/integrations/hdfs.md | 123 -- .../table-engines/integrations/index.md | 8 - .../table-engines/integrations/jdbc.md | 90 - .../table-engines/integrations/kafka.md | 180 -- .../table-engines/integrations/mysql.md | 105 - .../table-engines/integrations/odbc.md | 132 -- .../engines/table-engines/log-family/index.md | 47 - .../engines/table-engines/log-family/log.md | 16 - .../table-engines/log-family/stripelog.md | 95 - .../table-engines/log-family/tinylog.md | 16 - .../mergetree-family/aggregatingmergetree.md | 105 - .../mergetree-family/collapsingmergetree.md | 306 --- .../custom-partitioning-key.md | 127 -- .../mergetree-family/graphitemergetree.md | 174 -- .../table-engines/mergetree-family/index.md | 8 - .../mergetree-family/mergetree.md | 654 ------ .../mergetree-family/replacingmergetree.md | 69 - .../mergetree-family/replication.md | 218 -- .../mergetree-family/summingmergetree.md | 141 -- .../versionedcollapsingmergetree.md | 238 -- .../engines/table-engines/special/buffer.md | 71 - .../table-engines/special/dictionary.md | 97 - .../table-engines/special/distributed.md | 152 -- .../table-engines/special/external-data.md | 68 - docs/tr/engines/table-engines/special/file.md | 90 - .../engines/table-engines/special/generate.md | 61 - .../tr/engines/table-engines/special/index.md | 8 - docs/tr/engines/table-engines/special/join.md | 111 - .../table-engines/special/materializedview.md | 12 - .../engines/table-engines/special/memory.md | 19 - .../tr/engines/table-engines/special/merge.md | 70 - docs/tr/engines/table-engines/special/null.md | 14 - docs/tr/engines/table-engines/special/set.md | 19 - docs/tr/engines/table-engines/special/url.md | 82 - docs/tr/engines/table-engines/special/view.md | 12 - docs/tr/faq/general.md | 60 - docs/tr/faq/index.md | 8 - .../example-datasets/amplab-benchmark.md | 129 -- .../example-datasets/criteo.md | 81 - .../getting-started/example-datasets/index.md | 22 - .../example-datasets/metrica.md | 70 - .../example-datasets/nyc-taxi.md | 390 ---- .../example-datasets/ontime.md | 412 ---- .../example-datasets/star-schema.md | 370 ---- .../example-datasets/wikistat.md | 35 - docs/tr/getting-started/index.md | 17 - docs/tr/getting-started/install.md | 182 -- docs/tr/getting-started/playground.md | 48 - docs/tr/getting-started/tutorial.md | 666 ------ docs/tr/guides/apply-catboost-model.md | 239 -- docs/tr/guides/index.md | 16 - docs/tr/images | 1 - docs/tr/index.md | 95 - docs/tr/interfaces/cli.md | 149 -- docs/tr/interfaces/cpp.md | 12 - docs/tr/interfaces/formats.md | 1212 ----------- docs/tr/interfaces/http.md | 617 ------ docs/tr/interfaces/index.md | 29 - docs/tr/interfaces/jdbc.md | 15 - docs/tr/interfaces/mysql.md | 49 - docs/tr/interfaces/odbc.md | 12 - docs/tr/interfaces/tcp.md | 12 - .../third-party/client-libraries.md | 61 - docs/tr/interfaces/third-party/gui.md | 156 -- docs/tr/interfaces/third-party/index.md | 8 - .../tr/interfaces/third-party/integrations.md | 110 - docs/tr/interfaces/third-party/proxy.md | 46 - docs/tr/introduction/adopters.md | 86 - docs/tr/introduction/distinctive-features.md | 77 - docs/tr/introduction/history.md | 56 - docs/tr/introduction/index.md | 8 - docs/tr/introduction/performance.md | 32 - docs/tr/operations/access-rights.md | 143 -- docs/tr/operations/backup.md | 41 - docs/tr/operations/configuration-files.md | 57 - docs/tr/operations/index.md | 28 - docs/tr/operations/monitoring.md | 46 - .../optimizing-performance/index.md | 8 - .../sampling-query-profiler.md | 64 - docs/tr/operations/performance-test.md | 82 - docs/tr/operations/quotas.md | 112 - docs/tr/operations/requirements.md | 61 - .../server-configuration-parameters/index.md | 19 - .../settings.md | 906 -------- .../settings/constraints-on-settings.md | 75 - docs/tr/operations/settings/index.md | 33 - .../settings/permissions-for-queries.md | 61 - .../operations/settings/query-complexity.md | 302 --- .../operations/settings/settings-profiles.md | 81 - docs/tr/operations/settings/settings-users.md | 164 -- docs/tr/operations/settings/settings.md | 1254 ----------- docs/tr/operations/system-tables.md | 1168 ---------- docs/tr/operations/tips.md | 251 --- docs/tr/operations/troubleshooting.md | 146 -- docs/tr/operations/update.md | 20 - .../utilities/clickhouse-benchmark.md | 156 -- .../operations/utilities/clickhouse-copier.md | 176 -- .../operations/utilities/clickhouse-local.md | 81 - docs/tr/operations/utilities/index.md | 15 - .../aggregate-functions/combinators.md | 245 --- .../aggregate-functions/index.md | 62 - .../parametric-functions.md | 499 ----- .../aggregate-functions/reference.md | 1914 ----------------- docs/tr/sql-reference/ansi.md | 180 -- .../data-types/aggregatefunction.md | 70 - docs/tr/sql-reference/data-types/array.md | 77 - docs/tr/sql-reference/data-types/boolean.md | 12 - docs/tr/sql-reference/data-types/date.md | 14 - docs/tr/sql-reference/data-types/datetime.md | 129 -- .../tr/sql-reference/data-types/datetime64.md | 104 - docs/tr/sql-reference/data-types/decimal.md | 109 - .../sql-reference/data-types/domains/index.md | 34 - .../sql-reference/data-types/domains/ipv4.md | 84 - .../sql-reference/data-types/domains/ipv6.md | 86 - docs/tr/sql-reference/data-types/enum.md | 132 -- .../sql-reference/data-types/fixedstring.md | 63 - docs/tr/sql-reference/data-types/float.md | 87 - docs/tr/sql-reference/data-types/index.md | 15 - docs/tr/sql-reference/data-types/int-uint.md | 26 - .../data-types/lowcardinality.md | 1 - .../nested-data-structures/index.md | 12 - .../nested-data-structures/nested.md | 106 - docs/tr/sql-reference/data-types/nullable.md | 46 - .../data-types/simpleaggregatefunction.md | 40 - .../special-data-types/expression.md | 12 - .../data-types/special-data-types/index.md | 14 - .../data-types/special-data-types/interval.md | 85 - .../data-types/special-data-types/nothing.md | 26 - .../data-types/special-data-types/set.md | 12 - docs/tr/sql-reference/data-types/string.md | 20 - docs/tr/sql-reference/data-types/tuple.md | 52 - docs/tr/sql-reference/data-types/uuid.md | 77 - .../external-dicts-dict-hierarchical.md | 70 - .../external-dicts-dict-layout.md | 396 ---- .../external-dicts-dict-lifetime.md | 91 - .../external-dicts-dict-sources.md | 630 ------ .../external-dicts-dict-structure.md | 175 -- .../external-dicts-dict.md | 53 - .../external-dictionaries/external-dicts.md | 62 - .../external-dictionaries/index.md | 8 - docs/tr/sql-reference/dictionaries/index.md | 22 - .../dictionaries/internal-dicts.md | 55 - .../functions/arithmetic-functions.md | 87 - .../functions/array-functions.md | 1061 --------- docs/tr/sql-reference/functions/array-join.md | 37 - .../sql-reference/functions/bit-functions.md | 255 --- .../functions/bitmap-functions.md | 496 ----- .../functions/comparison-functions.md | 37 - .../functions/conditional-functions.md | 207 -- .../functions/date-time-functions.md | 450 ---- .../functions/encoding-functions.md | 175 -- .../functions/ext-dict-functions.md | 205 -- .../functions/functions-for-nulls.md | 312 --- docs/tr/sql-reference/functions/geo.md | 510 ----- .../sql-reference/functions/hash-functions.md | 484 ----- .../functions/higher-order-functions.md | 264 --- .../sql-reference/functions/in-functions.md | 26 - docs/tr/sql-reference/functions/index.md | 74 - .../sql-reference/functions/introspection.md | 310 --- .../functions/ip-address-functions.md | 248 --- .../sql-reference/functions/json-functions.md | 297 --- .../functions/logical-functions.md | 22 - .../functions/machine-learning-functions.md | 20 - .../sql-reference/functions/math-functions.md | 116 - .../functions/other-functions.md | 1205 ----------- .../functions/random-functions.md | 65 - .../functions/rounding-functions.md | 190 -- .../functions/splitting-merging-functions.md | 116 - .../functions/string-functions.md | 489 ----- .../functions/string-replace-functions.md | 94 - .../functions/string-search-functions.md | 383 ---- .../functions/type-conversion-functions.md | 534 ----- .../sql-reference/functions/url-functions.md | 209 -- .../sql-reference/functions/uuid-functions.md | 122 -- .../functions/ym-dict-functions.md | 155 -- docs/tr/sql-reference/index.md | 20 - docs/tr/sql-reference/operators/in.md | 1 - docs/tr/sql-reference/operators/index.md | 277 --- docs/tr/sql-reference/statements/alter.md | 602 ------ docs/tr/sql-reference/statements/create.md | 502 ----- docs/tr/sql-reference/statements/grant.md | 476 ---- docs/tr/sql-reference/statements/index.md | 8 - .../sql-reference/statements/insert-into.md | 80 - docs/tr/sql-reference/statements/misc.md | 358 --- docs/tr/sql-reference/statements/revoke.md | 1 - .../statements/select/array-join.md | 1 - .../statements/select/distinct.md | 1 - .../sql-reference/statements/select/format.md | 1 - .../sql-reference/statements/select/from.md | 1 - .../statements/select/group-by.md | 1 - .../sql-reference/statements/select/having.md | 1 - .../sql-reference/statements/select/index.md | 1 - .../statements/select/into-outfile.md | 1 - .../sql-reference/statements/select/join.md | 1 - .../statements/select/limit-by.md | 1 - .../sql-reference/statements/select/limit.md | 1 - .../statements/select/order-by.md | 1 - .../statements/select/prewhere.md | 1 - .../sql-reference/statements/select/sample.md | 1 - .../sql-reference/statements/select/union.md | 1 - .../sql-reference/statements/select/where.md | 1 - .../sql-reference/statements/select/with.md | 1 - docs/tr/sql-reference/statements/show.md | 169 -- docs/tr/sql-reference/statements/system.md | 113 - docs/tr/sql-reference/syntax.md | 187 -- docs/tr/sql-reference/table-functions/file.md | 121 -- .../sql-reference/table-functions/generate.md | 44 - docs/tr/sql-reference/table-functions/hdfs.md | 104 - .../tr/sql-reference/table-functions/index.md | 38 - .../tr/sql-reference/table-functions/input.md | 47 - docs/tr/sql-reference/table-functions/jdbc.md | 29 - .../tr/sql-reference/table-functions/merge.md | 14 - .../tr/sql-reference/table-functions/mysql.md | 86 - .../sql-reference/table-functions/numbers.md | 30 - docs/tr/sql-reference/table-functions/odbc.md | 108 - .../sql-reference/table-functions/remote.md | 85 - docs/tr/sql-reference/table-functions/url.md | 26 - docs/tr/whats-new/changelog/2017.md | 1 - docs/tr/whats-new/changelog/2018.md | 1 - docs/tr/whats-new/changelog/2019.md | 1 - docs/tr/whats-new/changelog/index.md | 1 - docs/tr/whats-new/index.md | 8 - docs/tr/whats-new/roadmap.md | 19 - docs/tr/whats-new/security-changelog.md | 76 - 487 files changed, 73315 deletions(-) delete mode 100644 docs/fa/commercial/cloud.md delete mode 100644 docs/fa/commercial/index.md delete mode 100644 docs/fa/commercial/support.md delete mode 100644 docs/fa/development/architecture.md delete mode 100644 docs/fa/development/browse-code.md delete mode 100644 docs/fa/development/build-cross-arm.md delete mode 100644 docs/fa/development/build-cross-osx.md delete mode 100644 docs/fa/development/build-osx.md delete mode 100644 docs/fa/development/build.md delete mode 100644 docs/fa/development/contrib.md delete mode 100644 docs/fa/development/developer-instruction.md delete mode 100644 docs/fa/development/index.md delete mode 100644 docs/fa/development/style.md delete mode 120000 docs/fa/development/tests.md delete mode 100644 docs/fa/engines/database-engines/index.md delete mode 100644 docs/fa/engines/database-engines/lazy.md delete mode 100644 docs/fa/engines/database-engines/mysql.md delete mode 100644 docs/fa/engines/index.md delete mode 100644 docs/fa/engines/table-engines/index.md delete mode 100644 docs/fa/engines/table-engines/integrations/hdfs.md delete mode 100644 docs/fa/engines/table-engines/integrations/index.md delete mode 100644 docs/fa/engines/table-engines/integrations/jdbc.md delete mode 100644 docs/fa/engines/table-engines/integrations/kafka.md delete mode 100644 docs/fa/engines/table-engines/integrations/mysql.md delete mode 100644 docs/fa/engines/table-engines/integrations/odbc.md delete mode 100644 docs/fa/engines/table-engines/log-family/index.md delete mode 100644 docs/fa/engines/table-engines/log-family/log.md delete mode 100644 docs/fa/engines/table-engines/log-family/stripelog.md delete mode 100644 docs/fa/engines/table-engines/log-family/tinylog.md delete mode 100644 docs/fa/engines/table-engines/mergetree-family/aggregatingmergetree.md delete mode 100644 docs/fa/engines/table-engines/mergetree-family/collapsingmergetree.md delete mode 100644 docs/fa/engines/table-engines/mergetree-family/custom-partitioning-key.md delete mode 100644 docs/fa/engines/table-engines/mergetree-family/graphitemergetree.md delete mode 100644 docs/fa/engines/table-engines/mergetree-family/index.md delete mode 100644 docs/fa/engines/table-engines/mergetree-family/mergetree.md delete mode 100644 docs/fa/engines/table-engines/mergetree-family/replacingmergetree.md delete mode 100644 docs/fa/engines/table-engines/mergetree-family/replication.md delete mode 100644 docs/fa/engines/table-engines/mergetree-family/summingmergetree.md delete mode 100644 docs/fa/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md delete mode 100644 docs/fa/engines/table-engines/special/buffer.md delete mode 100644 docs/fa/engines/table-engines/special/dictionary.md delete mode 100644 docs/fa/engines/table-engines/special/distributed.md delete mode 100644 docs/fa/engines/table-engines/special/external-data.md delete mode 100644 docs/fa/engines/table-engines/special/file.md delete mode 100644 docs/fa/engines/table-engines/special/generate.md delete mode 100644 docs/fa/engines/table-engines/special/index.md delete mode 100644 docs/fa/engines/table-engines/special/join.md delete mode 100644 docs/fa/engines/table-engines/special/materializedview.md delete mode 100644 docs/fa/engines/table-engines/special/memory.md delete mode 100644 docs/fa/engines/table-engines/special/merge.md delete mode 100644 docs/fa/engines/table-engines/special/null.md delete mode 100644 docs/fa/engines/table-engines/special/set.md delete mode 100644 docs/fa/engines/table-engines/special/url.md delete mode 100644 docs/fa/engines/table-engines/special/view.md delete mode 100644 docs/fa/faq/general.md delete mode 100644 docs/fa/faq/index.md delete mode 100644 docs/fa/getting-started/example-datasets/amplab-benchmark.md delete mode 100644 docs/fa/getting-started/example-datasets/criteo.md delete mode 100644 docs/fa/getting-started/example-datasets/index.md delete mode 100644 docs/fa/getting-started/example-datasets/metrica.md delete mode 100644 docs/fa/getting-started/example-datasets/nyc-taxi.md delete mode 100644 docs/fa/getting-started/example-datasets/ontime.md delete mode 100644 docs/fa/getting-started/example-datasets/star-schema.md delete mode 100644 docs/fa/getting-started/example-datasets/wikistat.md delete mode 100644 docs/fa/getting-started/index.md delete mode 100644 docs/fa/getting-started/install.md delete mode 100644 docs/fa/getting-started/playground.md delete mode 100644 docs/fa/getting-started/tutorial.md delete mode 100644 docs/fa/guides/apply-catboost-model.md delete mode 100644 docs/fa/guides/index.md delete mode 100644 docs/fa/images/column-oriented.gif delete mode 100644 docs/fa/images/logo.svg delete mode 100644 docs/fa/images/row-oriented.gif delete mode 100644 docs/fa/index.md delete mode 100644 docs/fa/interfaces/cli.md delete mode 100644 docs/fa/interfaces/cpp.md delete mode 100644 docs/fa/interfaces/formats.md delete mode 100644 docs/fa/interfaces/http.md delete mode 100644 docs/fa/interfaces/index.md delete mode 100644 docs/fa/interfaces/jdbc.md delete mode 100644 docs/fa/interfaces/mysql.md delete mode 100644 docs/fa/interfaces/odbc.md delete mode 100644 docs/fa/interfaces/tcp.md delete mode 100644 docs/fa/interfaces/third-party/client-libraries.md delete mode 100644 docs/fa/interfaces/third-party/gui.md delete mode 100644 docs/fa/interfaces/third-party/index.md delete mode 100644 docs/fa/interfaces/third-party/integrations.md delete mode 100644 docs/fa/interfaces/third-party/proxy.md delete mode 100644 docs/fa/introduction/adopters.md delete mode 100644 docs/fa/introduction/distinctive-features.md delete mode 100644 docs/fa/introduction/history.md delete mode 100644 docs/fa/introduction/index.md delete mode 100644 docs/fa/introduction/performance.md delete mode 100644 docs/fa/operations/access-rights.md delete mode 100644 docs/fa/operations/backup.md delete mode 100644 docs/fa/operations/configuration-files.md delete mode 100644 docs/fa/operations/index.md delete mode 100644 docs/fa/operations/monitoring.md delete mode 100644 docs/fa/operations/optimizing-performance/index.md delete mode 100644 docs/fa/operations/optimizing-performance/sampling-query-profiler.md delete mode 100644 docs/fa/operations/performance-test.md delete mode 100644 docs/fa/operations/quotas.md delete mode 100644 docs/fa/operations/requirements.md delete mode 100644 docs/fa/operations/server-configuration-parameters/index.md delete mode 100644 docs/fa/operations/server-configuration-parameters/settings.md delete mode 100644 docs/fa/operations/settings/constraints-on-settings.md delete mode 100644 docs/fa/operations/settings/index.md delete mode 100644 docs/fa/operations/settings/permissions-for-queries.md delete mode 100644 docs/fa/operations/settings/query-complexity.md delete mode 100644 docs/fa/operations/settings/settings-profiles.md delete mode 100644 docs/fa/operations/settings/settings-users.md delete mode 100644 docs/fa/operations/settings/settings.md delete mode 100644 docs/fa/operations/system-tables.md delete mode 100644 docs/fa/operations/tips.md delete mode 100644 docs/fa/operations/troubleshooting.md delete mode 100644 docs/fa/operations/update.md delete mode 100644 docs/fa/operations/utilities/clickhouse-benchmark.md delete mode 100644 docs/fa/operations/utilities/clickhouse-copier.md delete mode 100644 docs/fa/operations/utilities/clickhouse-local.md delete mode 100644 docs/fa/operations/utilities/index.md delete mode 100644 docs/fa/sql-reference/aggregate-functions/combinators.md delete mode 100644 docs/fa/sql-reference/aggregate-functions/index.md delete mode 100644 docs/fa/sql-reference/aggregate-functions/parametric-functions.md delete mode 100644 docs/fa/sql-reference/aggregate-functions/reference.md delete mode 100644 docs/fa/sql-reference/ansi.md delete mode 100644 docs/fa/sql-reference/data-types/aggregatefunction.md delete mode 100644 docs/fa/sql-reference/data-types/array.md delete mode 100644 docs/fa/sql-reference/data-types/boolean.md delete mode 100644 docs/fa/sql-reference/data-types/date.md delete mode 100644 docs/fa/sql-reference/data-types/datetime.md delete mode 100644 docs/fa/sql-reference/data-types/datetime64.md delete mode 100644 docs/fa/sql-reference/data-types/decimal.md delete mode 100644 docs/fa/sql-reference/data-types/domains/index.md delete mode 100644 docs/fa/sql-reference/data-types/domains/ipv4.md delete mode 100644 docs/fa/sql-reference/data-types/domains/ipv6.md delete mode 100644 docs/fa/sql-reference/data-types/enum.md delete mode 100644 docs/fa/sql-reference/data-types/fixedstring.md delete mode 100644 docs/fa/sql-reference/data-types/float.md delete mode 100644 docs/fa/sql-reference/data-types/index.md delete mode 100644 docs/fa/sql-reference/data-types/int-uint.md delete mode 100644 docs/fa/sql-reference/data-types/nested-data-structures/index.md delete mode 100644 docs/fa/sql-reference/data-types/nested-data-structures/nested.md delete mode 100644 docs/fa/sql-reference/data-types/nullable.md delete mode 100644 docs/fa/sql-reference/data-types/simpleaggregatefunction.md delete mode 100644 docs/fa/sql-reference/data-types/special-data-types/expression.md delete mode 100644 docs/fa/sql-reference/data-types/special-data-types/index.md delete mode 100644 docs/fa/sql-reference/data-types/special-data-types/interval.md delete mode 100644 docs/fa/sql-reference/data-types/special-data-types/nothing.md delete mode 100644 docs/fa/sql-reference/data-types/special-data-types/set.md delete mode 100644 docs/fa/sql-reference/data-types/string.md delete mode 100644 docs/fa/sql-reference/data-types/tuple.md delete mode 100644 docs/fa/sql-reference/data-types/uuid.md delete mode 100644 docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md delete mode 100644 docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md delete mode 100644 docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md delete mode 100644 docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md delete mode 100644 docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md delete mode 100644 docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md delete mode 100644 docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts.md delete mode 100644 docs/fa/sql-reference/dictionaries/external-dictionaries/index.md delete mode 100644 docs/fa/sql-reference/dictionaries/index.md delete mode 100644 docs/fa/sql-reference/dictionaries/internal-dicts.md delete mode 100644 docs/fa/sql-reference/functions/arithmetic-functions.md delete mode 100644 docs/fa/sql-reference/functions/array-functions.md delete mode 100644 docs/fa/sql-reference/functions/array-join.md delete mode 100644 docs/fa/sql-reference/functions/bit-functions.md delete mode 100644 docs/fa/sql-reference/functions/bitmap-functions.md delete mode 100644 docs/fa/sql-reference/functions/comparison-functions.md delete mode 100644 docs/fa/sql-reference/functions/conditional-functions.md delete mode 100644 docs/fa/sql-reference/functions/date-time-functions.md delete mode 100644 docs/fa/sql-reference/functions/encoding-functions.md delete mode 100644 docs/fa/sql-reference/functions/ext-dict-functions.md delete mode 100644 docs/fa/sql-reference/functions/functions-for-nulls.md delete mode 100644 docs/fa/sql-reference/functions/geo.md delete mode 100644 docs/fa/sql-reference/functions/hash-functions.md delete mode 100644 docs/fa/sql-reference/functions/higher-order-functions.md delete mode 100644 docs/fa/sql-reference/functions/in-functions.md delete mode 100644 docs/fa/sql-reference/functions/index.md delete mode 100644 docs/fa/sql-reference/functions/introspection.md delete mode 100644 docs/fa/sql-reference/functions/ip-address-functions.md delete mode 100644 docs/fa/sql-reference/functions/json-functions.md delete mode 100644 docs/fa/sql-reference/functions/logical-functions.md delete mode 100644 docs/fa/sql-reference/functions/machine-learning-functions.md delete mode 100644 docs/fa/sql-reference/functions/math-functions.md delete mode 100644 docs/fa/sql-reference/functions/other-functions.md delete mode 100644 docs/fa/sql-reference/functions/random-functions.md delete mode 100644 docs/fa/sql-reference/functions/rounding-functions.md delete mode 100644 docs/fa/sql-reference/functions/splitting-merging-functions.md delete mode 100644 docs/fa/sql-reference/functions/string-functions.md delete mode 100644 docs/fa/sql-reference/functions/string-replace-functions.md delete mode 100644 docs/fa/sql-reference/functions/string-search-functions.md delete mode 100644 docs/fa/sql-reference/functions/type-conversion-functions.md delete mode 100644 docs/fa/sql-reference/functions/url-functions.md delete mode 100644 docs/fa/sql-reference/functions/uuid-functions.md delete mode 100644 docs/fa/sql-reference/functions/ym-dict-functions.md delete mode 100644 docs/fa/sql-reference/index.md delete mode 100644 docs/fa/sql-reference/operators/in.md delete mode 100644 docs/fa/sql-reference/operators/index.md delete mode 100644 docs/fa/sql-reference/statements/alter.md delete mode 100644 docs/fa/sql-reference/statements/create.md delete mode 100644 docs/fa/sql-reference/statements/grant.md delete mode 100644 docs/fa/sql-reference/statements/index.md delete mode 100644 docs/fa/sql-reference/statements/insert-into.md delete mode 100644 docs/fa/sql-reference/statements/misc.md delete mode 100644 docs/fa/sql-reference/statements/revoke.md delete mode 100644 docs/fa/sql-reference/statements/select/array-join.md delete mode 100644 docs/fa/sql-reference/statements/select/distinct.md delete mode 100644 docs/fa/sql-reference/statements/select/format.md delete mode 100644 docs/fa/sql-reference/statements/select/from.md delete mode 100644 docs/fa/sql-reference/statements/select/group-by.md delete mode 100644 docs/fa/sql-reference/statements/select/having.md delete mode 100644 docs/fa/sql-reference/statements/select/index.md delete mode 100644 docs/fa/sql-reference/statements/select/into-outfile.md delete mode 100644 docs/fa/sql-reference/statements/select/join.md delete mode 100644 docs/fa/sql-reference/statements/select/limit-by.md delete mode 100644 docs/fa/sql-reference/statements/select/limit.md delete mode 100644 docs/fa/sql-reference/statements/select/order-by.md delete mode 100644 docs/fa/sql-reference/statements/select/prewhere.md delete mode 100644 docs/fa/sql-reference/statements/select/sample.md delete mode 100644 docs/fa/sql-reference/statements/select/union.md delete mode 100644 docs/fa/sql-reference/statements/select/where.md delete mode 100644 docs/fa/sql-reference/statements/select/with.md delete mode 100644 docs/fa/sql-reference/statements/show.md delete mode 100644 docs/fa/sql-reference/statements/system.md delete mode 100644 docs/fa/sql-reference/syntax.md delete mode 100644 docs/fa/sql-reference/table-functions/file.md delete mode 100644 docs/fa/sql-reference/table-functions/generate.md delete mode 100644 docs/fa/sql-reference/table-functions/hdfs.md delete mode 100644 docs/fa/sql-reference/table-functions/index.md delete mode 100644 docs/fa/sql-reference/table-functions/input.md delete mode 100644 docs/fa/sql-reference/table-functions/jdbc.md delete mode 100644 docs/fa/sql-reference/table-functions/merge.md delete mode 100644 docs/fa/sql-reference/table-functions/mysql.md delete mode 100644 docs/fa/sql-reference/table-functions/numbers.md delete mode 100644 docs/fa/sql-reference/table-functions/odbc.md delete mode 100644 docs/fa/sql-reference/table-functions/remote.md delete mode 100644 docs/fa/sql-reference/table-functions/url.md delete mode 120000 docs/fa/whats-new/changelog/2017.md delete mode 120000 docs/fa/whats-new/changelog/2018.md delete mode 120000 docs/fa/whats-new/changelog/2019.md delete mode 120000 docs/fa/whats-new/changelog/index.md delete mode 100644 docs/fa/whats-new/index.md delete mode 100644 docs/fa/whats-new/roadmap.md delete mode 100644 docs/fa/whats-new/security-changelog.md delete mode 100644 docs/tr/commercial/cloud.md delete mode 100644 docs/tr/commercial/index.md delete mode 120000 docs/tr/commercial/support.md delete mode 100644 docs/tr/development/architecture.md delete mode 100644 docs/tr/development/browse-code.md delete mode 100644 docs/tr/development/build-cross-arm.md delete mode 100644 docs/tr/development/build-cross-osx.md delete mode 100644 docs/tr/development/build-osx.md delete mode 100644 docs/tr/development/build.md delete mode 100644 docs/tr/development/contrib.md delete mode 100644 docs/tr/development/developer-instruction.md delete mode 100644 docs/tr/development/index.md delete mode 100644 docs/tr/development/style.md delete mode 120000 docs/tr/development/tests.md delete mode 100644 docs/tr/engines/database-engines/index.md delete mode 100644 docs/tr/engines/database-engines/lazy.md delete mode 100644 docs/tr/engines/database-engines/mysql.md delete mode 100644 docs/tr/engines/index.md delete mode 100644 docs/tr/engines/table-engines/index.md delete mode 100644 docs/tr/engines/table-engines/integrations/hdfs.md delete mode 100644 docs/tr/engines/table-engines/integrations/index.md delete mode 100644 docs/tr/engines/table-engines/integrations/jdbc.md delete mode 100644 docs/tr/engines/table-engines/integrations/kafka.md delete mode 100644 docs/tr/engines/table-engines/integrations/mysql.md delete mode 100644 docs/tr/engines/table-engines/integrations/odbc.md delete mode 100644 docs/tr/engines/table-engines/log-family/index.md delete mode 100644 docs/tr/engines/table-engines/log-family/log.md delete mode 100644 docs/tr/engines/table-engines/log-family/stripelog.md delete mode 100644 docs/tr/engines/table-engines/log-family/tinylog.md delete mode 100644 docs/tr/engines/table-engines/mergetree-family/aggregatingmergetree.md delete mode 100644 docs/tr/engines/table-engines/mergetree-family/collapsingmergetree.md delete mode 100644 docs/tr/engines/table-engines/mergetree-family/custom-partitioning-key.md delete mode 100644 docs/tr/engines/table-engines/mergetree-family/graphitemergetree.md delete mode 100644 docs/tr/engines/table-engines/mergetree-family/index.md delete mode 100644 docs/tr/engines/table-engines/mergetree-family/mergetree.md delete mode 100644 docs/tr/engines/table-engines/mergetree-family/replacingmergetree.md delete mode 100644 docs/tr/engines/table-engines/mergetree-family/replication.md delete mode 100644 docs/tr/engines/table-engines/mergetree-family/summingmergetree.md delete mode 100644 docs/tr/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md delete mode 100644 docs/tr/engines/table-engines/special/buffer.md delete mode 100644 docs/tr/engines/table-engines/special/dictionary.md delete mode 100644 docs/tr/engines/table-engines/special/distributed.md delete mode 100644 docs/tr/engines/table-engines/special/external-data.md delete mode 100644 docs/tr/engines/table-engines/special/file.md delete mode 100644 docs/tr/engines/table-engines/special/generate.md delete mode 100644 docs/tr/engines/table-engines/special/index.md delete mode 100644 docs/tr/engines/table-engines/special/join.md delete mode 100644 docs/tr/engines/table-engines/special/materializedview.md delete mode 100644 docs/tr/engines/table-engines/special/memory.md delete mode 100644 docs/tr/engines/table-engines/special/merge.md delete mode 100644 docs/tr/engines/table-engines/special/null.md delete mode 100644 docs/tr/engines/table-engines/special/set.md delete mode 100644 docs/tr/engines/table-engines/special/url.md delete mode 100644 docs/tr/engines/table-engines/special/view.md delete mode 100644 docs/tr/faq/general.md delete mode 100644 docs/tr/faq/index.md delete mode 100644 docs/tr/getting-started/example-datasets/amplab-benchmark.md delete mode 100644 docs/tr/getting-started/example-datasets/criteo.md delete mode 100644 docs/tr/getting-started/example-datasets/index.md delete mode 100644 docs/tr/getting-started/example-datasets/metrica.md delete mode 100644 docs/tr/getting-started/example-datasets/nyc-taxi.md delete mode 100644 docs/tr/getting-started/example-datasets/ontime.md delete mode 100644 docs/tr/getting-started/example-datasets/star-schema.md delete mode 100644 docs/tr/getting-started/example-datasets/wikistat.md delete mode 100644 docs/tr/getting-started/index.md delete mode 100644 docs/tr/getting-started/install.md delete mode 100644 docs/tr/getting-started/playground.md delete mode 100644 docs/tr/getting-started/tutorial.md delete mode 100644 docs/tr/guides/apply-catboost-model.md delete mode 100644 docs/tr/guides/index.md delete mode 120000 docs/tr/images delete mode 100644 docs/tr/index.md delete mode 100644 docs/tr/interfaces/cli.md delete mode 100644 docs/tr/interfaces/cpp.md delete mode 100644 docs/tr/interfaces/formats.md delete mode 100644 docs/tr/interfaces/http.md delete mode 100644 docs/tr/interfaces/index.md delete mode 100644 docs/tr/interfaces/jdbc.md delete mode 100644 docs/tr/interfaces/mysql.md delete mode 100644 docs/tr/interfaces/odbc.md delete mode 100644 docs/tr/interfaces/tcp.md delete mode 100644 docs/tr/interfaces/third-party/client-libraries.md delete mode 100644 docs/tr/interfaces/third-party/gui.md delete mode 100644 docs/tr/interfaces/third-party/index.md delete mode 100644 docs/tr/interfaces/third-party/integrations.md delete mode 100644 docs/tr/interfaces/third-party/proxy.md delete mode 100644 docs/tr/introduction/adopters.md delete mode 100644 docs/tr/introduction/distinctive-features.md delete mode 100644 docs/tr/introduction/history.md delete mode 100644 docs/tr/introduction/index.md delete mode 100644 docs/tr/introduction/performance.md delete mode 100644 docs/tr/operations/access-rights.md delete mode 100644 docs/tr/operations/backup.md delete mode 100644 docs/tr/operations/configuration-files.md delete mode 100644 docs/tr/operations/index.md delete mode 100644 docs/tr/operations/monitoring.md delete mode 100644 docs/tr/operations/optimizing-performance/index.md delete mode 100644 docs/tr/operations/optimizing-performance/sampling-query-profiler.md delete mode 100644 docs/tr/operations/performance-test.md delete mode 100644 docs/tr/operations/quotas.md delete mode 100644 docs/tr/operations/requirements.md delete mode 100644 docs/tr/operations/server-configuration-parameters/index.md delete mode 100644 docs/tr/operations/server-configuration-parameters/settings.md delete mode 100644 docs/tr/operations/settings/constraints-on-settings.md delete mode 100644 docs/tr/operations/settings/index.md delete mode 100644 docs/tr/operations/settings/permissions-for-queries.md delete mode 100644 docs/tr/operations/settings/query-complexity.md delete mode 100644 docs/tr/operations/settings/settings-profiles.md delete mode 100644 docs/tr/operations/settings/settings-users.md delete mode 100644 docs/tr/operations/settings/settings.md delete mode 100644 docs/tr/operations/system-tables.md delete mode 100644 docs/tr/operations/tips.md delete mode 100644 docs/tr/operations/troubleshooting.md delete mode 100644 docs/tr/operations/update.md delete mode 100644 docs/tr/operations/utilities/clickhouse-benchmark.md delete mode 100644 docs/tr/operations/utilities/clickhouse-copier.md delete mode 100644 docs/tr/operations/utilities/clickhouse-local.md delete mode 100644 docs/tr/operations/utilities/index.md delete mode 100644 docs/tr/sql-reference/aggregate-functions/combinators.md delete mode 100644 docs/tr/sql-reference/aggregate-functions/index.md delete mode 100644 docs/tr/sql-reference/aggregate-functions/parametric-functions.md delete mode 100644 docs/tr/sql-reference/aggregate-functions/reference.md delete mode 100644 docs/tr/sql-reference/ansi.md delete mode 100644 docs/tr/sql-reference/data-types/aggregatefunction.md delete mode 100644 docs/tr/sql-reference/data-types/array.md delete mode 100644 docs/tr/sql-reference/data-types/boolean.md delete mode 100644 docs/tr/sql-reference/data-types/date.md delete mode 100644 docs/tr/sql-reference/data-types/datetime.md delete mode 100644 docs/tr/sql-reference/data-types/datetime64.md delete mode 100644 docs/tr/sql-reference/data-types/decimal.md delete mode 100644 docs/tr/sql-reference/data-types/domains/index.md delete mode 100644 docs/tr/sql-reference/data-types/domains/ipv4.md delete mode 100644 docs/tr/sql-reference/data-types/domains/ipv6.md delete mode 100644 docs/tr/sql-reference/data-types/enum.md delete mode 100644 docs/tr/sql-reference/data-types/fixedstring.md delete mode 100644 docs/tr/sql-reference/data-types/float.md delete mode 100644 docs/tr/sql-reference/data-types/index.md delete mode 100644 docs/tr/sql-reference/data-types/int-uint.md delete mode 100644 docs/tr/sql-reference/data-types/lowcardinality.md delete mode 100644 docs/tr/sql-reference/data-types/nested-data-structures/index.md delete mode 100644 docs/tr/sql-reference/data-types/nested-data-structures/nested.md delete mode 100644 docs/tr/sql-reference/data-types/nullable.md delete mode 100644 docs/tr/sql-reference/data-types/simpleaggregatefunction.md delete mode 100644 docs/tr/sql-reference/data-types/special-data-types/expression.md delete mode 100644 docs/tr/sql-reference/data-types/special-data-types/index.md delete mode 100644 docs/tr/sql-reference/data-types/special-data-types/interval.md delete mode 100644 docs/tr/sql-reference/data-types/special-data-types/nothing.md delete mode 100644 docs/tr/sql-reference/data-types/special-data-types/set.md delete mode 100644 docs/tr/sql-reference/data-types/string.md delete mode 100644 docs/tr/sql-reference/data-types/tuple.md delete mode 100644 docs/tr/sql-reference/data-types/uuid.md delete mode 100644 docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md delete mode 100644 docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md delete mode 100644 docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md delete mode 100644 docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md delete mode 100644 docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md delete mode 100644 docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md delete mode 100644 docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts.md delete mode 100644 docs/tr/sql-reference/dictionaries/external-dictionaries/index.md delete mode 100644 docs/tr/sql-reference/dictionaries/index.md delete mode 100644 docs/tr/sql-reference/dictionaries/internal-dicts.md delete mode 100644 docs/tr/sql-reference/functions/arithmetic-functions.md delete mode 100644 docs/tr/sql-reference/functions/array-functions.md delete mode 100644 docs/tr/sql-reference/functions/array-join.md delete mode 100644 docs/tr/sql-reference/functions/bit-functions.md delete mode 100644 docs/tr/sql-reference/functions/bitmap-functions.md delete mode 100644 docs/tr/sql-reference/functions/comparison-functions.md delete mode 100644 docs/tr/sql-reference/functions/conditional-functions.md delete mode 100644 docs/tr/sql-reference/functions/date-time-functions.md delete mode 100644 docs/tr/sql-reference/functions/encoding-functions.md delete mode 100644 docs/tr/sql-reference/functions/ext-dict-functions.md delete mode 100644 docs/tr/sql-reference/functions/functions-for-nulls.md delete mode 100644 docs/tr/sql-reference/functions/geo.md delete mode 100644 docs/tr/sql-reference/functions/hash-functions.md delete mode 100644 docs/tr/sql-reference/functions/higher-order-functions.md delete mode 100644 docs/tr/sql-reference/functions/in-functions.md delete mode 100644 docs/tr/sql-reference/functions/index.md delete mode 100644 docs/tr/sql-reference/functions/introspection.md delete mode 100644 docs/tr/sql-reference/functions/ip-address-functions.md delete mode 100644 docs/tr/sql-reference/functions/json-functions.md delete mode 100644 docs/tr/sql-reference/functions/logical-functions.md delete mode 100644 docs/tr/sql-reference/functions/machine-learning-functions.md delete mode 100644 docs/tr/sql-reference/functions/math-functions.md delete mode 100644 docs/tr/sql-reference/functions/other-functions.md delete mode 100644 docs/tr/sql-reference/functions/random-functions.md delete mode 100644 docs/tr/sql-reference/functions/rounding-functions.md delete mode 100644 docs/tr/sql-reference/functions/splitting-merging-functions.md delete mode 100644 docs/tr/sql-reference/functions/string-functions.md delete mode 100644 docs/tr/sql-reference/functions/string-replace-functions.md delete mode 100644 docs/tr/sql-reference/functions/string-search-functions.md delete mode 100644 docs/tr/sql-reference/functions/type-conversion-functions.md delete mode 100644 docs/tr/sql-reference/functions/url-functions.md delete mode 100644 docs/tr/sql-reference/functions/uuid-functions.md delete mode 100644 docs/tr/sql-reference/functions/ym-dict-functions.md delete mode 100644 docs/tr/sql-reference/index.md delete mode 120000 docs/tr/sql-reference/operators/in.md delete mode 100644 docs/tr/sql-reference/operators/index.md delete mode 100644 docs/tr/sql-reference/statements/alter.md delete mode 100644 docs/tr/sql-reference/statements/create.md delete mode 100644 docs/tr/sql-reference/statements/grant.md delete mode 100644 docs/tr/sql-reference/statements/index.md delete mode 100644 docs/tr/sql-reference/statements/insert-into.md delete mode 100644 docs/tr/sql-reference/statements/misc.md delete mode 120000 docs/tr/sql-reference/statements/revoke.md delete mode 120000 docs/tr/sql-reference/statements/select/array-join.md delete mode 120000 docs/tr/sql-reference/statements/select/distinct.md delete mode 120000 docs/tr/sql-reference/statements/select/format.md delete mode 120000 docs/tr/sql-reference/statements/select/from.md delete mode 120000 docs/tr/sql-reference/statements/select/group-by.md delete mode 120000 docs/tr/sql-reference/statements/select/having.md delete mode 120000 docs/tr/sql-reference/statements/select/index.md delete mode 120000 docs/tr/sql-reference/statements/select/into-outfile.md delete mode 120000 docs/tr/sql-reference/statements/select/join.md delete mode 120000 docs/tr/sql-reference/statements/select/limit-by.md delete mode 120000 docs/tr/sql-reference/statements/select/limit.md delete mode 120000 docs/tr/sql-reference/statements/select/order-by.md delete mode 120000 docs/tr/sql-reference/statements/select/prewhere.md delete mode 120000 docs/tr/sql-reference/statements/select/sample.md delete mode 100644 docs/tr/sql-reference/statements/select/union.md delete mode 120000 docs/tr/sql-reference/statements/select/where.md delete mode 120000 docs/tr/sql-reference/statements/select/with.md delete mode 100644 docs/tr/sql-reference/statements/show.md delete mode 100644 docs/tr/sql-reference/statements/system.md delete mode 100644 docs/tr/sql-reference/syntax.md delete mode 100644 docs/tr/sql-reference/table-functions/file.md delete mode 100644 docs/tr/sql-reference/table-functions/generate.md delete mode 100644 docs/tr/sql-reference/table-functions/hdfs.md delete mode 100644 docs/tr/sql-reference/table-functions/index.md delete mode 100644 docs/tr/sql-reference/table-functions/input.md delete mode 100644 docs/tr/sql-reference/table-functions/jdbc.md delete mode 100644 docs/tr/sql-reference/table-functions/merge.md delete mode 100644 docs/tr/sql-reference/table-functions/mysql.md delete mode 100644 docs/tr/sql-reference/table-functions/numbers.md delete mode 100644 docs/tr/sql-reference/table-functions/odbc.md delete mode 100644 docs/tr/sql-reference/table-functions/remote.md delete mode 100644 docs/tr/sql-reference/table-functions/url.md delete mode 120000 docs/tr/whats-new/changelog/2017.md delete mode 120000 docs/tr/whats-new/changelog/2018.md delete mode 120000 docs/tr/whats-new/changelog/2019.md delete mode 120000 docs/tr/whats-new/changelog/index.md delete mode 100644 docs/tr/whats-new/index.md delete mode 100644 docs/tr/whats-new/roadmap.md delete mode 100644 docs/tr/whats-new/security-changelog.md diff --git a/docs/fa/commercial/cloud.md b/docs/fa/commercial/cloud.md deleted file mode 100644 index c41b2250608..00000000000 --- a/docs/fa/commercial/cloud.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 1 -toc_title: "\u0627\u0628\u0631" ---- - -# سرویس دهندگان سرویس ابری کلیک {#clickhouse-cloud-service-providers} - -!!! info "اطلاعات" - اگر شما راه اندازی یک ابر عمومی با مدیریت سرویس خانه کلیک, احساس رایگان به [درخواست کشش را باز کنید](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/commercial/cloud.md) اضافه کردن به لیست زیر. - -## ابر یاندکس {#yandex-cloud} - -[سرویس مدیریت یاندکس برای کلیک](https://cloud.yandex.com/services/managed-clickhouse?utm_source=referrals&utm_medium=clickhouseofficialsite&utm_campaign=link3) ویژگی های کلیدی زیر را فراهم می کند: - -- خدمات باغ وحش به طور کامل مدیریت برای [تکرار کلیک](../engines/table-engines/mergetree-family/replication.md) -- انتخاب نوع ذخیره سازی چندگانه -- کپی در مناطق مختلف در دسترس بودن -- رمزگذاری و جداسازی -- تعمیر و نگهداری خودکار - -{## [مقاله اصلی](https://clickhouse.tech/docs/en/commercial/cloud/) ##} diff --git a/docs/fa/commercial/index.md b/docs/fa/commercial/index.md deleted file mode 100644 index b6c9822f58d..00000000000 --- a/docs/fa/commercial/index.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0628\u0627\u0632\u0631\u06AF\u0627\u0646\u06CC" -toc_priority: 70 -toc_title: "\u0628\u0627\u0632\u0631\u06AF\u0627\u0646\u06CC" ---- - - diff --git a/docs/fa/commercial/support.md b/docs/fa/commercial/support.md deleted file mode 100644 index 6a3b0d1cac0..00000000000 --- a/docs/fa/commercial/support.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 3 -toc_title: "\u067E\u0634\u062A\u06CC\u0628\u0627\u0646\u06CC" ---- - -# سرویس دهنده های پشتیبانی تجاری کلیک {#clickhouse-commercial-support-service-providers} - -!!! info "اطلاعات" - اگر شما راه اندازی کرده اند یک سرویس پشتیبانی تجاری تاتر, در صورت تمایل به [درخواست کشش را باز کنید](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/commercial/support.md) اضافه کردن به لیست زیر. - -## درجهبندی {#altinity} - -التایمیت از 2017 پشتیبانی و خدمات سازمانی را پیشنهاد کرده است. مشتریان درجه بندی از فورچون 100 شرکت به راه اندازی. بازدید [www.altinity.com](https://www.altinity.com/) برای کسب اطلاعات بیشتر. - -## سکس پارتی {#mafiree} - -[شرح خدمات](http://mafiree.com/clickhouse-analytics-services.php) - -## مینروب {#minervadb} - -[شرح خدمات](https://minervadb.com/index.php/clickhouse-consulting-and-support-by-minervadb/) diff --git a/docs/fa/development/architecture.md b/docs/fa/development/architecture.md deleted file mode 100644 index 2b7796fd7fb..00000000000 --- a/docs/fa/development/architecture.md +++ /dev/null @@ -1,204 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 62 -toc_title: "\u0628\u0631\u0631\u0633\u06CC \u0627\u062C\u0645\u0627\u0644\u06CC \u0627\ - \u0632 \u0645\u0639\u0645\u0627\u0631\u06CC \u06A9\u0644\u06CC\u06A9" ---- - -# بررسی اجمالی از معماری کلیک {#overview-of-clickhouse-architecture} - -تاتر سندرم قبل از قاعدگی ستون گرا درست است. داده ها توسط ستون ها و در طول اجرای ارریس ذخیره می شود (بردارها و یا تکه های ستون). هر زمان ممکن, عملیات در ارریس اعزام, به جای در ارزش های فردی. این است که به نام “vectorized query execution,” و این کمک می کند کاهش هزینه پردازش داده های واقعی. - -> این ایده چیز جدیدی نیست. این قدمت به `APL` زبان برنامه نویسی و فرزندان خود را: `A +`, `J`, `K` و `Q`. برنامه نویسی مجموعه در پردازش داده های علمی استفاده می شود. نه این ایده چیزی جدید در پایگاه داده های رابطه ای است: مثلا در `Vectorwise` سیستم. - -دو روش مختلف برای سرعت بخشیدن به پردازش پرس و جو وجود دارد: اجرای پرس و جو و تولید کد زمان اجرا. در حالت دوم حذف تمام تغییر ناپذیر و اعزام پویا. هیچ کدام از این روش ها به شدت بهتر از دیگری نیست. تولید کد زمان اجرا می تواند بهتر باشد زمانی که فیوز بسیاری از عملیات, در نتیجه به طور کامل با استفاده از واحد اعدام پردازنده و خط لوله. اجرای پرس و جو بردار می تواند کمتر عملی باشد زیرا شامل بردار موقت است که باید به حافظه پنهان نوشته شود و به عقب برگردد. اگر داده های موقت در کش ال 2 مناسب نیست, این موضوع می شود. اما اجرای پرس و جو بردار به راحتی با بهره گیری از قابلیت سیم کارت از پردازنده. یک [مقاله پژوهشی](http://15721.courses.cs.cmu.edu/spring2016/papers/p5-sompolski.pdf) نوشته شده توسط دوستان ما نشان می دهد که بهتر است به ترکیب هر دو روش. تاتر با استفاده از اجرای پرس و جو بردار و حمایت اولیه برای تولید کد زمان اجرا محدود کرده است. - -## ستونها {#columns} - -`IColumn` رابط برای نشان دادن ستون ها در حافظه (در واقع تکه های ستون) استفاده می شود. این رابط فراهم می کند روش های کمکی برای اجرای اپراتورهای مختلف رابطه ای. تقریبا تمام عملیات تغییر ناپذیر است: ستون اصلی را تغییر نمی دهند اما یک تغییر جدید ایجاد می کنند. برای مثال `IColumn :: filter` روش یک ماسک بایت فیلتر می پذیرد. این برای استفاده می شود `WHERE` و `HAVING` اپراتورهای رابطه. نمونه های اضافی: `IColumn :: permute` روش پشتیبانی `ORDER BY` این `IColumn :: cut` روش پشتیبانی `LIMIT`. - -مختلف `IColumn` پیاده سازی (`ColumnUInt8`, `ColumnString` و به همین ترتیب) برای طرح حافظه ستون ها به عهده دارند. طرح حافظه معمولا یک مجموعه پیوسته است. برای نوع عدد صحیح ستون, این فقط یک مجموعه به هم پیوسته است, پسندیدن `std :: vector`. برای `String` و `Array` ستون ها, این دو بردار است: یکی برای همه عناصر مجموعه, به طور متناوب قرار داده شده, و یک ثانیه برای شیپور خاموشی به ابتدای هر مجموعه. همچنین وجود دارد `ColumnConst` که فروشگاه فقط یک ارزش در حافظه, اما به نظر می رسد مانند یک ستون. - -## زمینه {#field} - -با این اوصاف, ممکن است برای کار با ارزش های فردی و همچنین. برای نشان دادن ارزش فردی `Field` استفاده شده است. `Field` فقط یک اتحادیه تبعیض `UInt64`, `Int64`, `Float64`, `String` و `Array`. `IColumn` دارد `operator[]` روش برای دریافت ارزش ازت به عنوان یک `Field` و `insert` روش برای اضافه کردن یک `Field` به پایان یک ستون. این روش ها بسیار موثر نیستند زیرا نیاز به برخورد موقت دارند `Field` اشیا به نمایندگی از ارزش فردی. روش های موثر تر مانند `insertFrom`, `insertRangeFrom` و به همین ترتیب. - -`Field` اطلاعات کافی در مورد یک نوع داده خاص برای یک جدول ندارد. به عنوان مثال, `UInt8`, `UInt16`, `UInt32` و `UInt64` همه به عنوان نمایندگی `UInt64` در یک `Field`. - -## انتزاعی نشتی {#leaky-abstractions} - -`IColumn` روش هایی برای تحولات رابطه ای مشترک داده ها دارد اما همه نیازها را نمی بینند. به عنوان مثال, `ColumnUInt64` یک روش برای محاسبه مجموع دو ستون ندارد و `ColumnString` یک روش برای اجرای یک جستجو زیر رشته ندارد. این روال بی شماری در خارج از اجرا `IColumn`. - -توابع مختلف در ستون ها را می توان در یک روش عمومی و غیر موثر استفاده کرد `IColumn` مواد و روش ها برای استخراج `Field` ارزش, و یا در یک راه تخصصی با استفاده از دانش طرح حافظه داخلی از داده ها در یک خاص `IColumn` اجرا کردن. این است که توسط توابع ریخته گری به خاص اجرا شده است `IColumn` نوع و مقابله با نمایندگی داخلی به طور مستقیم. به عنوان مثال, `ColumnUInt64` دارد `getData` روشی که اشاره به مجموعه داخلی را برمی گرداند, سپس یک روال جداگانه می خواند و یا که مجموعه را پر می کند به طور مستقیم. ما “leaky abstractions” برای اجازه دادن به تخصص های موثر روال های مختلف. - -## انواع داده ها {#data_types} - -`IDataType` مسئول سریالسازی و deserialization: برای خواندن و نوشتن تکه های ستون یا فردی مقادیر دودویی یا به صورت متن. `IDataType` به طور مستقیم به انواع داده ها در جداول مربوط. مثلا, وجود دارد `DataTypeUInt32`, `DataTypeDateTime`, `DataTypeString` و به همین ترتیب. - -`IDataType` و `IColumn` فقط شل به یکدیگر مربوط. انواع داده های مختلف را می توان در حافظه توسط همان نشان داده شده است `IColumn` پیاده سازی. به عنوان مثال, `DataTypeUInt32` و `DataTypeDateTime` هر دو توسط نمایندگی `ColumnUInt32` یا `ColumnConstUInt32`. علاوه بر این, همان نوع داده را می توان با مختلف نشان `IColumn` پیاده سازی. به عنوان مثال, `DataTypeUInt8` می توان با نمایندگی `ColumnUInt8` یا `ColumnConstUInt8`. - -`IDataType` فقط فروشگاه ابرداده. به عنوان مثال, `DataTypeUInt8` هیچ چیزی را ذخیره نمی کند (به جز ویپر) و `DataTypeFixedString` فروشگاه ها فقط `N` (اندازه رشته های ثابت). - -`IDataType` دارای روش های کمکی برای فرمت های داده های مختلف. نمونه روش برای مرتب کردن یک مقدار با امکان نقل قول, برای مرتب کردن یک مقدار برای جانسون, و برای مرتب کردن یک مقدار به عنوان بخشی از فرمت میلی لیتر. هیچ مکاتبات مستقیم به فرمت های داده وجود دارد. برای مثال فرمت های داده های مختلف `Pretty` و `TabSeparated` می توانید همان استفاده کنید `serializeTextEscaped` روش کمکی از `IDataType` واسط. - -## بلوک {#block} - -A `Block` یک ظرف است که نشان دهنده یک زیر مجموعه است (تکه) از یک جدول در حافظه. این فقط مجموعه ای از سه برابر است: `(IColumn, IDataType, column name)`. در طول اجرای پرس و جو, داده ها توسط پردازش `Block`اگر ما یک `Block`, ما داده (در `IColumn` هدف), ما باید اطلاعات در مورد نوع خود (به `IDataType`) که به ما می گوید که چگونه به مقابله با این ستون, و ما باید نام ستون. این می تواند یا نام ستون اصلی از جدول و یا برخی از نام مصنوعی اختصاص داده شده برای گرفتن نتایج موقت از محاسبات. - -هنگامی که ما برخی از تابع محاسبه بیش از ستون در یک بلوک, ما اضافه کردن ستون دیگر با نتیجه خود را به بلوک, و ما ستون برای استدلال از تابع را لمس کنید چرا که عملیات تغییر ناپذیر هستند. بعد, ستون غیر ضروری را می توان از بلوک حذف, اما اصلاح نشده. مناسب برای از بین بردن اکسپرس مشترک است. - -بلوک برای هر تکه پردازش داده ها ایجاد شده است. توجه داشته باشید که برای همان نوع از محاسبه, نام ستون و انواع یکسان باقی می ماند برای بلوک های مختلف, و تنها ستون تغییرات داده. بهتر است داده های بلوک را از هدر بلوک تقسیم کنید زیرا اندازه های بلوک کوچک دارای سربار بالایی از رشته های موقت برای کپی کردن نام های شکسته و ستون هستند. - -## بلوک جریان {#block-streams} - -جریان بلوک برای پردازش داده ها می باشد. ما با استفاده از جریان بلوک به خواندن داده ها از جایی انجام داده تحولات و یا نوشتن داده ها به جایی. `IBlockInputStream` دارد `read` روش به بهانه بلوک بعدی در حالی که در دسترس. `IBlockOutputStream` دارد `write` روش به فشار بلوک جایی. - -جریان ها برای: - -1. خواندن و یا نوشتن به یک جدول. جدول فقط می گرداند یک جریان برای خواندن و یا نوشتن بلوک. -2. اجرای فرمت های داده. مثلا, اگر شما می خواهید به خروجی داده ها به یک ترمینال در `Pretty` شما یک جریان خروجی بلوک ایجاد می کنید که بلوک ها را فشار می دهید و فرمت می کند. -3. انجام تحولات داده ها. بیایید می گویند شما `IBlockInputStream` و می خواهید برای ایجاد یک جریان فیلتر شده است. شما ایجاد `FilterBlockInputStream` و با جریان خود را مقداردهی اولیه. سپس هنگامی که شما جلو و یک بلوک از `FilterBlockInputStream`, این نیش ترمزی میزند یک بلوک از جریان خود را, فیلتر, و گرداند بلوک فیلتر به شما. خطوط لوله اجرای پرس و جو در این راه نشان داده شده است. - -تحولات پیچیده تر وجود دارد. مثلا, زمانی که شما از جلو `AggregatingBlockInputStream` تمام داده ها را از منبع خود می خواند و جمع می کند و سپس جریان داده های جمع شده را برای شما باز می گرداند. مثال دیگر: `UnionBlockInputStream` می پذیرد بسیاری از منابع ورودی در سازنده و همچنین تعدادی از موضوعات. این راه اندازی موضوعات متعدد و بار خوانده شده از منابع مختلف به صورت موازی. - -> بلوک جریان استفاده از “pull” رویکرد به کنترل جریان: هنگامی که شما یک بلوک جلو و از جریان اول, در نتیجه می کشد بلوک های مورد نیاز از جریان تو در تو, و کل خط لوله اعدام کار خواهد کرد. نه “pull” نه “push” بهترین راه حل است زیرا جریان کنترل ضمنی است و محدودیت اجرای ویژگی های مختلف مانند اجرای همزمان چندین نمایش داده شد (ادغام بسیاری از خطوط لوله با هم). این محدودیت می تواند با کروتین ها و یا فقط در حال اجرا موضوعات اضافی که برای یکدیگر صبر غلبه بر. ما ممکن است امکانات بیشتری داشته باشیم اگر جریان کنترل را صریح کنیم: اگر منطق را برای عبور داده ها از یک واحد محاسبه به خارج دیگری از این واحدهای محاسبه قرار دهیم. خواندن این [مقاله](http://journal.stuffwithstuff.com/2013/01/13/iteration-inside-and-out/) برای افکار بیشتر. - -ما باید توجه داشته باشید که خط لوله اجرای پرس و جو ایجاد داده های موقت در هر مرحله. ما سعی می کنیم برای حفظ اندازه بلوک به اندازه کافی کوچک به طوری که داده های موقت متناسب در کش پردازنده. با این فرض, نوشتن و خواندن داده های موقت تقریبا رایگان در مقایسه با محاسبات دیگر است. ما می توانیم یک جایگزین در نظر, است که به فیوز بسیاری از عملیات در خط لوله با هم. این می تواند خط لوله به عنوان کوتاه که ممکن است و حذف بسیاری از اطلاعات موقت, که می تواند یک مزیت, اما همچنین دارای اشکالاتی. مثلا, یک خط لوله تقسیم باعث می شود به راحتی پیاده سازی ذخیره داده متوسط, سرقت اطلاعات متوسط از نمایش داده شد مشابه در حال اجرا در همان زمان, و ادغام خطوط لوله برای نمایش داده شد مشابه. - -## فرشها {#formats} - -فرمت های داده ها با جریان بلوک اجرا شده است. وجود دارد “presentational” فرمت تنها مناسب برای خروجی داده ها به مشتری مانند `Pretty` قالب, فراهم می کند که تنها `IBlockOutputStream`. و فرمت های ورودی / خروجی مانند `TabSeparated` یا `JSONEachRow`. - -همچنین جریان ردیف وجود دارد: `IRowInputStream` و `IRowOutputStream`. اجازه می دهد شما را به جلو و/فشار داده های ردیف های فردی, نه با بلوک. و فقط برای ساده سازی اجرای فرمت های ردیف گرا نیاز دارند. لفافه `BlockInputStreamFromRowInputStream` و `BlockOutputStreamFromRowOutputStream` به شما اجازه تبدیل جریان ردیف گرا به جریان بلوک گرا به طور منظم. - -## I/O {#io} - -برای ورودی بایت گرا / خروجی, وجود دارد `ReadBuffer` و `WriteBuffer` کلاس های انتزاعی. به جای ج++استفاده می شود `iostream`نگران نباشید: هر پروژه سی++ بالغ با استفاده از چیزی غیر از `iostream`به دلایل خوب. - -`ReadBuffer` و `WriteBuffer` فقط یک بافر پیوسته و مکان نما با اشاره به موقعیت در بافر که. پیاده سازی ممکن است خود و یا حافظه برای بافر خود را ندارد. یک روش مجازی برای پر کردن بافر با داده های زیر وجود دارد (برای `ReadBuffer`) و یا به خیط و پیت کردن بافر جایی (برای `WriteBuffer`). روش های مجازی به ندرت نامیده می شود. - -پیاده سازی از `ReadBuffer`/`WriteBuffer` برای کار با فایل ها و توصیفگر فایل ها و سوکت های شبکه برای اجرای فشرده سازی استفاده می شود (`CompressedWriteBuffer` is initialized with another WriteBuffer and performs compression before writing data to it), and for other purposes – the names `ConcatReadBuffer`, `LimitReadBuffer` و `HashingWriteBuffer` صحبت برای خود. - -خواندن / نویسنده تنها با بایت برخورد. توابع از وجود دارد `ReadHelpers` و `WriteHelpers` فایل های هدر برای کمک به قالب بندی ورودی / خروجی. برای مثال یاران به نوشتن یک شماره در قالب دهدهی وجود دارد. - -بیایید نگاه کنیم که چه اتفاقی می افتد زمانی که می خواهید نتیجه را بنویسید `JSON` قالب به چاق و چله. شما باید یک نتیجه مجموعه ای از ذهن می شود `IBlockInputStream`. شما ایجاد `WriteBufferFromFileDescriptor(STDOUT_FILENO)` برای نوشتن بایت به چاق و چله. شما ایجاد `JSONRowOutputStream`, مقداردهی اولیه با که `WriteBuffer` برای نوشتن ردیف در `JSON` به چاق و چله. شما ایجاد `BlockOutputStreamFromRowOutputStream` در بالای این, برای نشان دادن به عنوان `IBlockOutputStream`. سپس با شما تماس `copyData` برای انتقال داده ها از `IBlockInputStream` به `IBlockOutputStream`, و همه چیز کار می کند. داخلی, `JSONRowOutputStream` خواهد شمارشگر های مختلف جانسون ارسال و پاسخ `IDataType::serializeTextJSON` روش با اشاره به `IColumn` و شماره ردیف به عنوان استدلال. در نتیجه, `IDataType::serializeTextJSON` یک روش از پاسخ `WriteHelpers.h`: به عنوان مثال, `writeText` برای انواع عددی و `writeJSONString` برای `DataTypeString`. - -## جداول {#tables} - -این `IStorage` رابط نشان دهنده جداول. پیاده سازی های مختلف که رابط موتورهای جدول متفاوت است. مثالها عبارتند از `StorageMergeTree`, `StorageMemory` و به همین ترتیب. نمونه هایی از این کلاس ها فقط جداول. - -کلید `IStorage` روش `read` و `write`. همچنین وجود دارد `alter`, `rename`, `drop` و به همین ترتیب. این `read` روش استدلال های زیر را می پذیرد: مجموعه ای از ستون ها برای خواندن از یک جدول `AST` پرس و جو را در نظر بگیرید, و تعداد مورد نظر از جریان به بازگشت. این گرداند یک یا چند `IBlockInputStream` اشیا و اطلاعات در مورد مرحله پردازش داده ها که در داخل یک موتور جدول در طول اجرای پرس و جو تکمیل شد. - -در بیشتر موارد, روش خواندن تنها برای خواندن ستون مشخص شده از یک جدول است, نه برای هر پردازش داده ها بیشتر. تمام پردازش داده های بیشتر توسط مترجم پرس و جو انجام می شود و خارج از وظیفه است `IStorage`. - -اما استثنا قابل توجه وجود دارد: - -- پرس و جو اس تی به تصویب رسید `read` روش, و موتور جدول می توانید استفاده کنید به استفاده از شاخص و به خواندن اطلاعات کمتر از یک جدول. -- گاهی موتور جدول می تواند داده های خود را به یک مرحله خاص پردازش. به عنوان مثال, `StorageDistributed` می توانید یک پرس و جو به سرور از راه دور ارسال, از او بخواهید برای پردازش داده ها به مرحله ای که داده ها را از سرور های مختلف از راه دور را می توان با هم ادغام شدند, و بازگشت که داده های پیش پردازش. مترجم پرس و جو سپس پس از اتمام پردازش داده ها. - -جدول `read` روش می تواند چندین بازگشت `IBlockInputStream` اشیا اجازه می دهد تا پردازش داده های موازی. این جریان ورودی بلوک های متعدد می توانید از یک جدول به صورت موازی به عنوان خوانده شده. سپس شما می توانید این جریان با تحولات مختلف قرار دادن (مانند ارزیابی بیان و یا فیلتر) است که می تواند به طور مستقل محاسبه و ایجاد یک `UnionBlockInputStream` برای خواندن از جریانهای چندگانه به صورت موازی. - -همچنین وجود دارد `TableFunction`این توابع که بازگشت موقت هستند `IStorage` شی برای استفاده در `FROM` بند یک پرس و جو. - -برای دریافت یک ایده سریع از نحوه پیاده سازی موتور جدول خود را در چیزی ساده مانند نگاه `StorageMemory` یا `StorageTinyLog`. - -> به عنوان نتیجه `read` روش, `IStorage` بازگشت `QueryProcessingStage` – information about what parts of the query were already calculated inside storage. - -## Parsers {#parsers} - -تجزیه کننده تبار بازگشتی دست نوشته تجزیه کننده تجزیه پرس و جو تجزیه. به عنوان مثال, `ParserSelectQuery` فقط به صورت بازگشتی تماس تجزیه کننده زمینه ای برای بخش های مختلف از پرس و جو. تجزیه کننده ایجاد یک `AST`. این `AST` توسط گره هایی که نمونه هایی از `IAST`. - -> ژنراتور تجزیه کننده به دلایل تاریخی استفاده نمی شود. - -## مترجمین {#interpreters} - -مترجمین برای ایجاد خط لوله اجرای پرس و جو از `AST`. می مفسران ساده وجود دارد, مانند `InterpreterExistsQuery` و `InterpreterDropQuery` یا پیچیده تر `InterpreterSelectQuery`. خط لوله اجرای پرس و جو ترکیبی از ورودی بلوک یا جریان خروجی است. برای مثال نتیجه تفسیر `SELECT` پرس و جو است `IBlockInputStream` برای خواندن نتیجه مجموعه ای از; نتیجه پرس و جو درج است `IBlockOutputStream` برای نوشتن داده ها برای درج به, و در نتیجه تفسیر `INSERT SELECT` پرس و جو است `IBlockInputStream` که نتیجه خالی را برمی گرداند مجموعه ای در خواندن برای اولین بار, اما نسخه داده ها از `SELECT` به `INSERT` در همان زمان. - -`InterpreterSelectQuery` استفاده `ExpressionAnalyzer` و `ExpressionActions` ماشین برای تجزیه و تحلیل پرس و جو و تحولات. این جایی است که اکثر بهینه سازی پرس و جو مبتنی بر قانون انجام می شود. `ExpressionAnalyzer` کاملا کثیف است و باید بازنویسی شود: تحولات پرس و جو های مختلف و بهینه سازی باید استخراج به کلاس های جداگانه اجازه می دهد تا تحولات مدولار و یا پرس و جو. - -## توابع {#functions} - -توابع عادی و توابع کل وجود دارد. برای توابع کل, بخش بعدی را ببینید. - -Ordinary functions don't change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for `Block`'ثانیه از داده ها برای پیاده سازی اجرای پرس و جو بردار. - -برخی از توابع متفرقه مانند وجود دارد [blockSize](../sql-reference/functions/other-functions.md#function-blocksize), [رفع موانع](../sql-reference/functions/other-functions.md#function-rownumberinblock) و [خرابی اجرا](../sql-reference/functions/other-functions.md#function-runningaccumulate), که بهره برداری از پردازش بلوک و نقض استقلال ردیف. - -تاتر تایپ قوی, بنابراین هیچ تبدیل نوع ضمنی وجود دارد. اگر یک تابع یک ترکیب خاص از انواع پشتیبانی نمی کند, این می اندازد یک استثنا. اما توابع می توانند کار کنند (غیرمنتظره) برای بسیاری از ترکیبات مختلف از انواع. برای مثال `plus` تابع (برای پیاده سازی `+` اپراتور) برای هر ترکیبی از انواع عددی کار می کند: `UInt8` + `Float32`, `UInt16` + `Int8` و به همین ترتیب. همچنین, برخی از توابع مختلف می توانید هر تعداد از استدلال قبول, مانند `concat` تابع. - -اجرای یک تابع ممکن است کمی ناخوشایند به دلیل یک تابع به صراحت اعزام انواع داده ها پشتیبانی و پشتیبانی `IColumns`. برای مثال `plus` تابع دارای کد تولید شده توسط نمونه از یک ج++ قالب برای هر ترکیبی از انواع عددی, و استدلال چپ و راست ثابت یا غیر ثابت. - -این یک محل عالی برای اجرای تولید کد زمان اجرا برای جلوگیری از نفخ کد الگو است. همچنین, این امکان را برای اضافه کردن توابع ذوب مانند ذوب ضرب-اضافه کردن و یا به مقایسه های متعدد در یک تکرار حلقه. - -با توجه به اجرای پرس و جو بردار, توابع کوتاه دور نیست. مثلا, اگر شما ارسال `WHERE f(x) AND g(y)`, هر دو طرف محاسبه می شود, حتی برای ردیف, وقتی که `f(x)` صفر است (به جز زمانی که `f(x)` بیان ثابت صفر است). اما اگر انتخاب از `f(x)` شرایط بالا و محاسبه است `f(x)` بسیار ارزان تر از `g(y)` بهتر است محاسبات چند گذر را اجرا کنید. این برای اولین بار محاسبه `f(x)` سپس ستون ها را با نتیجه فیلتر کنید و سپس محاسبه کنید `g(y)` فقط برای کوچکتر, تکه های فیلتر شده از داده ها. - -## توابع مجموع {#aggregate-functions} - -توابع مجموع توابع نفرت انگیز هستند. جمع ارزش به برخی از دولت منتقل می شود و به شما اجازه دریافت نتایج از دولت. با مدیریت `IAggregateFunction` واسط. ایالات می تواند نسبتا ساده (دولت برای `AggregateFunctionCount` تنها یک `UInt64` ارزش) و یا کاملا پیچیده (دولت از `AggregateFunctionUniqCombined` ترکیبی از یک مجموعه خطی است, یک جدول هش, و یک `HyperLogLog` ساختار داده احتمالاتی). - -ایالات در اختصاص داده `Arena` (یک استخر حافظه) برای مقابله با کشورهای مختلف در حالی که اجرای یک کارتیت بالا `GROUP BY` پرس و جو. ایالات می تواند یک سازنده غیر بدیهی و مخرب دارند: مثلا, کشورهای تجمع پیچیده می توانید حافظه اضافی خود اختصاص. این نیاز به توجه به ایجاد و از بین بردن ایالات و به درستی عبور مالکیت و تخریب سفارش خود را. - -تجمع متحده می تواند سرهمی و deserialized به تصویب بیش از این شبکه در توزیع پرس و جو اعدام و یا ارسال آنها را بر روی دیسک که در آن وجود دارد به اندازه کافی نمی رم. حتی می توانند در یک جدول با `DataTypeAggregateFunction` اجازه می دهد تا تجمع افزایشی از داده ها. - -> فرمت داده سریال برای عملکرد کل ایالات در حال حاضر نسخه نیست. این خوب است اگر کشورهای کل تنها به طور موقت ذخیره می شود. اما ما `AggregatingMergeTree` موتور جدول برای تجمع افزایشی, و مردم در حال حاضر با استفاده از در تولید. این دلیل که چرا سازگاری عقب مورد نیاز است در هنگام تغییر فرمت سریال برای هر تابع کل در اینده است. - -## کارگزار {#server} - -سرور پیاده سازی چندین رابط های مختلف: - -- رابط قام برای هر مشتریان خارجی. -- یک رابط واحد کنترل گیربکس اتوماتیک برای مشتری خانه رعیتی بومی و برای ارتباط متقابل سرور در طول اجرای پرس و جو توزیع شده است. -- رابط کاربری برای انتقال داده ها برای تکرار. - -داخلی, این فقط یک سرور چند رشته ای بدوی بدون کروتین یا الیاف است. از زمانی که سرور طراحی نشده است برای پردازش نرخ بالای ساده نمایش داده شد اما برای پردازش نسبتا پایین نرخ پیچیده نمایش داده شد هر یک می تواند روند مقدار زیادی از داده ها برای تجزیه و تحلیل ترافیک. - -سرور مقدار دهی اولیه `Context` کلاس با محیط لازم برای اجرای پرس و جو: لیستی از پایگاه داده های موجود, کاربران و حقوق دسترسی, تنظیمات, خوشه, لیست فرایند, ورود به سیستم پرس و جو, و غیره. مفسران استفاده از این محیط. - -ما سازگاری کامل رو به عقب و رو به جلو را برای پروتکل سرور تی سی پی حفظ می کنیم: مشتریان قدیمی می توانند با سرورهای جدید صحبت کنند و مشتریان جدید می توانند با سرورهای قدیمی صحبت کنند. اما ما نمی خواهیم تا ابد حفظ کنیم و پس از حدود یک سال پشتیبانی از نسخه های قدیمی را از بین می بریم. - -!!! note "یادداشت" - برای اکثر برنامه های کاربردی خارجی, توصیه می کنیم با استفاده از رابط اچ تی پی به دلیل ساده و ساده برای استفاده است. پروتکل تی سی پی به شدت با ساختارهای داده داخلی ارتباط دارد: از فرمت داخلی برای عبور بلوک های داده استفاده می کند و از فریم های سفارشی برای داده های فشرده استفاده می کند. ما یک کتابخانه سی که برای پروتکل منتشر نشده است چرا که نیاز به ارتباط بسیاری از کدهای کلیکهاوس, که عملی نیست. - -## اجرای پرس و جو توزیع شده {#distributed-query-execution} - -سرور در راه اندازی خوشه عمدتا مستقل هستند. شما می توانید یک `Distributed` جدول در یک یا تمام سرور در یک خوشه. این `Distributed` table does not store data itself – it only provides a “view” به تمام جداول محلی در گره های متعدد از یک خوشه. هنگامی که شما از یک انتخاب `Distributed` جدول, بازنویسی است که پرس و جو, را گره از راه دور با توجه به بار تنظیمات متعادل, و پرس و جو را به او می فرستد. این `Distributed` درخواست جدول سرور از راه دور برای پردازش یک پرس و جو فقط تا مرحله ای که نتایج متوسط از سرور های مختلف را می توان با هم ادغام شدند. سپس نتایج متوسط را دریافت می کند و ادغام می کند. جدول توزیع تلاش می کند برای توزیع کار به همان اندازه که ممکن است به سرور از راه دور می کند و داده های متوسط بسیار بیش از شبکه ارسال کنید. - -همه چیز پیچیده تر می شود زمانی که شما زیر کشتیها در و یا پیوستن به بند, و هر یک از استفاده از یک `Distributed` جدول ما استراتژی های مختلف برای اجرای این نمایش داده شد. - -هیچ برنامه پرس و جو جهانی برای اجرای پرس و جو توزیع وجود دارد. هر گره دارای برنامه پرس و جو محلی خود را برای بخشی خود را از کار. ما فقط ساده یک پاس اجرای پرس و جو توزیع شده: ما ارسال نمایش داده شد برای گره های از راه دور و سپس ادغام نتایج. اما این امکان پذیر نیست برای نمایش داده شد پیچیده با بالا کار افتادگی گروه بورس و یا با مقدار زیادی از داده های موقت برای پیوستن به. در چنین مواردی ما نیاز به “reshuffle” داده ها بین سرور, که نیاز به هماهنگی اضافی. کلیک هاوس از این نوع اجرای پرس و جو پشتیبانی نمی کند و ما باید روش کار کنیم. - -## ادغام درخت {#merge-tree} - -`MergeTree` یک خانواده از موتورهای ذخیره سازی است که پشتیبانی از نمایه سازی توسط کلید اصلی است. کلید اصلی می تواند یک تاپل دلخواه از ستون ها و یا عبارات. داده ها در یک `MergeTree` جدول در ذخیره می شود “parts”. هر بخش ذخیره داده ها در جهت کلید اولیه, بنابراین داده ها از لحاظ واژگان توسط تاپل کلید اولیه دستور داد. تمام ستون های جدول به صورت جداگانه ذخیره می شوند `column.bin` فایل ها در این بخش. فایل ها از بلوک های فشرده تشکیل شده است. هر بلوک است که معمولا از 64 کیلوبایت به 1 مگابایت از داده های غیر فشرده, بسته به اندازه مقدار متوسط. بلوک از مقادیر ستون بعد از دیگری قرار داده شده به طور یکنواخت تشکیل شده است. مقادیر ستون در همان جهت برای هر ستون هستند (کلید اصلی سفارش را تعریف می کند), تا زمانی که شما توسط بسیاری از ستون تکرار, شما ارزش برای ردیف مربوطه. - -کلید اصلی خود است “sparse”. این کار هر سطر رسیدگی نمی, اما تنها برخی از محدوده داده. جدا `primary.idx` فایل دارای ارزش کلید اصلی برای هر سطر نفر که نفر نامیده می شود `index_granularity` (معمولا, نفر = 8192). همچنین, برای هر ستون, ما داریم `column.mrk` پروندهها با “marks,” که ناراحتی به هر سطر نفر هفتم در فایل داده ها. هر علامت یک جفت است: افست در فایل به ابتدای بلوک فشرده و افست در بلوک فشرده به ابتدای داده ها. معمولا, بلوک های فشرده توسط علامت تراز وسط قرار دارد, و افست در بلوک فشرده صفر است. داده ها برای `primary.idx` همیشه در حافظه ساکن, و داده ها را برای `column.mrk` فایل های ذخیره شده است. - -هنگامی که ما می رویم به خواندن چیزی از یک شرکت در `MergeTree` ما نگاه می کنیم `primary.idx` داده ها و تعیین محل محدوده است که می تواند حاوی اطلاعات درخواست شده و سپس نگاه `column.mrk` داده ها و محاسبه شیپور خاموشی برای جایی که شروع به خواندن این محدوده. به دلیل نرمی, اطلاعات اضافی ممکن است به عنوان خوانده شده. تاتر مناسب برای یک بار بالا از نمایش داده شد نقطه ساده نیست, چرا که کل محدوده با `index_granularity` ردیف باید برای هر کلید به عنوان خوانده شده, و کل بلوک فشرده باید برای هر ستون از حالت فشرده خارج. ما جرقه شاخص ساخته شده چرا که ما باید قادر به حفظ تریلیون ردیف در هر سرور بدون مصرف حافظه قابل توجه برای شاخص باشد. همچنین, به دلیل کلید اصلی پراکنده است, منحصر به فرد نیست: این می تواند وجود کلید در جدول در زمان درج بررسی نمی. شما می توانید ردیف های بسیاری را با همان کلید در یک جدول داشته باشید. - -هنگامی که شما `INSERT` یک دسته از داده ها به `MergeTree`, که دسته مرتب شده بر اساس کلید اصلی سفارش و به شکل یک بخش جدید. موضوعات پس زمینه وجود دارد که به صورت دوره ای برخی از قطعات را انتخاب می کنند و به یک بخش مرتب شده اند تا تعداد قطعات نسبتا کم باشد. به همین دلیل است که نامیده می شود `MergeTree`. البته ادغام منجر به “write amplification”. تمام قطعات تغییر ناپذیر هستند: تنها ایجاد و حذف, اما اصلاح نشده. هنگامی که انتخاب اجرا شده است, دارای یک تصویر لحظهای از جدول (مجموعه ای از قطعات). پس از ادغام, ما همچنین قطعات قدیمی برای برخی از زمان به بهبود پس از شکست ساده تر نگه, بنابراین اگر ما می بینیم که برخی از بخش ادغام شده است که احتمالا شکسته, ما می توانیم با قطعات منبع خود را جایگزین. - -`MergeTree` یک درخت ل اس ام نیست زیرا حاوی نیست “memtable” و “log”: inserted data is written directly to the filesystem. This makes it suitable only to INSERT data in batches, not by individual row and not very frequently – about once per second is ok, but a thousand times a second is not. We did it this way for simplicity's sake, and because we are already inserting data in batches in our applications. - -> جداول ادغام تنها می توانید یک دارند (اولیه) شاخص: هیچ شاخص ثانویه وجود ندارد. این امر می تواند خوب اجازه می دهد تا بازنمایی فیزیکی متعدد تحت یک جدول منطقی, مثلا, برای ذخیره داده ها در بیش از یک نظم فیزیکی و یا حتی اجازه می دهد تا بازنمایی با داده های از پیش جمع همراه با داده های اصلی. - -موتورهای ادغام است که کار اضافی در طول پس زمینه ادغام انجام می دهند وجود دارد. مثالها عبارتند از `CollapsingMergeTree` و `AggregatingMergeTree`. این می تواند به عنوان پشتیبانی ویژه ای برای به روز رسانی درمان می شود. به خاطر داشته باشید که این به روز رسانی واقعی نیست چرا که کاربران معمولا هیچ کنترلی بر زمان هنگامی که پس زمینه ادغام اجرا می شوند, و داده ها در یک `MergeTree` جدول تقریبا همیشه در بیش از یک بخش ذخیره می شود, نه در فرم به طور کامل با هم ادغام شدند. - -## تکرار {#replication} - -تکرار در کلیک خانه را می توان بر اساس هر جدول پیکربندی شده است. شما می توانید برخی از تکرار و برخی از جداول غیر تکرار بر روی همان سرور. شما همچنین می تواند جداول تکرار در راه های مختلف, مانند یک جدول با تکرار دو عامل و دیگری با سه عامل. - -تکرار در اجرا `ReplicatedMergeTree` موتور ذخیره سازی. مسیر در `ZooKeeper` به عنوان یک پارامتر برای موتور ذخیره سازی مشخص شده است. تمام جداول با همان مسیر در `ZooKeeper` تبدیل کپی از یکدیگر: همگام سازی داده های خود و حفظ ثبات. کپی می تواند اضافه شود و به صورت پویا به سادگی با ایجاد و یا حذف یک جدول حذف شده است. - -تکرار با استفاده از یک طرح چند استاد ناهمزمان. شما می توانید داده ها را به هر ماکت است که یک جلسه با وارد `ZooKeeper`, و داده ها به تمام کپی های دیگر غیر همزمان تکرار. چون کلیک هاوس به روز رسانی را پشتیبانی نمی کند, تکرار بدون درگیری است. همانطور که هیچ اذعان حد نصاب درج وجود دارد, داده فقط قرار داده ممکن است از دست داده اگر یک گره نتواند. - -فراداده برای تکرار در باغ وحش ذخیره می شود. ورود به سیستم تکرار است که لیست چه اقداماتی را انجام دهید وجود دارد. اقدامات عبارتند از: دریافت بخشی; ادغام قطعات; رها کردن یک پارتیشن, و غیره. هر ماکت کپی ورود تکرار به صف خود و سپس اجرا اقدامات از صف. برای مثال در درج “get the part” عمل در ورود به سیستم ایجاد, و هر دانلود ماکت که بخشی. ادغام بین کپی هماهنگ برای دریافت بایت - نتایج یکسان. تمام قطعات در همان راه در تمام کپی با هم ادغام شدند. این است که با انتخاب یک ماکت به عنوان رهبر دست, و این ماکت شروع ادغام و می نویسد “merge parts” عملیات به ورود به سیستم. - -تکرار فیزیکی است: تنها قطعات فشرده بین گره منتقل, نمایش داده شد نیست. ادغام در هر ماکت پردازش به طور مستقل در اکثر موارد به کاهش هزینه های شبکه با اجتناب از تقویت شبکه. قطعات با هم ادغام شدند بزرگ بر روی شبکه تنها در موارد تاخیر تکرار قابل توجهی ارسال. - -بعلاوه, هر ماکت ذخیره دولت خود را در باغ وحش به عنوان مجموعه ای از قطعات و چک سام خود. هنگامی که دولت در فایل سیستم محلی واگرا از دولت مرجع در باغ وحش, ماکت بازیابی سازگاری خود را با دانلود قطعات گم شده و شکسته از دیگر کپی. هنگامی که برخی از داده های غیر منتظره و یا شکسته در فایل سیستم محلی وجود دارد, خانه را حذف کنید, اما حرکت می کند به یک دایرکتوری جداگانه و فراموش. - -!!! note "یادداشت" - خوشه محل کلیک متشکل از خرده ریز مستقل, و هر سفال شامل کپی. خوشه است **الاستیک نیست**, بنابراین پس از اضافه کردن یک سفال جدید, داده ها بین خرده ریز به طور خودکار توازن نیست. بجای, بار خوشه قرار است تنظیم شود ناهموار. این پیاده سازی به شما کنترل بیشتری می دهد و برای خوشه های نسبتا کوچک مانند ده ها گره مناسب است. اما برای خوشه با صدها گره که ما با استفاده از در تولید, این رویکرد یک نقطه ضعف قابل توجهی می شود. ما باید یک موتور جدول است که دهانه در سراسر خوشه با مناطق به صورت پویا تکرار است که می تواند تقسیم و متعادل کننده شده بین خوشه به طور خودکار پیاده سازی. - -{## [مقاله اصلی](https://clickhouse.tech/docs/en/development/architecture/) ##} diff --git a/docs/fa/development/browse-code.md b/docs/fa/development/browse-code.md deleted file mode 100644 index 9d5e709b760..00000000000 --- a/docs/fa/development/browse-code.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 63 -toc_title: "\u0645\u0631\u0648\u0631 \u06A9\u062F \u0645\u0646\u0628\u0639" ---- - -# مرور کد منبع کلیک {#browse-clickhouse-source-code} - -شما می توانید استفاده کنید **ووبوک** مرورگر کد اینترنتی موجود است [اینجا](https://clickhouse.tech/codebrowser/html_report/ClickHouse/src/index.html). این فراهم می کند ناوبری کد و برجسته معنایی, جستجو و نمایه سازی. عکس فوری کد روزانه به روز می شود. - -همچنین شما می توانید فهرست منابع در [گیتهاب](https://github.com/ClickHouse/ClickHouse) طبق معمول - -اگر شما علاقه مند هستید چه IDE استفاده کنید توصیه می کنیم CLion, QT Creator, مقابل کد و KDevelop (با هشدارهای). شما می توانید هر محیط برنامه نویسی مورد علاقه استفاده کنید. ویم و ایمکس نیز حساب می کنند. diff --git a/docs/fa/development/build-cross-arm.md b/docs/fa/development/build-cross-arm.md deleted file mode 100644 index fd13441b87e..00000000000 --- a/docs/fa/development/build-cross-arm.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 67 -toc_title: "\u0686\u06AF\u0648\u0646\u0647 \u0628\u0631\u0627\u06CC \u0633\u0627\u062E\ - \u062A ClickHouse \u062F\u0631 \u0644\u06CC\u0646\u0648\u06A9\u0633 \u0628\u0631\ - \u0627\u06CC AARCH64 (ARM64)" ---- - -# چگونه برای ساخت ClickHouse در لینوکس برای AARCH64 (ARM64) معماری {#how-to-build-clickhouse-on-linux-for-aarch64-arm64-architecture} - -این برای مواردی است که شما دستگاه لینوکس دارید و می خواهید از این برای ساخت استفاده کنید `clickhouse` دودویی که در یک ماشین لینوکس دیگر با معماری پردازنده عاشق64 اجرا خواهد شد. این است که برای چک ادغام مداوم است که بر روی سرور های لینوکس اجرا در نظر گرفته شده. - -صلیب-ساخت برای AARCH64 است که بر اساس [ساخت دستورالعمل](build.md) اول دنبالشون کن - -# نصب کلانگ-8 {#install-clang-8} - -دستورالعمل از دنبال https://apt.llvm.org / برای اوبونتو یا دبیان راه اندازی خود را. -مثلا, در اوبونتو بیونیک شما می توانید دستورات زیر استفاده کنید: - -``` bash -echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" | sudo tee /etc/apt/sources.list.d/llvm.list -sudo apt-get update -sudo apt-get install clang-8 -``` - -# نصب مجموعه ابزار صلیب کشی {#install-cross-compilation-toolset} - -``` bash -cd ClickHouse -mkdir -p build-aarch64/cmake/toolchain/linux-aarch64 -wget 'https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en' -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build-aarch64/cmake/toolchain/linux-aarch64 --strip-components=1 -``` - -# ساخت خانه کلیک {#build-clickhouse} - -``` bash -cd ClickHouse -mkdir build-arm64 -CC=clang-8 CXX=clang++-8 cmake . -Bbuild-arm64 -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-aarch64.cmake -ninja -C build-arm64 -``` - -باینری حاصل تنها در لینوکس با معماری پردازنده اروچ64 اجرا خواهد شد. diff --git a/docs/fa/development/build-cross-osx.md b/docs/fa/development/build-cross-osx.md deleted file mode 100644 index 9b963ad1e38..00000000000 --- a/docs/fa/development/build-cross-osx.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 66 -toc_title: "\u0686\u06AF\u0648\u0646\u0647 \u0628\u0631\u0627\u06CC \u0633\u0627\u062E\ - \u062A \u062A\u0627\u062A\u0631 \u062F\u0631 \u0644\u06CC\u0646\u0648\u06A9\u0633\ - \ \u0628\u0631\u0627\u06CC \u0633\u06CC\u0633\u062A\u0645 \u0639\u0627\u0645\u0644\ - \ \u0645\u06A9 \u0627\u06CC\u06A9\u0633" ---- - -# چگونه برای ساخت تاتر در لینوکس برای سیستم عامل مک ایکس {#how-to-build-clickhouse-on-linux-for-mac-os-x} - -این برای مواردی است که شما دستگاه لینوکس دارید و می خواهید از این برای ساخت استفاده کنید `clickhouse` این است که برای چک ادغام مداوم است که بر روی سرور های لینوکس اجرا در نظر گرفته شده. اگر شما می خواهید برای ساخت خانه کلیک به طور مستقیم در سیستم عامل مک ایکس, سپس با ادامه [دستورالعمل دیگر](build-osx.md). - -کراس ساخت برای سیستم عامل مک ایکس بر اساس [ساخت دستورالعمل](build.md) اول دنبالشون کن - -# نصب کلانگ-8 {#install-clang-8} - -دستورالعمل از دنبال https://apt.llvm.org / برای اوبونتو یا دبیان راه اندازی خود را. -به عنوان مثال دستورات برای بیونیک مانند: - -``` bash -sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" >> /etc/apt/sources.list -sudo apt-get install clang-8 -``` - -# نصب مجموعه ابزار صلیب کشی {#install-cross-compilation-toolset} - -بیایید مسیری را که ما نصب می کنیم به یاد داشته باشیم `cctools` به عنوان ${CCTOOLS} - -``` bash -mkdir ${CCTOOLS} - -git clone https://github.com/tpoechtrager/apple-libtapi.git -cd apple-libtapi -INSTALLPREFIX=${CCTOOLS} ./build.sh -./install.sh -cd .. - -git clone https://github.com/tpoechtrager/cctools-port.git -cd cctools-port/cctools -./configure --prefix=${CCTOOLS} --with-libtapi=${CCTOOLS} --target=x86_64-apple-darwin -make install -``` - -همچنین, ما نیاز به دانلود ماکو ایکس انحراف معیار به درخت کار. - -``` bash -cd ClickHouse -wget 'https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz' -mkdir -p build-darwin/cmake/toolchain/darwin-x86_64 -tar xJf MacOSX10.14.sdk.tar.xz -C build-darwin/cmake/toolchain/darwin-x86_64 --strip-components=1 -``` - -# ساخت خانه کلیک {#build-clickhouse} - -``` bash -cd ClickHouse -mkdir build-osx -CC=clang-8 CXX=clang++-8 cmake . -Bbuild-osx -DCMAKE_TOOLCHAIN_FILE=cmake/darwin/toolchain-x86_64.cmake \ - -DCMAKE_AR:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ar \ - -DCMAKE_RANLIB:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ranlib \ - -DLINKER_NAME=${CCTOOLS}/bin/x86_64-apple-darwin-ld -ninja -C build-osx -``` - -باینری حاصل یک فرمت اجرایی ماخ ای داشته باشد و نمی تواند در لینوکس اجرا شود. diff --git a/docs/fa/development/build-osx.md b/docs/fa/development/build-osx.md deleted file mode 100644 index 4002dad1f40..00000000000 --- a/docs/fa/development/build-osx.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 65 -toc_title: "\u0686\u06AF\u0648\u0646\u0647 \u0628\u0631\u0627\u06CC \u0633\u0627\u062E\ - \u062A \u062A\u0627\u062A\u0631 \u062F\u0631 \u0633\u06CC\u0633\u062A\u0645 \u0639\ - \u0627\u0645\u0644 \u0645\u06A9 \u0627\u06CC\u06A9\u0633" ---- - -# چگونه برای ساخت تاتر در سیستم عامل مک ایکس {#how-to-build-clickhouse-on-mac-os-x} - -ساخت باید در سیستم عامل مک ایکس کار 10.15 (کاتالینا) - -## نصب گشتن {#install-homebrew} - -``` bash -$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -``` - -## نصب کامپایلرهای مورد نیاز, ابزار, و کتابخانه {#install-required-compilers-tools-and-libraries} - -``` bash -$ brew install cmake ninja libtool gettext -``` - -## پرداخت منابع کلیک {#checkout-clickhouse-sources} - -``` bash -$ git clone --recursive git@github.com:ClickHouse/ClickHouse.git -``` - -یا - -``` bash -$ git clone --recursive https://github.com/ClickHouse/ClickHouse.git - -$ cd ClickHouse -``` - -## ساخت خانه کلیک {#build-clickhouse} - -``` bash -$ mkdir build -$ cd build -$ cmake .. -DCMAKE_CXX_COMPILER=`which clang++` -DCMAKE_C_COMPILER=`which clang` -$ ninja -$ cd .. -``` - -## هشدارها {#caveats} - -اگر شما قصد اجرای clickhouse-سرور مطمئن شوید که برای افزایش سیستم maxfiles متغیر است. - -!!! info "یادداشت" - باید از سودو استفاده کنی - -برای انجام این کار فایل زیر را ایجاد کنید: - -/Library/LaunchDaemons/محدود می کند.مکسفیلزجان کلام: - -``` xml - - - - - Label - limit.maxfiles - ProgramArguments - - launchctl - limit - maxfiles - 524288 - 524288 - - RunAtLoad - - ServiceIPC - - - -``` - -دستور زیر را اجرا کنید: - -``` bash -$ sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist -``` - -راه اندازی مجدد. - -برای بررسی اگر این کار, شما می توانید استفاده کنید `ulimit -n` فرمان. - -[مقاله اصلی](https://clickhouse.tech/docs/en/development/build_osx/) diff --git a/docs/fa/development/build.md b/docs/fa/development/build.md deleted file mode 100644 index 8126578f9d5..00000000000 --- a/docs/fa/development/build.md +++ /dev/null @@ -1,142 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 64 -toc_title: "\u0646\u062D\u0648\u0647 \u0633\u0627\u062E\u062A \u06A9\u0644\u06CC\u06A9\ - \ \u062F\u0631 \u0644\u06CC\u0646\u0648\u06A9\u0633" ---- - -# چگونه برای ساخت خانه کلیک برای توسعه {#how-to-build-clickhouse-for-development} - -راهنمای زیر بر اساس سیستم لینوکس اوبونتو است. -با تغییرات مناسب, همچنین باید بر روی هر توزیع لینوکس دیگر کار. -سیستم عامل های پشتیبانی شده: ایکس86_64 و عاشق64. پشتیبانی از قدرت9 تجربی است. - -## شرح متنی (توضیحات سایت) در صفحات شما دیده نمی شود {#install-git-cmake-python-and-ninja} - -``` bash -$ sudo apt-get install git cmake python ninja-build -``` - -یا سیمک 3 به جای کیک در سیستم های قدیمی تر. - -## نصب شورای همکاری خلیج فارس 9 {#install-gcc-10} - -راه های مختلفی برای انجام این کار وجود دارد. - -### نصب از یک بسته پپا {#install-from-a-ppa-package} - -``` bash -$ sudo apt-get install software-properties-common -$ sudo apt-add-repository ppa:ubuntu-toolchain-r/test -$ sudo apt-get update -$ sudo apt-get install gcc-10 g++-10 -``` - -### نصب از منابع {#install-from-sources} - -نگاه کن [utils/ci/build-gcc-from-sources.sh](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh) - -## استفاده از شورای همکاری خلیج فارس 10 برای ساخت {#use-gcc-10-for-builds} - -``` bash -$ export CC=gcc-10 -$ export CXX=g++-10 -``` - -## پرداخت منابع کلیک {#checkout-clickhouse-sources} - -``` bash -$ git clone --recursive git@github.com:ClickHouse/ClickHouse.git -``` - -یا - -``` bash -$ git clone --recursive https://github.com/ClickHouse/ClickHouse.git -``` - -## ساخت خانه کلیک {#build-clickhouse} - -``` bash -$ cd ClickHouse -$ mkdir build -$ cd build -$ cmake .. -$ ninja -$ cd .. -``` - -برای ایجاد یک اجرایی, اجرا `ninja clickhouse`. -این ایجاد خواهد شد `programs/clickhouse` قابل اجرا است که می تواند با استفاده `client` یا `server` بحث کردن. - -# چگونه برای ساخت کلیک بر روی هر لینوکس {#how-to-build-clickhouse-on-any-linux} - -ساخت نیاز به اجزای زیر دارد: - -- دستگاه گوارش (استفاده می شود تنها به پرداخت منابع مورد نیاز برای ساخت) -- کیک 3.10 یا جدیدتر -- نینجا (توصیه می شود) و یا -- ج ++ کامپایلر: شورای همکاری خلیج فارس 10 یا صدای شیپور 8 یا جدیدتر -- لینکر: لیلند یا طلا (کلاسیک گنو الدی کار نخواهد کرد) -- پایتون (فقط در داخل ساخت لورم استفاده می شود و اختیاری است) - -اگر تمام اجزای نصب شده, شما ممکن است در همان راه به عنوان مراحل بالا ساخت. - -به عنوان مثال برای اوبونتو ایوان: - - sudo apt update - sudo apt install git cmake ninja-build g++ python - git clone --recursive https://github.com/ClickHouse/ClickHouse.git - mkdir build && cd build - cmake ../ClickHouse - ninja - -به عنوان مثال برای لینوکس تاج خروس: - - sudo zypper install git cmake ninja gcc-c++ python lld - git clone --recursive https://github.com/ClickHouse/ClickHouse.git - mkdir build && cd build - cmake ../ClickHouse - ninja - -به عنوان مثال برای فدورا پوست دباغی نشده: - - sudo yum update - yum --nogpg install git cmake make gcc-c++ python3 - git clone --recursive https://github.com/ClickHouse/ClickHouse.git - mkdir build && cd build - cmake ../ClickHouse - make -j $(nproc) - -# شما لازم نیست برای ساخت کلیک {#you-dont-have-to-build-clickhouse} - -تاتر در فایل های باینری از پیش ساخته شده و بسته های موجود است. فایل های باینری قابل حمل هستند و می تواند بر روی هر عطر و طعم لینوکس اجرا شود. - -تا زمانی که برای هر متعهد به کارشناسی کارشناسی ارشد و برای هر درخواست کشش ساخته شده است برای انتشار پایدار و قابل پرست و تست. - -برای پیدا کردن تازه ترین ساخت از `master` برو به [مرتکب صفحه](https://github.com/ClickHouse/ClickHouse/commits/master), با کلیک بر روی اولین علامت سبز یا صلیب قرمز در نزدیکی ارتکاب, کلیک کنید و به “Details” پیوند درست بعد از “ClickHouse Build Check”. - -# چگونه برای ساخت مخزن دبیان بسته {#how-to-build-clickhouse-debian-package} - -## نصب برنامه جی تی و پل ساز {#install-git-and-pbuilder} - -``` bash -$ sudo apt-get update -$ sudo apt-get install git python pbuilder debhelper lsb-release fakeroot sudo debian-archive-keyring debian-keyring -``` - -## پرداخت منابع کلیک {#checkout-clickhouse-sources-1} - -``` bash -$ git clone --recursive --branch master https://github.com/ClickHouse/ClickHouse.git -$ cd ClickHouse -``` - -## اجرای اسکریپت انتشار {#run-release-script} - -``` bash -$ ./release -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/development/build/) diff --git a/docs/fa/development/contrib.md b/docs/fa/development/contrib.md deleted file mode 100644 index 2ee5fc73369..00000000000 --- a/docs/fa/development/contrib.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 70 -toc_title: "\u06A9\u062A\u0627\u0628\u062E\u0627\u0646\u0647 \u0647\u0627\u06CC \u0634\ - \u062E\u0635 \u062B\u0627\u0644\u062B \u0627\u0633\u062A\u0641\u0627\u062F\u0647\ - \ \u0645\u06CC \u0634\u0648\u062F" ---- - -# کتابخانه های شخص ثالث استفاده می شود {#third-party-libraries-used} - -| کتابخانه | مجوز | -|-----------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| پایگاه64 | [لیسانس 2 بند](https://github.com/aklomp/base64/blob/a27c565d1b6c676beaf297fe503c4518185666f7/LICENSE) | -| افزایش | [افزایش مجوز نرم افزار 1.0](https://github.com/ClickHouse-Extras/boost-extra/blob/6883b40449f378019aec792f9983ce3afc7ff16e/LICENSE_1_0.txt) | -| برتلی | [MIT](https://github.com/google/brotli/blob/master/LICENSE) | -| کاپپروتو | [MIT](https://github.com/capnproto/capnproto/blob/master/LICENSE) | -| رکتتز | [نمایی مجوز 2.0](https://github.com/google/cctz/blob/4f9776a310f4952454636363def82c2bf6641d5f/LICENSE.txt) | -| دو تبدیل | [لیسانس 3 بند](https://github.com/google/double-conversion/blob/cf2f0f3d547dc73b4612028a155b80536902ba02/LICENSE) | -| گام به گام | [MIT](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libmemcpy/impl/LICENSE) | -| googletest | [لیسانس 3 بند](https://github.com/google/googletest/blob/master/LICENSE) | -| اچ 3 | [نمایی مجوز 2.0](https://github.com/uber/h3/blob/master/LICENSE) | -| hyperscan | [لیسانس 3 بند](https://github.com/intel/hyperscan/blob/master/LICENSE) | -| شکنجه نوجوان | [BSD + MIT](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libglibc-compatibility/libcxxabi/LICENSE.TXT) | -| لیبیدوید | [مجوز زلب](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libdivide/LICENSE.txt) | -| نوشیدن شراب | [الجی پی ال2.1](https://github.com/ClickHouse-Extras/libgsasl/blob/3b8948a4042e34fb00b4fb987535dc9e02e39040/LICENSE) | -| لیبهدفس3 | [نمایی مجوز 2.0](https://github.com/ClickHouse-Extras/libhdfs3/blob/bd6505cbb0c130b0db695305b9a38546fa880e5a/LICENSE.txt) | -| کشتی کج | [نمایی مجوز 2.0](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libmetrohash/LICENSE) | -| سوالات متداول | [نمایی مجوز 2.0](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libpcg-random/LICENSE-APACHE.txt) | -| libressl | [OpenSSL مجوز](https://github.com/ClickHouse-Extras/ssl/blob/master/COPYING) | -| کتابدار | [لیسانس 2 بند](https://github.com/edenhill/librdkafka/blob/363dcad5a23dc29381cc626620e68ae418b3af19/LICENSE) | -| _عرض | [CC0 1.0 جهانی](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libwidechar_width/LICENSE) | -| llvm | [لیسانس 3 بند](https://github.com/ClickHouse-Extras/llvm/blob/163def217817c90fb982a6daf384744d8472b92b/llvm/LICENSE.TXT) | -| lz4 | [لیسانس 2 بند](https://github.com/lz4/lz4/blob/c10863b98e1503af90616ae99725ecd120265dfb/LICENSE) | -| ماریادب-اتصال-ج | [الجی پی ال2.1](https://github.com/ClickHouse-Extras/mariadb-connector-c/blob/3.1/COPYING.LIB) | -| سوفلهاش | [دامنه عمومی](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/murmurhash/LICENSE) | -| رایانه های جیبی | [مجوز زلب](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/pdqsort/license.txt) | -| پوکو | [افزایش مجوز نرم افزار-نسخه 1.0](https://github.com/ClickHouse-Extras/poco/blob/fe5505e56c27b6ecb0dcbc40c49dc2caf4e9637f/LICENSE) | -| protobuf | [لیسانس 3 بند](https://github.com/ClickHouse-Extras/protobuf/blob/12735370922a35f03999afff478e1c6d7aa917a4/LICENSE) | -| شماره 2 | [لیسانس 3 بند](https://github.com/google/re2/blob/7cf8b88e8f70f97fd4926b56aa87e7f53b2717e0/LICENSE) | -| UnixODBC | [الجی پی ال2.1](https://github.com/ClickHouse-Extras/UnixODBC/tree/b0ad30f7f6289c12b76f04bfb9d466374bb32168) | -| زلب نانوگرم | [مجوز زلب](https://github.com/ClickHouse-Extras/zlib-ng/blob/develop/LICENSE.md) | -| زد | [لیسانس 3 بند](https://github.com/facebook/zstd/blob/dev/LICENSE) | diff --git a/docs/fa/development/developer-instruction.md b/docs/fa/development/developer-instruction.md deleted file mode 100644 index 9284d3a511a..00000000000 --- a/docs/fa/development/developer-instruction.md +++ /dev/null @@ -1,289 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 61 -toc_title: "\u062F\u0633\u062A\u0648\u0631\u0627\u0644\u0639\u0645\u0644 \u062A\u0648\ - \u0633\u0639\u0647 \u062F\u0647\u0646\u062F\u0647 \u06A9\u0644\u06CC\u06A9 \u0645\ - \u0628\u062A\u062F\u06CC" ---- - -ساختمان از خانه کلیک بر روی لینوکس پشتیبانی, بورس و سیستم عامل مک ایکس. - -# در صورت استفاده از ویندوز {#if-you-use-windows} - -اگر شما استفاده از ویندوز, شما نیاز به ایجاد یک ماشین مجازی با اوبونتو. برای شروع کار با یک ماشین مجازی لطفا مجازی نصب کنید. شما می توانید اوبونتو را از وب سایت دانلود کنید: https://www.ubuntu.com/#download.لطفا یک ماشین مجازی از تصویر دانلود شده ایجاد کنید (شما باید حداقل 4 گیگابایت رم را رزرو کنید). برای اجرای یک ترمینال خط فرمان در اوبونتو, لطفا یک برنامه حاوی کلمه قرار “terminal” به نام (گنوم ترمینال, کنسول و غیره.) یا فقط کنترل را فشار دهید - -# اگر از یک سیستم 32 بیتی استفاده می کنید {#if-you-use-a-32-bit-system} - -تاتر نمی تواند کار کند و یا ساخت بر روی یک سیستم 32 بیتی. شما باید دسترسی به یک سیستم 64 بیتی کسب و شما می توانید ادامه مطلب. - -# ایجاد یک مخزن در گیتهاب {#creating-a-repository-on-github} - -برای شروع کار با مخزن کلیک شما یک حساب گیتهاب نیاز. - -شما احتمالا در حال حاضر یکی, اما اگر اینکار را نکنید, لطفا ثبت نام در https://github.com. در صورتی که کلید های سش را ندارید باید تولید کنید و سپس در گیتهاب بارگذاری کنید. این برای ارسال بیش از تکه های خود را مورد نیاز است. همچنین ممکن است به استفاده از کلید همان جلسه که شما با هر سرور جلسه دیگر استفاده کنید - احتمالا شما در حال حاضر کسانی که. - -ایجاد یک چنگال مخزن مخزن مخزن. برای انجام این کار لطفا بر روی کلیک کنید “fork” دکمه در گوشه سمت راست بالا در https://github.com/ClickHouse/ClickHouse. آن را به چنگال خود کپی ClickHouse/ClickHouse به حساب کاربری خود. - -روند توسعه شامل اولین ارتکاب تغییرات در نظر گرفته شده را به چنگال خود را از خانه رعیتی و سپس ایجاد یک “pull request” برای این تغییرات پذیرفته می شود به مخزن اصلی (ClickHouse/ClickHouse). - -برای کار با مخازن دستگاه گوارش, لطفا نصب کنید `git`. - -برای انجام این کار در اوبونتو شما در ترمینال خط فرمان اجرا می کنید: - - sudo apt update - sudo apt install git - -کتابچه راهنمای مختصر در استفاده از دستگاه گوارش را می توان یافت: https://education.github.com/git-cheat-sheet-education.pdf. -برای یک کتابچه راهنمای دقیق در دستگاه گوارش را ببینید https://git-scm.com/book/en/v2. - -# شبیه سازی یک مخزن به دستگاه توسعه خود را {#cloning-a-repository-to-your-development-machine} - -بعد, شما نیاز به دانلود فایل های منبع بر روی دستگاه کار خود را. این است که به نام “to clone a repository” زیرا ایجاد یک کپی محلی از مخزن بر روی دستگاه کار خود را. - -در خط فرمان ترمینال اجرا: - - git clone --recursive git@github.com:your_github_username/ClickHouse.git - cd ClickHouse - -توجه: لطفا جایگزین کنید *تغییر _نامهی تو* با چه مناسب است! - -این دستور یک دایرکتوری ایجاد خواهد کرد `ClickHouse` حاوی کپی کار از پروژه. - -مهم این است که مسیر به دایرکتوری کار شامل هیچ فضای سفید به عنوان ممکن است به مشکلات در حال اجرا سیستم ساخت منجر شود. - -لطفا توجه داشته باشید که مخزن کلیک استفاده می کند `submodules`. That is what the references to additional repositories are called (i.e. external libraries on which the project depends). It means that when cloning the repository you need to specify the `--recursive` پرچم همانطور که در مثال بالا. اگر مخزن بدون زیر دستی مسدود شده باشد باید موارد زیر را دانلود کنید: - - git submodule init - git submodule update - -شما می توانید وضعیت را با فرمان بررسی کنید: `git submodule status`. - -اگر پیغام خطای زیر را دریافت کنید: - - Permission denied (publickey). - fatal: Could not read from remote repository. - - Please make sure you have the correct access rights - and the repository exists. - -به طور کلی به این معنی است که کلید های برش برای اتصال به گیتهاب از دست رفته است. این کلید ها به طور معمول در واقع `~/.ssh`. برای کلید های جلسه پذیرفته می شود شما نیاز به ارسال در بخش تنظیمات رابط کاربر گیتهاب. - -شما همچنین می توانید مخزن از طریق پروتکل قام کلون: - - git clone https://github.com/ClickHouse/ClickHouse.git - -این, با این حال, نمی خواهد به شما اجازه تغییرات خود را به سرور ارسال. شما هنوز هم می توانید به طور موقت استفاده کنید و اضافه کردن کلید های جلسه بعد جایگزین نشانی از راه دور از مخزن با `git remote` فرمان. - -شما همچنین می توانید نشانی اصلی مخزن مخزن محلی خود را اضافه کنید به جلو و به روز رسانی از وجود دارد: - - git remote add upstream git@github.com:ClickHouse/ClickHouse.git - -پس از موفقیت در حال اجرا این دستور شما قادر خواهید بود به جلو و به روز رسانی از مخزن کلیک اصلی در حال اجرا خواهد بود `git pull upstream master`. - -## کار با Submodules {#working-with-submodules} - -کار با زیربول در دستگاه گوارش می تواند دردناک باشد. دستورات بعدی کمک خواهد کرد که برای مدیریت: - - # ! each command accepts --recursive - # Update remote URLs for submodules. Barely rare case - git submodule sync - # Add new submodules - git submodule init - # Update existing submodules to the current state - git submodule update - # Two last commands could be merged together - git submodule update --init - -دستورات بعدی کمک خواهد کرد که شما را به تنظیم مجدد تمام زیربول به حالت اولیه (!هشدار! - هر گونه تغییر در داخل حذف خواهد شد): - - # Synchronizes submodules' remote URL with .gitmodules - git submodule sync --recursive - # Update the registered submodules with initialize not yet initialized - git submodule update --init --recursive - # Reset all changes done after HEAD - git submodule foreach git reset --hard - # Clean files from .gitignore - git submodule foreach git clean -xfd - # Repeat last 4 commands for all submodule - git submodule foreach git submodule sync --recursive - git submodule foreach git submodule update --init --recursive - git submodule foreach git submodule foreach git reset --hard - git submodule foreach git submodule foreach git clean -xfd - -# ساخت سیستم {#build-system} - -تاتر با استفاده از کیک و نینجا برای ساخت و ساز. - -کیک-یک سیستم متا ساخت است که می تواند فایل های نینجا (ساخت وظایف) تولید کند. -نینجا-یک سیستم ساخت کوچکتر با تمرکز بر سرعت مورد استفاده برای اجرای این کارهای تولید کیک. - -برای نصب در اوبونتو, دبیان و یا نعنا اجرا `sudo apt install cmake ninja-build`. - -در حال بارگذاری `sudo yum install cmake ninja-build`. - -اگر شما استفاده از قوس یا جنتو, شما احتمالا خودتان می دانید که چگونه به نصب کیک. - -برای نصب کیک و نینجا در سیستم عامل مک ایکس اول گشتن نصب و سپس نصب هر چیز دیگری از طریق دم: - - /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - brew install cmake ninja - -بعد, بررسی نسخه از کیک: `cmake --version`. اگر زیر 3.3, شما باید یک نسخه جدیدتر از وب سایت نصب: https://cmake.org/download/. - -# کتابخانه های خارجی اختیاری {#optional-external-libraries} - -تاتر با استفاده از چندین کتابخانه خارجی برای ساخت و ساز. همه آنها نمی نیاز به نصب به طور جداگانه به عنوان آنها ساخته شده است همراه با ClickHouse از منابع واقع در submodules. شما می توانید لیست را بررسی کنید `contrib`. - -# ج ++ کامپایلر {#c-compiler} - -کامپایلر شورای همکاری خلیج فارس با شروع از نسخه 10 و صدای شیپور نسخه 8 یا بالاتر برای ساخت و ساز خانه عروسکی پشتیبانی می کند. - -یاندکس رسمی ایجاد شده در حال حاضر با استفاده از شورای همکاری خلیج فارس به دلیل تولید کد ماشین از عملکرد کمی بهتر (بازده تفاوت تا چند درصد با توجه به معیار ما). و کلانگ معمولا برای توسعه راحت تر است. هر چند, ادغام مداوم ما (سی) پلت فرم اجرا می شود چک برای حدود یک دوجین از ترکیب ساخت. - -برای نصب شورای همکاری خلیج فارس در اوبونتو اجرای: `sudo apt install gcc g++` - -بررسی نسخه شورای همکاری خلیج فارس: `gcc --version`. اگر زیر است 10, سپس دستورالعمل اینجا را دنبال کنید: https://clickhouse.tech/docs/fa/development/build/#install-gcc-10. - -سیستم عامل مک ایکس ساخت فقط برای صدای جرنگ جرنگ پشتیبانی می شود. فقط فرار کن `brew install llvm` - -اگر شما تصمیم به استفاده از صدای شیپور, شما همچنین می توانید نصب `libc++` و `lld`, اگر شما می دانید چه چیزی است. با استفاده از `ccache` همچنین توصیه می شود. - -# روند ساخت و ساز {#the-building-process} - -حالا که اماده ساخت خانه عروسکی هستید توصیه می کنیم یک دایرکتوری جداگانه ایجاد کنید `build` داخل `ClickHouse` که شامل تمام مصنوعات ساخت: - - mkdir build - cd build - -شما می توانید چندین دایرکتوری های مختلف (build_release, build_debug ، ) برای انواع مختلف ساخت. - -در حالی که در داخل `build` فهرست, پیکربندی ساخت خود را با در حال اجرا کیک. قبل از اولین اجرا, شما نیاز به تعریف متغیرهای محیطی که کامپایلر را مشخص (نسخه 10 کامپایلر شورای همکاری خلیج فارس در این مثال). - -لینوکس: - - export CC=gcc-10 CXX=g++-10 - cmake .. - -سیستم عامل مک ایکس: - - export CC=clang CXX=clang++ - cmake .. - -این `CC` متغیر کامپایلر برای ج مشخص (کوتاه برای کامپایلر ج), و `CXX` دستور متغیر که سی++ کامپایلر است که برای ساخت و ساز استفاده می شود. - -برای ساخت سریع تر, شما می توانید به توسل `debug` نوع ساخت-ساخت بدون بهینه سازی. برای عرضه پارامتر زیر `-D CMAKE_BUILD_TYPE=Debug`: - - cmake -D CMAKE_BUILD_TYPE=Debug .. - -شما می توانید نوع ساخت را با اجرای این دستور در تغییر دهید `build` فهرست راهنما. - -اجرای نینجا برای ساخت: - - ninja clickhouse-server clickhouse-client - -فقط باینری مورد نیاز در حال رفتن به در این مثال ساخته شده است. - -اگر شما نیاز به ساخت تمام فایل های باینری (تاسیسات و تست), شما باید نینجا بدون پارامتر اجرا: - - ninja - -ساخت کامل نیاز به حدود 30 گیگابایت فضای دیسک رایگان یا 15 گیگابایت برای ساخت باینری اصلی دارد. - -هنگامی که مقدار زیادی از رم در ساخت دستگاه در دسترس است شما باید تعداد وظایف ساخت به صورت موازی با اجرا محدود می کند `-j` پرم: - - ninja -j 1 clickhouse-server clickhouse-client - -در ماشین با 4 گیگابایت رم, توصیه می شود برای مشخص 1, برای 8گیگابایت رم `-j 2` توصیه می شود. - -اگر پیام را دریافت کنید: `ninja: error: loading 'build.ninja': No such file or directory`, به این معنی که تولید یک پیکربندی ساخت شکست خورده است و شما نیاز به بازرسی پیام بالا. - -پس از شروع موفق از روند ساخت و ساز, شما پیشرفت ساخت را ببینید-تعداد کارهای پردازش شده و تعداد کل وظایف. - -در حالی که ساختمان پیام در مورد protobuf فایل در libhdfs2 کتابخانه مانند `libprotobuf WARNING` ممکن است نشان دهد تا. هیچ چیز تاثیر می گذارد و امن نادیده گرفته می شود. - -پس از ساخت موفق شما یک فایل اجرایی دریافت کنید `ClickHouse//programs/clickhouse`: - - ls -l programs/clickhouse - -# اجرای اجرایی ساخته شده از خانه کلیک {#running-the-built-executable-of-clickhouse} - -برای اجرای سرور تحت کاربر فعلی شما نیاز به حرکت به `ClickHouse/programs/server/` (واقع در خارج از `build`) و اجرا: - - ../../build/programs/clickhouse server - -در این مورد, تاتر خواهد فایل های پیکربندی واقع در دایرکتوری جاری استفاده. شما می توانید اجرا کنید `clickhouse server` از هر دایرکتوری مشخص کردن مسیر به یک فایل پیکربندی به عنوان یک پارامتر خط فرمان `--config-file`. - -برای اتصال به ClickHouse با clickhouse-مشتری در یکی دیگر از ترمینال حرکت به `ClickHouse/build/programs/` و فرار کن `./clickhouse client`. - -اگر شما `Connection refused` سعی کنید مشخص نشانی میزبان 127.0.0.1: - - clickhouse client --host 127.0.0.1 - -شما می توانید جایگزین تولید نسخه ClickHouse باینری در سیستم شما نصب شده خود را با سفارشی ساخته شده ClickHouse دودویی. برای انجام این کار نصب کلیک بر روی دستگاه خود را به دنبال دستورالعمل از وب سایت رسمی. بعد زیر را اجرا کنید: - - sudo service clickhouse-server stop - sudo cp ClickHouse/build/programs/clickhouse /usr/bin/ - sudo service clickhouse-server start - -توجه داشته باشید که `clickhouse-client`, `clickhouse-server` و دیگران به طور معمول به اشتراک گذاشته می شوند `clickhouse` دودویی. - -شما همچنین می توانید خود را سفارشی ساخته شده ClickHouse دودویی با فایل پیکربندی از ClickHouse بسته نصب شده در سیستم شما: - - sudo service clickhouse-server stop - sudo -u clickhouse ClickHouse/build/programs/clickhouse server --config-file /etc/clickhouse-server/config.xml - -# محیط توسعه یکپارچه) {#ide-integrated-development-environment} - -اگر شما نمی دانید که محیط برنامه نویسی برای استفاده, توصیه می کنیم که شما با استفاده از کلون. کلوون نرم افزار تجاری است, اما 30 روز رایگان دوره محاکمه. این نیز رایگان برای دانشجویان. CLion می توان هم بر روی لینوکس و Mac OS X. - -KDevelop و QTCreator دیگر از جایگزین های بسیار خوبی از یک IDE برای توسعه ClickHouse. توسعه و توسعه به عنوان یک محیط برنامه نویسی بسیار مفید هر چند ناپایدار. اگر توسعه پس از مدتی پس از باز کردن پروژه سقوط, شما باید کلیک کنید “Stop All” دکمه به محض این که لیستی از فایل های پروژه را باز کرده است. پس از انجام این کار کدولاپ باید خوب باشد برای کار با. - -به عنوان ویراستاران کد ساده, شما می توانید متن والا و یا کد ویژوال استودیو استفاده, یا کیت (که همه در دسترس هستند در لینوکس). - -فقط در مورد لازم به ذکر است که CLion ایجاد `build` مسیر خود را نیز در انتخاب خود `debug` برای ساخت نوع پیکربندی آن را با استفاده از یک نسخه از CMake که تعریف شده است در CLion و نه یک نصب شده توسط شما, و در نهایت, CLion استفاده خواهد کرد `make` برای اجرای وظایف ساخت به جای `ninja`. این رفتار طبیعی است, فقط نگه دارید که در ذهن برای جلوگیری از سردرگمی. - -# نوشتن کد {#writing-code} - -شرح ClickHouse معماری را می توان در اینجا یافت نشد: https://clickhouse.فناوری / اسناد/مهندسی / توسعه / معماری/ - -راهنمای سبک کد: https://clickhouse.فناوری / اسناد/در/توسعه / سبک/ - -تست نوشتن: https://clickhouse.فناوری / اسناد/توسعه/تست/ - -فهرست تکلیفها: https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aopen+is%3Aissue+label%3A%22easy+task%22 - -# داده های تست {#test-data} - -در حال توسعه تاتر اغلب نیاز به بارگذاری مجموعه داده های واقع بینانه است. این امر به ویژه برای تست عملکرد مهم است. ما یک مجموعه خاص تهیه شده از داده های ناشناس از یاندکس.متریکا این علاوه بر برخی از 3 گیگابایت فضای دیسک رایگان نیاز دارد. توجه داشته باشید که این داده ها مورد نیاز است برای به انجام رساندن بسیاری از وظایف توسعه. - - sudo apt install wget xz-utils - - wget https://datasets.clickhouse.tech/hits/tsv/hits_v1.tsv.xz - wget https://datasets.clickhouse.tech/visits/tsv/visits_v1.tsv.xz - - xz -v -d hits_v1.tsv.xz - xz -v -d visits_v1.tsv.xz - - clickhouse-client - - CREATE DATABASE IF NOT EXISTS test - - CREATE TABLE test.hits ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime); - - CREATE TABLE test.visits ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), `Goals.ID` Array(UInt32), `Goals.Serial` Array(UInt32), `Goals.EventTime` Array(DateTime), `Goals.Price` Array(Int64), `Goals.OrderID` Array(String), `Goals.CurrencyID` Array(UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, `TraficSource.ID` Array(Int8), `TraficSource.SearchEngineID` Array(UInt16), `TraficSource.AdvEngineID` Array(UInt8), `TraficSource.PlaceID` Array(UInt16), `TraficSource.SocialSourceNetworkID` Array(UInt8), `TraficSource.Domain` Array(String), `TraficSource.SearchPhrase` Array(String), `TraficSource.SocialSourcePage` Array(String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), `Market.Type` Array(UInt8), `Market.GoalID` Array(UInt32), `Market.OrderID` Array(String), `Market.OrderPrice` Array(Int64), `Market.PP` Array(UInt32), `Market.DirectPlaceID` Array(UInt32), `Market.DirectOrderID` Array(UInt32), `Market.DirectBannerID` Array(UInt32), `Market.GoodID` Array(String), `Market.GoodName` Array(String), `Market.GoodQuantity` Array(Int32), `Market.GoodPrice` Array(Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID); - - clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.hits FORMAT TSV" < hits_v1.tsv - clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.visits FORMAT TSV" < visits_v1.tsv - -# ایجاد درخواست کشش {#creating-pull-request} - -حرکت به مخزن چنگال خود را در رابط کاربر گیتهاب است. اگر شما شده اند در حال توسعه در یک شاخه, شما نیاز به انتخاب کنید که شاخه. وجود خواهد داشت “Pull request” دکمه واقع بر روی صفحه نمایش. در اصل این به این معنی است “create a request for accepting my changes into the main repository”. - -درخواست کشش را می توان ایجاد حتی اگر کار کامل نشده است. در این مورد لطفا کلمه را قرار دهید “WIP” (کار در حال پیشرفت) در ابتدای عنوان می تواند بعدا تغییر کند. این برای بررسی تعاونی و بحث در مورد تغییرات و همچنین برای اجرای تمام تست های موجود مفید است. این مهم است که شما شرح مختصری از تغییرات خود را فراهم, بعد برای تولید تغییرات انتشار استفاده خواهد شد. - -تست شروع خواهد شد به عنوان به زودی به عنوان کارکنان یاندکس برچسب روابط عمومی خود را با یک برچسب “can be tested”. The results of some first checks (e.g. code style) will come in within several minutes. Build check results will arrive within half an hour. And the main set of tests will report itself within an hour. - -این سیستم خواهد باینری کلیک ایجاد شده برای درخواست کشش خود را به صورت جداگانه تهیه. برای بازیابی این ایجاد کلیک کنید “Details” پیوند بعدی به “ClickHouse build check” ورود در لیست چک. وجود دارد شما لینک مستقیم به ساخته شده پیدا کنید .بسته دب از تاتر که شما می توانید حتی بر روی سرور تولید خود را استقرار (اگر شما هیچ ترس). - -به احتمال زیاد برخی از ایجاد خواهد شد شکست در اولین بار. این به خاطر این واقعیت است که ما بررسی می کنیم ایجاد هر دو با شورای همکاری خلیج فارس و همچنین با صدای جرنگ, با تقریبا تمام هشدارهای موجود (همیشه با `-Werror` پرچم) را فعال کنید برای صدای جرنگ جرنگ. در همان صفحه, شما می توانید تمام سیاهههای مربوط ساخت پیدا به طوری که شما لازم نیست که برای ساخت خانه در تمام راه های ممکن. diff --git a/docs/fa/development/index.md b/docs/fa/development/index.md deleted file mode 100644 index 398d497336a..00000000000 --- a/docs/fa/development/index.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u062A\u0648\u0633\u0639\u0647" -toc_hidden: true -toc_priority: 58 -toc_title: "\u0645\u062E\u0641\u06CC" ---- - -# توسعه کلیک {#clickhouse-development} - -[مقاله اصلی](https://clickhouse.tech/docs/en/development/) diff --git a/docs/fa/development/style.md b/docs/fa/development/style.md deleted file mode 100644 index c265a7426bd..00000000000 --- a/docs/fa/development/style.md +++ /dev/null @@ -1,842 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 68 -toc_title: "\u0686\u06AF\u0648\u0646\u0647 \u0628\u0631\u0627\u06CC \u0646\u0648\u0634\ - \u062A\u0646 \u062C++ \u06A9\u062F" ---- - -# چگونه برای نوشتن ج++ کد {#how-to-write-c-code} - -## توصیه های عمومی {#general-recommendations} - -**1.** در زیر توصیه, مورد نیاز نیست. - -**2.** اگر شما در حال ویرایش کد, این را حس می کند به دنبال قالب بندی کد های موجود. - -**3.** سبک کد برای سازگاری مورد نیاز است. سازگاری خواندن کد را ساده تر می کند و همچنین باعث می شود که کد را جستجو کنید. - -**4.** بسیاری از قوانین دلایل منطقی ندارند; دیکته شده توسط شیوه های تاسیس شده است. - -## قالببندی {#formatting} - -**1.** بسیاری از قالب بندی به صورت خودکار انجام می شود `clang-format`. - -**2.** فرورفتگی فضاهای 4 هستند. پیکربندی محیط توسعه خود را به طوری که یک تب اضافه می کند چهار فضا. - -**3.** باز و بسته شدن براکت فرفری باید در یک خط جداگانه باشد. - -``` cpp -inline void readBoolText(bool & x, ReadBuffer & buf) -{ - char tmp = '0'; - readChar(tmp, buf); - x = tmp != '0'; -} -``` - -**4.** اگر کل بدن تابع یک است `statement`, این را می توان در یک خط قرار داده شده. فضاهای محل در اطراف پرانتز فرفری (علاوه بر فضای در پایان خط). - -``` cpp -inline size_t mask() const { return buf_size() - 1; } -inline size_t place(HashValue x) const { return x & mask(); } -``` - -**5.** برای توابع. فضاهای اطراف براکت قرار ندهید. - -``` cpp -void reinsert(const Value & x) -``` - -``` cpp -memcpy(&buf[place_value], &x, sizeof(x)); -``` - -**6.** داخل `if`, `for`, `while` و عبارت دیگر, یک فضای در مقابل براکت باز قرار داده(به عنوان مخالف به عملکرد تماس). - -``` cpp -for (size_t i = 0; i < rows; i += storage.index_granularity) -``` - -**7.** اضافه کردن فضاهای اطراف اپراتورهای دودویی (`+`, `-`, `*`, `/`, `%`, …) and the ternary operator `?:`. - -``` cpp -UInt16 year = (s[0] - '0') * 1000 + (s[1] - '0') * 100 + (s[2] - '0') * 10 + (s[3] - '0'); -UInt8 month = (s[5] - '0') * 10 + (s[6] - '0'); -UInt8 day = (s[8] - '0') * 10 + (s[9] - '0'); -``` - -**8.** اگر یک خوراک خط وارد شده است, قرار دادن اپراتور در یک خط جدید و افزایش تورفتگی قبل از. - -``` cpp -if (elapsed_ns) - message << " (" - << rows_read_on_server * 1000000000 / elapsed_ns << " rows/s., " - << bytes_read_on_server * 1000.0 / elapsed_ns << " MB/s.) "; -``` - -**9.** شما می توانید فضاهای برای هم ترازی در یک خط استفاده, در صورت دلخواه. - -``` cpp -dst.ClickLogID = click.LogID; -dst.ClickEventID = click.EventID; -dst.ClickGoodEvent = click.GoodEvent; -``` - -**10.** از فضاهای اطراف اپراتورها استفاده نکنید `.`, `->`. - -در صورت لزوم اپراتور می تواند به خط بعدی پیچیده شود. در این مورد جبران در مقابل افزایش می یابد. - -**11.** از فضا برای جدا کردن اپراتورهای غیر ضروری استفاده نکنید (`--`, `++`, `*`, `&`, …) from the argument. - -**12.** بعد از ویرگول فاصله بگیر ولی نه قبل از اون همین قاعده برای یک نقطه و ویرگول در داخل یک `for` اصطلاح. - -**13.** از فضاها برای جدا کردن استفاده نکنید `[]` اپراتور - -**14.** در یک `template <...>` عبارت, استفاده از یک فضای بین `template` و `<`; بدون فاصله پس از `<` یا قبل از `>`. - -``` cpp -template -struct AggregatedStatElement -{} -``` - -**15.** در کلاس ها و سازه, نوشتن `public`, `private` و `protected` در همان سطح به عنوان `class/struct`, و تورفتگی بقیه کد. - -``` cpp -template -class MultiVersion -{ -public: - /// Version of object for usage. shared_ptr manage lifetime of version. - using Version = std::shared_ptr; - ... -} -``` - -**16.** اگر همان `namespace` برای کل فایل استفاده می شود, و هر چیز دیگری قابل توجهی وجود ندارد, افست در داخل لازم نیست `namespace`. - -**17.** اگر بلوک برای `if`, `for`, `while`, یا عبارت دیگر متشکل از یک `statement`, براکت فرفری اختیاری هستند. محل `statement` در یک خط جداگانه, در عوض. این قانون نیز برای تو در تو معتبر `if`, `for`, `while`, … - -اما اگر درونی `statement` شامل براکت های فرفری یا `else` بلوک خارجی باید در براکت های فرفری نوشته شود. - -``` cpp -/// Finish write. -for (auto & stream : streams) - stream.second->finalize(); -``` - -**18.** نباید در انتهای خطوط هیچ فضایی وجود داشته باشد. - -**19.** فایل های منبع هستند وزارت مخابرات 8 کد گذاری. - -**20.** شخصیت های غیر ASCII استفاده می شود string literals. - -``` cpp -<< ", " << (timer.elapsed() / chunks_stats.hits) << " μsec/hit."; -``` - -**21.** هنوز عبارات متعدد در یک خط ارسال نمی. - -**22.** بخش های گروهی کد در داخل توابع و با بیش از یک خط خالی جدا می شوند. - -**23.** توابع جداگانه, کلاس, و به همین ترتیب با یک یا دو خط خالی. - -**24.** `A const` (مربوط به ارزش) باید قبل از نام نوع نوشته شده است. - -``` cpp -//correct -const char * pos -const std::string & s -//incorrect -char const * pos -``` - -**25.** هنگام اعلام اشاره گر یا مرجع `*` و `&` نمادها باید با فاصله در هر دو طرف از هم جدا. - -``` cpp -//correct -const char * pos -//incorrect -const char* pos -const char *pos -``` - -**26.** هنگام استفاده از انواع قالب ها با نام مستعار `using` کلمه کلیدی (به جز در ساده ترین موارد). - -به عبارت دیگر پارامترهای قالب فقط در `using` و در کد تکرار نمی شود. - -`using` می توان به صورت محلی اعلام کرد, مانند داخل یک تابع. - -``` cpp -//correct -using FileStreams = std::map>; -FileStreams streams; -//incorrect -std::map> streams; -``` - -**27.** هنوز متغیرهای مختلفی از انواع مختلف در یک بیانیه اعلام نمی. - -``` cpp -//incorrect -int x, *y; -``` - -**28.** هنوز کست ج سبک استفاده نمی. - -``` cpp -//incorrect -std::cerr << (int)c <<; std::endl; -//correct -std::cerr << static_cast(c) << std::endl; -``` - -**29.** در کلاس ها و ساختار, اعضای گروه و توابع به طور جداگانه در داخل هر دامنه دید. - -**30.** برای کلاس های کوچک و structs آن است که لازم نیست برای جدا کردن روش بیانیه از اجرای. - -همان درست است برای روش های کوچک در هر کلاس و یا ساختار است. - -برای templated کلاس و structs نیست جداگانه روش اعلامیه از اجرای (زیرا در غیر این صورت آنها باید تعریف شده در ترجمه همان واحد). - -**31.** شما می توانید خطوط در بسته بندی 140 شخصیت, بجای 80. - -**32.** همیشه پیشوند اپراتورهای افزایش/کاهش استفاده کنید اگر پسوند مورد نیاز نمی باشد. - -``` cpp -for (Names::const_iterator it = column_names.begin(); it != column_names.end(); ++it) -``` - -## توضیحات {#comments} - -**1.** حتما برای اضافه کردن نظر برای تمام بخش های غیر بدیهی از کد. - -این بسیار مهم است. نوشتن نظر ممکن است به شما کمک کند متوجه شوید که کد لازم نیست یا اشتباه طراحی شده است. - -``` cpp -/** Part of piece of memory, that can be used. - * For example, if internal_buffer is 1MB, and there was only 10 bytes loaded to buffer from file for reading, - * then working_buffer will have size of only 10 bytes - * (working_buffer.end() will point to position right after those 10 bytes available for read). - */ -``` - -**2.** نظرات می تواند به عنوان دقیق که لازم است. - -**3.** محل نظرات قبل از کد توصیف می کنند. در موارد نادر, نظرات می تواند پس از کد است, در همان خط. - -``` cpp -/** Parses and executes the query. -*/ -void executeQuery( - ReadBuffer & istr, /// Where to read the query from (and data for INSERT, if applicable) - WriteBuffer & ostr, /// Where to write the result - Context & context, /// DB, tables, data types, engines, functions, aggregate functions... - BlockInputStreamPtr & query_plan, /// Here could be written the description on how query was executed - QueryProcessingStage::Enum stage = QueryProcessingStage::Complete /// Up to which stage process the SELECT query - ) -``` - -**4.** نظرات فقط باید به زبان انگلیسی نوشته شود. - -**5.** اگر شما در حال نوشتن یک کتابخانه, شامل نظرات دقیق توضیح در فایل هدر اصلی. - -**6.** هنوز نظر که اطلاعات اضافی را فراهم نمی کند اضافه کنید. به خصوص, نظرات خالی مثل این را ترک کنید: - -``` cpp -/* -* Procedure Name: -* Original procedure name: -* Author: -* Date of creation: -* Dates of modification: -* Modification authors: -* Original file name: -* Purpose: -* Intent: -* Designation: -* Classes used: -* Constants: -* Local variables: -* Parameters: -* Date of creation: -* Purpose: -*/ -``` - -به عنوان مثال با اقتباس از منابع http://home.tamk.fi/~jaalto/دوره آموزشی/برنامه نویسی به سبک/doc/قابل نگهداشت-کد/. - -**7.** هنوز نظرات زباله ارسال کنید (نویسنده, تاریخ ایجاد ..) در ابتدای هر فایل. - -**8.** نظرات تک خط با سه اسلش شروع می شود: `///` و نظرات چند خط با شروع `/**`. این نظرات در نظر گرفته شده است “documentation”. - -توجه: شما می توانید داکسیژن برای تولید اسناد از این نظرات استفاده کنید. اما داکسیگن به طور کلی استفاده نمی شود زیرا راحت تر است که کد را در محیط برنامه نویسی حرکت دهید. - -**9.** نظرات چند خط باید خطوط خالی در ابتدا و پایان ندارد (به جز خط که بسته یک نظر چند خط). - -**10.** برای اظهار نظر از کد, استفاده از نظرات اساسی, نه “documenting” نظر. - -**11.** حذف بخش هایی از کد اظهار نظر قبل از ارتکاب. - -**12.** هنوز ناسزا در نظرات و یا کد استفاده کنید. - -**13.** از حروف بزرگ استفاده نکنید. هنوز نقطه گذاری بیش از حد استفاده کنید. - -``` cpp -/// WHAT THE FAIL??? -``` - -**14.** هنوز نظر را به محیطی استفاده نمی. - -``` cpp -///****************************************************** -``` - -**15.** هنوز بحث در نظرات شروع نشد. - -``` cpp -/// Why did you do this stuff? -``` - -**16.** بدون نیاز به نوشتن نظر در پایان یک بلوک توصیف چه بود وجود دارد. - -``` cpp -/// for -``` - -## نامها {#names} - -**1.** استفاده از حروف کوچک با رکورد در نام متغیرها و اعضای کلاس. - -``` cpp -size_t max_block_size; -``` - -**2.** نام توابع (روش) استفاده از camelCase آغاز با حروف کوچک نامه. - -``` cpp -std::string getName() const override { return "Memory"; } -``` - -**3.** برای نام کلاس ها (structs) استفاده از CamelCase آغاز با حروف بزرگ نامه. پیشوند دیگر از من برای رابط استفاده نمی شود. - -``` cpp -class StorageMemory : public IStorage -``` - -**4.** `using` به همان شیوه به عنوان کلاس به نام, و یا با `_t` در پایان. - -**5.** نام استدلال نوع الگو: در موارد ساده, استفاده `T`; `T`, `U`; `T1`, `T2`. - -برای موارد پیچیده تر, هم پیروی از قوانین برای نام کلاس, و یا اضافه کردن پیشوند `T`. - -``` cpp -template -struct AggregatedStatElement -``` - -**6.** نام استدلال ثابت الگو: هم پیروی از قوانین برای نام متغیر, و یا استفاده `N` در موارد ساده. - -``` cpp -template -struct ExtractDomain -``` - -**7.** برای کلاس های انتزاعی (رابط) شما می توانید اضافه کنید `I` پیشوند. - -``` cpp -class IBlockInputStream -``` - -**8.** اگر شما استفاده از یک متغیر به صورت محلی, شما می توانید نام کوتاه استفاده. - -در تمام موارد دیگر, استفاده از یک نام است که معنای توصیف. - -``` cpp -bool info_successfully_loaded = false; -``` - -**9.** اسامی `define`بازدید کنندگان و ثابت جهانی استفاده از همه_کاپ با زیرخط. - -``` cpp -#define MAX_SRC_TABLE_NAMES_TO_STORE 1000 -``` - -**10.** نام فایل باید همان سبک به عنوان مطالب خود استفاده کنید. - -اگر یک فایل شامل یک کلاس, نام فایل به همان شیوه به عنوان کلاس (بالش). - -اگر فایل شامل یک تابع واحد, نام فایل به همان شیوه به عنوان تابع (بالش). - -**11.** اگر نام شامل مخفف, سپس: - -- برای نام متغیر مخفف حروف کوچک استفاده کنید `mysql_connection` ) نه `mySQL_connection`). -- برای نام کلاس ها و توابع, نگه داشتن حروف بزرگ در مخفف`MySQLConnection` ) نه `MySqlConnection`). - -**12.** استدلال سازنده استفاده می شود که فقط به مقداردهی اولیه اعضای کلاس باید به همان شیوه به عنوان اعضای کلاس به نام, اما با تاکید در پایان. - -``` cpp -FileQueueProcessor( - const std::string & path_, - const std::string & prefix_, - std::shared_ptr handler_) - : path(path_), - prefix(prefix_), - handler(handler_), - log(&Logger::get("FileQueueProcessor")) -{ -} -``` - -پسوند تاکید می توان حذف اگر استدلال در بدن سازنده استفاده نمی شود. - -**13.** هیچ تفاوتی در نام متغیرهای محلی و اعضای کلاس وجود دارد (هیچ پیشوندهای مورد نیاز). - -``` cpp -timer (not m_timer) -``` - -**14.** برای ثابت در یک `enum` استفاده از CamelCase با حرف بزرگ. ت_کاپها نیز قابل قبول است. اگر `enum` غیر محلی است, استفاده از یک `enum class`. - -``` cpp -enum class CompressionMethod -{ - QuickLZ = 0, - LZ4 = 1, -}; -``` - -**15.** همه نامها باید به زبان انگلیسی باشد. ترجمه کلمات روسی مجاز نیست. - - not Stroka - -**16.** اختصارات قابل قبول هستند در صورتی که به خوبی شناخته شده است (زمانی که شما به راحتی می توانید معنای مخفف در ویکیپدیا و یا در یک موتور جستجو پیدا کنید). - - `AST`, `SQL`. - - Not `NVDH` (some random letters) - -کلمات ناقص قابل قبول است اگر نسخه کوتاه استفاده مشترک است. - -شما همچنین می توانید مخفف استفاده کنید اگر نام کامل در کنار در نظرات گنجانده شده است. - -**17.** نام فایل با ج++ کد منبع باید `.cpp` گسترش. فایل های هدر باید داشته باشند `.h` گسترش. - -## نحوه نوشتن کد {#how-to-write-code} - -**1.** مدیریت حافظه. - -تخصیص حافظه دستی (`delete`) تنها می تواند در کد کتابخانه استفاده می شود. - -در کد کتابخانه `delete` اپراتور تنها می تواند در مخرب استفاده می شود. - -در کد برنامه, حافظه باید توسط شی که صاحب رهایی. - -مثالها: - -- ساده ترین راه این است که یک شی را روی پشته قرار دهید یا عضو یک کلاس دیگر شوید. -- برای تعداد زیادی از اشیای کوچک از ظروف استفاده کنید. -- برای تخصیص خودکار تعداد کمی از اشیا که در پشته قرار دارند استفاده کنید `shared_ptr/unique_ptr`. - -**2.** مدیریت منابع. - -استفاده `RAII` و بالا را ببینید. - -**3.** رفع خطا. - -استفاده از استثنا. در بیشتر موارد, شما فقط نیاز به پرتاب یک استثنا, و لازم نیست برای گرفتن (به دلیل `RAII`). - -در برنامه های پردازش داده ها نیست, اغلب قابل قبول برای گرفتن استثنا نیست. - -در سرور هایی که رسیدگی به درخواست کاربر, این معمولا به اندازه کافی برای گرفتن استثنا در سطح بالای کنترل اتصال. - -در توابع موضوع, شما باید گرفتن و نگه داشتن همه استثنا به تجدید نظر در موضوع اصلی پس از `join`. - -``` cpp -/// If there weren't any calculations yet, calculate the first block synchronously -if (!started) -{ - calculate(); - started = true; -} -else /// If calculations are already in progress, wait for the result - pool.wait(); - -if (exception) - exception->rethrow(); -``` - -هرگز استثنا بدون دست زدن به پنهان. هرگز فقط کورکورانه قرار دادن همه استثنا برای ورود به سیستم. - -``` cpp -//Not correct -catch (...) {} -``` - -اگر شما نیاز به چشم پوشی از چند استثنا, انجام این کار تنها برای افراد خاص و تجدید نظر بقیه. - -``` cpp -catch (const DB::Exception & e) -{ - if (e.code() == ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION) - return nullptr; - else - throw; -} -``` - -هنگام استفاده از توابع با کدهای پاسخ یا `errno`, همیشه نتیجه را بررسی کنید و پرتاب یک استثنا در صورت خطا. - -``` cpp -if (0 != close(fd)) - throwFromErrno("Cannot close file " + file_name, ErrorCodes::CANNOT_CLOSE_FILE); -``` - -`Do not use assert`. - -**4.** انواع استثنا. - -بدون نیاز به استفاده از سلسله مراتب استثنا پیچیده در کد نرم افزار وجود دارد. متن استثنا باید قابل فهم برای یک مدیر سیستم. - -**5.** پرتاب استثنا از destructors. - -این توصیه نمی شود, اما مجاز است. - -از گزینه های زیر استفاده کنید: - -- ایجاد یک تابع (`done()` یا `finalize()`) که همه کار در پیش است که ممکن است منجر به یک استثنا انجام دهد. در صورتی که تابع نامیده می شد, باید بدون استثنا در مخرب بعد وجود داشته باشد. -- کارهایی که بیش از حد پیچیده هستند (مانند ارسال پیام بر روی شبکه) را می توان در روش جداگانه قرار داده است که کاربر کلاس باید قبل از تخریب تماس بگیرید. -- اگر یک استثنا در مخرب وجود دارد, بهتر است به سیستم وارد شوید از برای مخفی کردن (اگر چوب در دسترس است). -- در برنامه های ساده, قابل قبول است به تکیه بر `std::terminate` (برای موارد `noexcept` به طور پیش فرض در ج++11) برای رسیدگی به استثنا. - -**6.** بلوک کد ناشناس. - -شما می توانید یک بلوک کد جداگانه در داخل یک تابع واحد به منظور ایجاد متغیرهای خاص محلی ایجاد, به طوری که مخرب نامیده می شوند در هنگام خروج از بلوک. - -``` cpp -Block block = data.in->read(); - -{ - std::lock_guard lock(mutex); - data.ready = true; - data.block = block; -} - -ready_any.set(); -``` - -**7.** چند رشته. - -در برنامههای پردازش داده برونخط: - -- سعی کنید بهترین عملکرد ممکن را در یک هسته پردازنده تک دریافت کنید. سپس می توانید کد خود را در صورت لزوم موازی کنید. - -در برنامه های سرور: - -- استفاده از استخر موضوع برای پردازش درخواست. در این مرحله, ما هیچ وظایفی که مورد نیاز تعویض زمینه فضای کاربری نداشته اند. - -چنگال برای موازی سازی استفاده نمی شود. - -**8.** همگام سازی موضوعات. - -اغلب ممکن است موضوعات مختلف از سلول های حافظه مختلف (حتی بهتر: خطوط کش مختلف) استفاده کنند و از هماهنگ سازی موضوع (به جز `joinAll`). - -اگر هماهنگ سازی مورد نیاز است, در بیشتر موارد, کافی است به استفاده از امکانپذیر تحت `lock_guard`. - -در موارد دیگر استفاده از شکلهای هندسی اولیه هماهنگ سازی سیستم. هنوز انتظار مشغول استفاده کنید. - -عملیات اتمی باید تنها در ساده ترین موارد استفاده می شود. - -سعی نکنید ساختارهای داده ای بدون قفل را اجرا کنید مگر اینکه منطقه اصلی تخصص شما باشد. - -**9.** اشاره گر در مقابل مراجع. - -در بیشتر موارد, ترجیح می دهند مراجع. - -**10.** توایع. - -استفاده از منابع ثابت اشاره گر به ثابت, `const_iterator`, و روش توایع. - -در نظر بگیرید `const` به طور پیش فرض و استفاده غیر-`const` فقط در صورت لزوم. - -هنگام عبور متغیرها بر اساس ارزش, با استفاده از `const` معمولا معنی ندارد. - -**11.** امضا نشده. - -استفاده `unsigned` در صورت لزوم - -**12.** انواع عددی. - -استفاده از انواع `UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32` و `Int64`, و همچنین `size_t`, `ssize_t` و `ptrdiff_t`. - -از این نوع برای اعداد استفاده نکنید: `signed/unsigned long`, `long long`, `short`, `signed/unsigned char`, `char`. - -**13.** عبور استدلال. - -رمز عبور مقادیر پیچیده توسط مرجع (محتوی `std::string`). - -اگر یک تابع قطاری مالکیت یک شی ایجاد شده در پشته, را از نوع استدلال `shared_ptr` یا `unique_ptr`. - -**14.** ارزش بازگشت. - -در اکثر موارد فقط استفاده کنید `return`. ننویس `return std::move(res)`. - -اگر تابع یک شی در پشته اختصاص و بازده, استفاده `shared_ptr` یا `unique_ptr`. - -در موارد نادر شما ممکن است نیاز به بازگشت به ارزش از طریق بحث و جدل. در این مورد استدلال باید مرجع باشد. - -``` cpp -using AggregateFunctionPtr = std::shared_ptr; - -/** Allows creating an aggregate function by its name. - */ -class AggregateFunctionFactory -{ -public: - AggregateFunctionFactory(); - AggregateFunctionPtr get(const String & name, const DataTypes & argument_types) const; -``` - -**15.** فضای نام. - -بدون نیاز به استفاده از یک جداگانه وجود دارد `namespace` برای کد برنامه. - -کتابخانه های کوچک هم به این نیاز ندارند. - -برای کتابخانه های متوسط تا بزرگ, همه چیز را در یک `namespace`. - -در کتابخانه `.h` پرونده, شما می توانید استفاده کنید `namespace detail` برای مخفی کردن اطلاعات پیاده سازی برای کد برنامه مورد نیاز نیست. - -در یک `.cpp` پرونده, شما می توانید یک استفاده `static` یا فضای نام ناشناس برای مخفی کردن نمادها. - -همچنین یک `namespace` می تواند برای یک استفاده شود `enum` برای جلوگیری از نام های مربوطه را از افتادن به یک خارجی `namespace` (اما بهتر است از یک `enum class`). - -**16.** مقدار دهی اولیه معوق. - -اگر استدلال برای مقدار دهی اولیه مورد نیاز, سپس شما به طور معمول باید یک سازنده به طور پیش فرض ارسال کنید. - -اگر بعد شما نیاز به تاخیر دهی اولیه, شما می توانید یک سازنده به طور پیش فرض است که یک شی نامعتبر ایجاد اضافه. یا برای تعداد کمی از اشیا می توانید استفاده کنید `shared_ptr/unique_ptr`. - -``` cpp -Loader(DB::Connection * connection_, const std::string & query, size_t max_block_size_); - -/// For deferred initialization -Loader() {} -``` - -**17.** توابع مجازی. - -اگر کلاس برای استفاده چند شکل در نظر گرفته شده, شما لازم نیست که به توابع مجازی. این نیز به مخرب اعمال می شود. - -**18.** کدگذاریها. - -استفاده از اوتیف - 8 در همه جا. استفاده `std::string`و`char *`. استفاده نشود `std::wstring`و`wchar_t`. - -**19.** ثبت. - -نمونه در همه جا در کد را ببینید. - -قبل از ارتکاب, حذف همه بی معنی و اشکال زدایی ورود به سیستم, و هر نوع دیگری از خروجی اشکال زدایی. - -ورود به چرخه باید حتی در سطح ردیابی اجتناب شود. - -سیاهههای مربوط باید در هر سطح ورود به سیستم قابل خواندن باشد. - -ورود به سیستم تنها باید در کد نرم افزار مورد استفاده قرار, در بیشتر قسمت ها. - -ورود پیام باید به زبان انگلیسی نوشته شده است. - -ورود ترجیحا باید برای مدیر سیستم قابل فهم باشد. - -هنوز ناسزا در ورود به سیستم استفاده کنید. - -استفاده از جی تی اف 8 را پشتیبانی می کند در ورود به سیستم. در موارد نادر شما می توانید شخصیت های غیر اسکی در ورود به سیستم استفاده کنید. - -**20.** ورودی-خروجی. - -استفاده نکنید `iostreams` در چرخه های داخلی که برای عملکرد برنامه حیاتی هستند (و هرگز استفاده نکنید `stringstream`). - -استفاده از `DB/IO` کتابخانه به جای. - -**21.** تاریخ و زمان. - -دیدن `DateLUT` کتابخونه. - -**22.** شامل شدن. - -همیشه استفاده کنید `#pragma once` به جای شامل نگهبانان. - -**23.** با استفاده از. - -`using namespace` استفاده نمی شود. شما می توانید استفاده کنید `using` با چیزی خاص. اما محلی در داخل یک کلاس و یا تابع را. - -**24.** استفاده نشود `trailing return type` برای توابع مگر اینکه لازم باشد. - -``` cpp -auto f() -> void -``` - -**25.** اعلامیه و مقدار دهی اولیه از متغیرهای. - -``` cpp -//right way -std::string s = "Hello"; -std::string s{"Hello"}; - -//wrong way -auto s = std::string{"Hello"}; -``` - -**26.** برای توابع مجازی, نوشتن `virtual` در کلاس پایه, اما ارسال `override` به جای `virtual` در کلاس های نسل نو. - -## ویژگی های استفاده نشده از سی++ {#unused-features-of-c} - -**1.** ارث مجازی استفاده نمی شود. - -**2.** ویژگی استثنا از ج++03 استفاده نمی شود. - -## سکو {#platform} - -**1.** ما نوشتن کد برای یک پلت فرم خاص. - -اما چیزهای دیگر برابر بودن, کراس پلت فرم و یا کد قابل حمل ترجیح داده می شود. - -**2.** زبان: ج++20. - -**3.** کامپایلر: `gcc`. در این زمان (اوت 2020), کد با استفاده از نسخه وارد شده 9.3. (همچنین می تواند با استفاده از وارد شود `clang 8`.) - -کتابخانه استاندارد استفاده شده است (`libc++`). - -**4.**سیستم عامل: لینوکس اوبونتو, مسن تر از دقیق نیست. - -**5.**کد برای معماری پردازنده ایکس86_64 نوشته شده است. - -مجموعه دستورالعمل پردازنده حداقل مجموعه پشتیبانی در میان سرورهای ما است. در حال حاضر, این سوس است 4.2. - -**6.** استفاده `-Wall -Wextra -Werror` پرچم تلفیقی. - -**7.** استفاده از لینک کردن استاتیک با تمام کتابخانه ها به جز کسانی که به سختی برای اتصال به استاتیک (خروجی را ببینید `ldd` فرمان). - -**8.** کد توسعه یافته است و با تنظیمات انتشار دیباگ. - -## ابزارها {#tools} - -**1.** KDevelop خوب است IDE. - -**2.** برای اشکالزدایی, استفاده `gdb`, `valgrind` (`memcheck`), `strace`, `-fsanitize=...` یا `tcmalloc_minimal_debug`. - -**3.** برای پروفایل استفاده کنید `Linux Perf`, `valgrind` (`callgrind`), یا `strace -cf`. - -**4.** منابع در دستگاه گوارش هستند. - -**5.** استفاده مجمع `CMake`. - -**6.** برنامه ها با استفاده از منتشر `deb` بسته. - -**7.** مرتکب به استاد باید ساخت شکستن نیست. - -هر چند تجدید نظر تنها انتخاب شده قابل اجرا در نظر گرفته. - -**8.** مرتکب به عنوان اغلب به عنوان امکان پذیر است, حتی اگر کد تنها تا حدی اماده. - -استفاده از شاخه برای این منظور. - -اگر کد شما در `master` شاخه هنوز قابل ساختن نیست و از قبل از ساخت حذف می شود `push`. باید تمومش کنی یا ظرف چند روز حذفش کنی - -**9.** برای تغییرات غیر بدیهی از شاخه ها استفاده کنید و بر روی سرور منتشر کنید. - -**10.** کد استفاده نشده است از مخزن حذف شده است. - -## کتابخانهها {#libraries} - -**1.** ج++20 کتابخانه استاندارد استفاده شده است (پسوند تجربی مجاز), و همچنین `boost` و `Poco` چارچوب. - -**2.** در صورت لزوم, شما می توانید هر کتابخانه شناخته شده موجود در بسته سیستم عامل استفاده. - -اگر یک راه حل خوب در حال حاضر در دسترس وجود دارد, سپس استفاده کنید, حتی اگر به این معنی شما باید برای نصب کتابخانه دیگر. - -(اما برای حذف کتابخانه های بد از کد تهیه می شود.) - -**3.** شما می توانید یک کتابخانه است که در بسته نیست نصب, اگر بسته لازم نیست که چه شما نیاز دارید و یا یک نسخه منسوخ شده و یا نوع اشتباه از تلفیقی. - -**4.** اگر کتابخانه کوچک است و سیستم ساخت پیچیده خود را ندارد, قرار دادن فایل های منبع در `contrib` پوشه - -**5.** اولویت همیشه به کتابخانه هایی که در حال حاضر در حال استفاده هستند داده می شود. - -## توصیه های عمومی {#general-recommendations-1} - -**1.** ارسال کد به عنوان کوچک که ممکن است. - -**2.** ساده ترین راه حل را امتحان کنید. - -**3.** کد را بنویسید تا بدانید چگونه کار می کند و چگونه حلقه داخلی عمل می کند. - -**4.** در ساده ترین موارد استفاده کنید `using` به جای کلاس و یا ساختار. - -**5.** در صورت امکان, انجام سازنده کپی ارسال کنید, اپراتورهای انتساب, مخرب (به غیر از یک مجازی, اگر کلاس شامل حداقل یک تابع مجازی), حرکت سازنده و یا اپراتورهای انتساب حرکت. به عبارت دیگر, توابع کامپایلر تولید باید به درستی کار. شما می توانید استفاده کنید `default`. - -**6.** ساده سازی کد تشویق می شود. کاهش اندازه کد خود را در صورت امکان. - -## توصیه های اضافی {#additional-recommendations} - -**1.** به صراحت مشخص `std::` برای انواع از `stddef.h` - -توصیه نمی شود. به عبارت دیگر توصیه می کنیم نوشتن کنید `size_t` در عوض `std::size_t` چون کوتاهتر است . - -این قابل قبول است برای اضافه کردن `std::`. - -**2.** به صراحت مشخص `std::` برای توابع از کتابخانه استاندارد ج - -توصیه نمی شود. به عبارت دیگر, نوشتن `memcpy` به جای `std::memcpy`. - -دلیل این است که توابع غیر استاندارد مشابه وجود دارد, مانند `memmem`. ما با استفاده از این توابع در مناسبت. این توابع در وجود ندارد `namespace std`. - -اگر شما ارسال `std::memcpy` به جای `memcpy` پس همه جا `memmem` بدون `std::` نگاه عجیب و غریب. - -با این اوصاف, شما هنوز هم می توانید استفاده کنید `std::` اگر شما ترجیح می دهند. - -**3.** با استفاده از توابع از ج زمانی که همان در استاندارد ج++ کتابخانه در دسترس هستند. - -این قابل قبول است اگر کارایی بیشتری داشته باشد. - -برای مثال استفاده کنید `memcpy` به جای `std::copy` برای کپی کردن تکه های زیادی از حافظه است. - -**4.** استدلال تابع چند خطی. - -هر یک از سبک های بسته بندی زیر مجاز است: - -``` cpp -function( - T1 x1, - T2 x2) -``` - -``` cpp -function( - size_t left, size_t right, - const & RangesInDataParts ranges, - size_t limit) -``` - -``` cpp -function(size_t left, size_t right, - const & RangesInDataParts ranges, - size_t limit) -``` - -``` cpp -function(size_t left, size_t right, - const & RangesInDataParts ranges, - size_t limit) -``` - -``` cpp -function( - size_t left, - size_t right, - const & RangesInDataParts ranges, - size_t limit) -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/development/style/) diff --git a/docs/fa/development/tests.md b/docs/fa/development/tests.md deleted file mode 120000 index c03d36c3916..00000000000 --- a/docs/fa/development/tests.md +++ /dev/null @@ -1 +0,0 @@ -../../en/development/tests.md \ No newline at end of file diff --git a/docs/fa/engines/database-engines/index.md b/docs/fa/engines/database-engines/index.md deleted file mode 100644 index 40c7b75c83f..00000000000 --- a/docs/fa/engines/database-engines/index.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0645\u0648\u062A\u0648\u0631\u0647\u0627\u06CC \u067E\u0627\u06CC\ - \u06AF\u0627\u0647 \u062F\u0627\u062F\u0647" -toc_priority: 27 -toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" ---- - -# موتورهای پایگاه داده {#database-engines} - -موتورهای پایگاه داده به شما اجازه کار با جداول. - -به طور پیش فرض, تاتر با استفاده از موتور پایگاه داده مادری خود, فراهم می کند که تنظیم [موتورهای جدول](../../engines/table-engines/index.md) و یک [شمارهگیری](../../sql-reference/syntax.md). - -شما همچنین می توانید موتورهای پایگاه داده زیر استفاده کنید: - -- [MySQL](mysql.md) - -- [تنبل](lazy.md) - -[مقاله اصلی](https://clickhouse.tech/docs/en/database_engines/) diff --git a/docs/fa/engines/database-engines/lazy.md b/docs/fa/engines/database-engines/lazy.md deleted file mode 100644 index addac167334..00000000000 --- a/docs/fa/engines/database-engines/lazy.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 31 -toc_title: "\u062A\u0646\u0628\u0644" ---- - -# تنبل {#lazy} - -نگه می دارد جداول در رم تنها `expiration_time_in_seconds` ثانیه پس از دسترسی گذشته. را می توان تنها با استفاده \*جداول ورود به سیستم. - -این برای ذخیره سازی بسیاری از جداول کوچک \*ورود به سیستم بهینه شده است که فاصله زمانی طولانی بین دسترسی ها وجود دارد. - -## ایجاد یک پایگاه داده {#creating-a-database} - - CREATE DATABASE testlazy ENGINE = Lazy(expiration_time_in_seconds); - -[مقاله اصلی](https://clickhouse.tech/docs/en/database_engines/lazy/) diff --git a/docs/fa/engines/database-engines/mysql.md b/docs/fa/engines/database-engines/mysql.md deleted file mode 100644 index b8a9a4d1d2b..00000000000 --- a/docs/fa/engines/database-engines/mysql.md +++ /dev/null @@ -1,135 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 30 -toc_title: MySQL ---- - -# MySQL {#mysql} - -اجازه می دهد تا برای اتصال به پایگاه داده بر روی یک سرور خروجی از راه دور و انجام `INSERT` و `SELECT` نمایش داده شد به تبادل اطلاعات بین کلیک و خروجی زیر. - -این `MySQL` موتور پایگاه داده ترجمه نمایش داده شد به سرور خروجی زیر بنابراین شما می توانید عملیات مانند انجام `SHOW TABLES` یا `SHOW CREATE TABLE`. - -شما می توانید نمایش داده شد زیر را انجام دهد: - -- `RENAME` -- `CREATE TABLE` -- `ALTER` - -## ایجاد یک پایگاه داده {#creating-a-database} - -``` sql -CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] -ENGINE = MySQL('host:port', ['database' | database], 'user', 'password') -``` - -**پارامترهای موتور** - -- `host:port` — MySQL server address. -- `database` — Remote database name. -- `user` — MySQL user. -- `password` — User password. - -## پشتیبانی از انواع داده ها {#data_types-support} - -| MySQL | فاحشه خانه | -|----------------------------------|------------------------------------------------------------| -| UNSIGNED TINYINT | [UInt8](../../sql-reference/data-types/int-uint.md) | -| TINYINT | [Int8](../../sql-reference/data-types/int-uint.md) | -| UNSIGNED SMALLINT | [UInt16](../../sql-reference/data-types/int-uint.md) | -| SMALLINT | [Int16](../../sql-reference/data-types/int-uint.md) | -| UNSIGNED INT, UNSIGNED MEDIUMINT | [UInt32](../../sql-reference/data-types/int-uint.md) | -| INT, MEDIUMINT | [Int32](../../sql-reference/data-types/int-uint.md) | -| UNSIGNED BIGINT | [UInt64](../../sql-reference/data-types/int-uint.md) | -| BIGINT | [Int64](../../sql-reference/data-types/int-uint.md) | -| FLOAT | [Float32](../../sql-reference/data-types/float.md) | -| DOUBLE | [جسم شناور64](../../sql-reference/data-types/float.md) | -| DATE | [تاریخ](../../sql-reference/data-types/date.md) | -| DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) | -| BINARY | [رشته ثابت](../../sql-reference/data-types/fixedstring.md) | - -همه انواع داده خروجی زیر دیگر به تبدیل [رشته](../../sql-reference/data-types/string.md). - -[Nullable](../../sql-reference/data-types/nullable.md) پشتیبانی می شود. - -## نمونه هایی از استفاده {#examples-of-use} - -جدول در خروجی زیر: - -``` text -mysql> USE test; -Database changed - -mysql> CREATE TABLE `mysql_table` ( - -> `int_id` INT NOT NULL AUTO_INCREMENT, - -> `float` FLOAT NOT NULL, - -> PRIMARY KEY (`int_id`)); -Query OK, 0 rows affected (0,09 sec) - -mysql> insert into mysql_table (`int_id`, `float`) VALUES (1,2); -Query OK, 1 row affected (0,00 sec) - -mysql> select * from mysql_table; -+------+-----+ -| int_id | value | -+------+-----+ -| 1 | 2 | -+------+-----+ -1 row in set (0,00 sec) -``` - -پایگاه داده در خانه, تبادل داده ها با سرور خروجی زیر: - -``` sql -CREATE DATABASE mysql_db ENGINE = MySQL('localhost:3306', 'test', 'my_user', 'user_password') -``` - -``` sql -SHOW DATABASES -``` - -``` text -┌─name─────┐ -│ default │ -│ mysql_db │ -│ system │ -└──────────┘ -``` - -``` sql -SHOW TABLES FROM mysql_db -``` - -``` text -┌─name─────────┐ -│ mysql_table │ -└──────────────┘ -``` - -``` sql -SELECT * FROM mysql_db.mysql_table -``` - -``` text -┌─int_id─┬─value─┐ -│ 1 │ 2 │ -└────────┴───────┘ -``` - -``` sql -INSERT INTO mysql_db.mysql_table VALUES (3,4) -``` - -``` sql -SELECT * FROM mysql_db.mysql_table -``` - -``` text -┌─int_id─┬─value─┐ -│ 1 │ 2 │ -│ 3 │ 4 │ -└────────┴───────┘ -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/database_engines/mysql/) diff --git a/docs/fa/engines/index.md b/docs/fa/engines/index.md deleted file mode 100644 index cd705cc3df5..00000000000 --- a/docs/fa/engines/index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0645\u0648\u062A\u0648\u0631\u0647\u0627" -toc_priority: 25 ---- - - diff --git a/docs/fa/engines/table-engines/index.md b/docs/fa/engines/table-engines/index.md deleted file mode 100644 index a78f9e1d453..00000000000 --- a/docs/fa/engines/table-engines/index.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0645\u0648\u062A\u0648\u0631\u0647\u0627\u06CC \u062C\u062F\u0648\ - \u0644" -toc_priority: 26 -toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" ---- - -# موتورهای جدول {#table_engines} - -موتور جدول (نوع جدول) تعیین می کند: - -- چگونه و در کجا اطلاعات ذخیره شده است, جایی که برای نوشتن به, و از کجا به خواندن از. -- که نمایش داده شد پشتیبانی می شوند, و چگونه. -- همزمان دسترسی به داده ها. -- استفاده از شاخص, در صورت وجود. -- این که اجرای درخواست چند رشته ای امکان پذیر باشد. -- پارامترهای تکرار داده. - -## خانواده موتور {#engine-families} - -### ادغام {#mergetree} - -موتورهای جدول جهانی ترین و کاربردی برای وظایف بار بالا. اموال به اشتراک گذاشته شده توسط این موتور درج داده های سریع با پردازش داده های پس زمینه های بعدی است. `MergeTree` موتورهای خانواده از تکرار داده ها پشتیبانی می کنند (با [تکرار\*](mergetree-family/replication.md#table_engines-replication) نسخه موتورهای) پارتیشن بندی و ویژگی های دیگر در موتورهای دیگر پشتیبانی نمی شود. - -موتورهای در خانواده: - -- [ادغام](mergetree-family/mergetree.md#mergetree) -- [جایگزینی](mergetree-family/replacingmergetree.md#replacingmergetree) -- [سامینگمرگتری](mergetree-family/summingmergetree.md#summingmergetree) -- [ریزدانه](mergetree-family/aggregatingmergetree.md#aggregatingmergetree) -- [سقوط غذای اصلی](mergetree-family/collapsingmergetree.md#table_engine-collapsingmergetree) -- [در حال بارگذاری](mergetree-family/versionedcollapsingmergetree.md#versionedcollapsingmergetree) -- [نمودار](mergetree-family/graphitemergetree.md#graphitemergetree) - -### ثبت {#log} - -سبک [موتورها](log-family/index.md) با حداقل قابلیت. هنگامی که شما نیاز به سرعت نوشتن بسیاری از جداول کوچک (تا حدود 1 میلیون ردیف) و خواندن بعد به عنوان یک کل موثر ترین هستند. - -موتورهای در خانواده: - -- [جمع شدن](log-family/tinylog.md#tinylog) -- [خط زدن](log-family/stripelog.md#stripelog) -- [ثبت](log-family/log.md#log) - -### موتورهای یکپارچه سازی {#integration-engines} - -موتورهای برای برقراری ارتباط با دیگر ذخیره سازی داده ها و سیستم های پردازش. - -موتورهای در خانواده: - -- [کافکا](integrations/kafka.md#kafka) -- [MySQL](integrations/mysql.md#mysql) -- [ODBC](integrations/odbc.md#table-engine-odbc) -- [JDBC](integrations/jdbc.md#table-engine-jdbc) -- [HDFS](integrations/hdfs.md#hdfs) - -### موتورهای ویژه {#special-engines} - -موتورهای در خانواده: - -- [توزیع شده](special/distributed.md#distributed) -- [ماده بینی](special/materializedview.md#materializedview) -- [واژهنامه](special/dictionary.md#dictionary) -- پردازشگر پشتیبانی شده: -- [پرونده](special/file.md#file) -- [خالی](special/null.md#null) -- [تنظیم](special/set.md#set) -- [پیوستن](special/join.md#join) -- [URL](special/url.md#table_engines-url) -- [نما](special/view.md#table_engines-view) -- [حافظه](special/memory.md#memory) -- [بافر](special/buffer.md#buffer) - -## ستونهای مجازی {#table_engines-virtual_columns} - -ستون مجازی یک ویژگی موتور جدول انتگرال است که در کد منبع موتور تعریف شده است. - -شما باید ستون مجازی در مشخص نیست `CREATE TABLE` پرس و جو کنید و نمی توانید ببینید `SHOW CREATE TABLE` و `DESCRIBE TABLE` نتایج پرس و جو. ستون مجازی نیز فقط خواندنی, بنابراین شما می توانید داده ها را به ستون مجازی وارد کنید. - -برای انتخاب داده ها از یک ستون مجازی, شما باید نام خود را در مشخص `SELECT` پرس و جو. `SELECT *` مقادیر از ستون های مجازی بازگشت نیست. - -اگر شما یک جدول با یک ستون است که به همین نام به عنوان یکی از ستون های مجازی جدول ایجاد, ستون مجازی غیر قابل دسترس می شود. ما توصیه نمی انجام این کار. برای کمک به جلوگیری از درگیری, نام ستون مجازی معمولا با تاکید پیشوند. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/) diff --git a/docs/fa/engines/table-engines/integrations/hdfs.md b/docs/fa/engines/table-engines/integrations/hdfs.md deleted file mode 100644 index 36837362e07..00000000000 --- a/docs/fa/engines/table-engines/integrations/hdfs.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 36 -toc_title: HDFS ---- - -# HDFS {#table_engines-hdfs} - -این موتور ادغام با فراهم می کند [Apache Hadoop](https://en.wikipedia.org/wiki/Apache_Hadoop) اکوسیستم با اجازه دادن به مدیریت داده ها در [HDFS](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html)از طریق کلیکهاوس. این موتور مشابه است -به [پرونده](../special/file.md#table_engines-file) و [URL](../special/url.md#table_engines-url) موتورهای, اما فراهم می کند ویژگی های هادوپ خاص. - -## استفاده {#usage} - -``` sql -ENGINE = HDFS(URI, format) -``` - -این `URI` پارامتر تمام فایل نشانی اینترنتی در اچ دی است. -این `format` پارامتر یکی از فرمت های فایل های موجود را مشخص می کند. برای انجام -`SELECT` نمایش داده شد, فرمت باید برای ورودی پشتیبانی می شود, و به انجام -`INSERT` queries – for output. The available formats are listed in the -[فرشها](../../../interfaces/formats.md#formats) بخش. -قسمت مسیر `URI` ممکن است حاوی دل تنگی. در این مورد جدول قابل خواندن خواهد بود. - -**مثال:** - -**1.** تنظیم `hdfs_engine_table` جدول: - -``` sql -CREATE TABLE hdfs_engine_table (name String, value UInt32) ENGINE=HDFS('hdfs://hdfs1:9000/other_storage', 'TSV') -``` - -**2.** پر کردن پرونده: - -``` sql -INSERT INTO hdfs_engine_table VALUES ('one', 1), ('two', 2), ('three', 3) -``` - -**3.** پرسوجوی داده: - -``` sql -SELECT * FROM hdfs_engine_table LIMIT 2 -``` - -``` text -┌─name─┬─value─┐ -│ one │ 1 │ -│ two │ 2 │ -└──────┴───────┘ -``` - -## پیاده سازی اطلاعات {#implementation-details} - -- می خواند و می نویسد می تواند موازی -- پشتیبانی نمیشود: - - `ALTER` و `SELECT...SAMPLE` عملیات. - - شاخص. - - تکرار. - -**دل تنگی در مسیر** - -اجزای مسیر چندگانه می تواند دل تنگی دارند. برای پردازش فایل باید وجود داشته باشد و مسابقات به الگوی کل مسیر. لیست فایل های تعیین در طول `SELECT` (نه در `CREATE` لحظه). - -- `*` — Substitutes any number of any characters except `/` از جمله رشته خالی. -- `?` — Substitutes any single character. -- `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. -- `{N..M}` — Substitutes any number in range from N to M including both borders. - -سازه با `{}` شبیه به [دور](../../../sql-reference/table-functions/remote.md) تابع جدول. - -**مثال** - -1. فرض کنید ما چندین فایل را در قالب فیلم با اوریس زیر در اچ دی ها داریم: - -- ‘hdfs://hdfs1:9000/some_dir/some_file_1’ -- ‘hdfs://hdfs1:9000/some_dir/some_file_2’ -- ‘hdfs://hdfs1:9000/some_dir/some_file_3’ -- ‘hdfs://hdfs1:9000/another_dir/some_file_1’ -- ‘hdfs://hdfs1:9000/another_dir/some_file_2’ -- ‘hdfs://hdfs1:9000/another_dir/some_file_3’ - -1. راه های مختلفی برای ایجاد یک جدول متشکل از تمام شش فایل وجود دارد: - - - -``` sql -CREATE TABLE table_with_range (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/{some,another}_dir/some_file_{1..3}', 'TSV') -``` - -راه دیگر: - -``` sql -CREATE TABLE table_with_question_mark (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/{some,another}_dir/some_file_?', 'TSV') -``` - -جدول شامل تمام فایل ها در هر دو دایرکتوری (تمام فایل ها باید فرمت و طرح توصیف شده در پرس و جو راضی): - -``` sql -CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/{some,another}_dir/*', 'TSV') -``` - -!!! warning "اخطار" - اگر فهرستی از فایل های حاوی محدوده تعداد با صفر پیشرو, استفاده از ساخت و ساز با پرانتز برای هر رقم به طور جداگانه و یا استفاده `?`. - -**مثال** - -ایجاد جدول با فایل های به نام `file000`, `file001`, … , `file999`: - -``` sql -CREARE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/big_dir/file{0..9}{0..9}{0..9}', 'CSV') -``` - -## ستونهای مجازی {#virtual-columns} - -- `_path` — Path to the file. -- `_file` — Name of the file. - -**همچنین نگاه کنید به** - -- [ستونهای مجازی](../index.md#table_engines-virtual_columns) - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/hdfs/) diff --git a/docs/fa/engines/table-engines/integrations/index.md b/docs/fa/engines/table-engines/integrations/index.md deleted file mode 100644 index 4c6a285ff72..00000000000 --- a/docs/fa/engines/table-engines/integrations/index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u06CC\u06A9\u067E\u0627\u0631\u0686\u06AF\u06CC" -toc_priority: 30 ---- - - diff --git a/docs/fa/engines/table-engines/integrations/jdbc.md b/docs/fa/engines/table-engines/integrations/jdbc.md deleted file mode 100644 index 2a49d649ec0..00000000000 --- a/docs/fa/engines/table-engines/integrations/jdbc.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 34 -toc_title: JDBC ---- - -# JDBC {#table-engine-jdbc} - -اجازه می دهد تا تاتر برای اتصال به پایگاه داده های خارجی از طریق [JDBC](https://en.wikipedia.org/wiki/Java_Database_Connectivity). - -برای پیاده سازی اتصال جدی بی سی, خانه با استفاده از برنامه جداگانه [هومز-جد بی سی-پل](https://github.com/alex-krash/clickhouse-jdbc-bridge) که باید به عنوان یک شبح اجرا شود. - -این موتور از [Nullable](../../../sql-reference/data-types/nullable.md) نوع داده. - -## ایجاد یک جدول {#creating-a-table} - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name -( - columns list... -) -ENGINE = JDBC(dbms_uri, external_database, external_table) -``` - -**پارامترهای موتور** - -- `dbms_uri` — URI of an external DBMS. - - قالب: `jdbc:://:/?user=&password=`. - به عنوان مثال برای خروجی زیر: `jdbc:mysql://localhost:3306/?user=root&password=root`. - -- `external_database` — Database in an external DBMS. - -- `external_table` — Name of the table in `external_database`. - -## مثال طریقه استفاده {#usage-example} - -ایجاد یک جدول در سرور خروجی زیر با اتصال مستقیم با مشتری کنسول: - -``` text -mysql> CREATE TABLE `test`.`test` ( - -> `int_id` INT NOT NULL AUTO_INCREMENT, - -> `int_nullable` INT NULL DEFAULT NULL, - -> `float` FLOAT NOT NULL, - -> `float_nullable` FLOAT NULL DEFAULT NULL, - -> PRIMARY KEY (`int_id`)); -Query OK, 0 rows affected (0,09 sec) - -mysql> insert into test (`int_id`, `float`) VALUES (1,2); -Query OK, 1 row affected (0,00 sec) - -mysql> select * from test; -+------+----------+-----+----------+ -| int_id | int_nullable | float | float_nullable | -+------+----------+-----+----------+ -| 1 | NULL | 2 | NULL | -+------+----------+-----+----------+ -1 row in set (0,00 sec) -``` - -ایجاد یک جدول در سرور کلیک و انتخاب داده ها از: - -``` sql -CREATE TABLE jdbc_table -( - `int_id` Int32, - `int_nullable` Nullable(Int32), - `float` Float32, - `float_nullable` Nullable(Float32) -) -ENGINE JDBC('jdbc:mysql://localhost:3306/?user=root&password=root', 'test', 'test') -``` - -``` sql -SELECT * -FROM jdbc_table -``` - -``` text -┌─int_id─┬─int_nullable─┬─float─┬─float_nullable─┐ -│ 1 │ ᴺᵁᴸᴸ │ 2 │ ᴺᵁᴸᴸ │ -└────────┴──────────────┴───────┴────────────────┘ -``` - -## همچنین نگاه کنید به {#see-also} - -- [تابع جدول جدی بی سی](../../../sql-reference/table-functions/jdbc.md). - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/jdbc/) diff --git a/docs/fa/engines/table-engines/integrations/kafka.md b/docs/fa/engines/table-engines/integrations/kafka.md deleted file mode 100644 index bdadd8b8381..00000000000 --- a/docs/fa/engines/table-engines/integrations/kafka.md +++ /dev/null @@ -1,180 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 32 -toc_title: "\u06A9\u0627\u0641\u06A9\u0627" ---- - -# کافکا {#kafka} - -این موتور با این نسخهها کار [نمایی کافکا](http://kafka.apache.org/). - -کافکا به شما امکان می دهد: - -- انتشار یا اشتراک در جریان داده ها. -- سازماندهی ذخیره سازی مقاوم در برابر خطا. -- روند جریان به عنوان در دسترس تبدیل شده است. - -## ایجاد یک جدول {#table_engine-kafka-creating-a-table} - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... -) ENGINE = Kafka() -SETTINGS - kafka_broker_list = 'host:port', - kafka_topic_list = 'topic1,topic2,...', - kafka_group_name = 'group_name', - kafka_format = 'data_format'[,] - [kafka_row_delimiter = 'delimiter_symbol',] - [kafka_schema = '',] - [kafka_num_consumers = N,] - [kafka_max_block_size = 0,] - [kafka_skip_broken_messages = N,] - [kafka_commit_every_batch = 0] -``` - -پارامترهای مورد نیاز: - -- `kafka_broker_list` – A comma-separated list of brokers (for example, `localhost:9092`). -- `kafka_topic_list` – A list of Kafka topics. -- `kafka_group_name` – A group of Kafka consumers. Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere. -- `kafka_format` – Message format. Uses the same notation as the SQL `FORMAT` تابع مانند `JSONEachRow`. برای کسب اطلاعات بیشتر, دیدن [فرشها](../../../interfaces/formats.md) بخش. - -پارامترهای اختیاری: - -- `kafka_row_delimiter` – Delimiter character, which ends the message. -- `kafka_schema` – Parameter that must be used if the format requires a schema definition. For example, [سروان نیا](https://capnproto.org/) نیاز به مسیر به فایل طرح و نام ریشه `schema.capnp:Message` اعتراض. -- `kafka_num_consumers` – The number of consumers per table. Default: `1`. مشخص مصرف کنندگان بیشتر اگر توان عملیاتی یک مصرف کننده کافی است. تعداد کل مصرف کنندگان باید تعداد پارتیشن در موضوع تجاوز نمی, از تنها یک مصرف کننده را می توان در هر پارتیشن اختصاص داده. -- `kafka_max_block_size` - حداکثر اندازه دسته ای (در پیام) برای نظرسنجی (پیش فرض: `max_block_size`). -- `kafka_skip_broken_messages` – Kafka message parser tolerance to schema-incompatible messages per block. Default: `0`. اگر `kafka_skip_broken_messages = N` سپس موتور پرش *N* پیام کافکا که نمی تواند تجزیه شود (یک پیام برابر یک ردیف از داده ها). -- `kafka_commit_every_batch` - متعهد هر دسته مصرف و به کار گرفته به جای یک مرتکب پس از نوشتن یک بلوک کامل (به طور پیش فرض: `0`). - -مثالها: - -``` sql - CREATE TABLE queue ( - timestamp UInt64, - level String, - message String - ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); - - SELECT * FROM queue LIMIT 5; - - CREATE TABLE queue2 ( - timestamp UInt64, - level String, - message String - ) ENGINE = Kafka SETTINGS kafka_broker_list = 'localhost:9092', - kafka_topic_list = 'topic', - kafka_group_name = 'group1', - kafka_format = 'JSONEachRow', - kafka_num_consumers = 4; - - CREATE TABLE queue2 ( - timestamp UInt64, - level String, - message String - ) ENGINE = Kafka('localhost:9092', 'topic', 'group1') - SETTINGS kafka_format = 'JSONEachRow', - kafka_num_consumers = 4; -``` - -
- -روش منسوخ برای ایجاد یک جدول - -!!! attention "توجه" - از این روش در پروژه های جدید استفاده نکنید. در صورت امکان, تغییر پروژه های قدیمی به روش بالا توضیح. - -``` sql -Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format - [, kafka_row_delimiter, kafka_schema, kafka_num_consumers, kafka_skip_broken_messages]) -``` - -
- -## توصیف {#description} - -پیام تحویل به طور خودکار ردیابی, بنابراین هر پیام در یک گروه تنها یک بار شمارش. اگر شما می خواهید برای دریافت داده ها دو بار, سپس یک کپی از جدول با نام گروه دیگری ایجاد. - -گروه انعطاف پذیر هستند و همگام سازی در خوشه. برای مثال, اگر شما 10 موضوعات و 5 نسخه از یک جدول در یک خوشه, سپس هر کپی می شود 2 موضوعات. اگر تعداد نسخه تغییر, موضوعات در سراسر نسخه توزیع به طور خودکار. اطلاعات بیشتر در مورد این در http://kafka.apache.org/intro. - -`SELECT` به خصوص برای خواندن پیام های مفید نیست (به جز اشکال زدایی), چرا که هر پیام را می توان تنها یک بار به عنوان خوانده شده. این عملی تر است برای ایجاد موضوعات در زمان واقعی با استفاده از نمایش محقق. برای انجام این کار: - -1. از موتور برای ایجاد یک مصرف کننده کافکا استفاده کنید و جریان داده را در نظر بگیرید. -2. ایجاد یک جدول با ساختار مورد نظر. -3. یک دیدگاه محقق ایجاد کنید که داده ها را از موتور تبدیل می کند و به یک جدول قبلا ایجاد شده تبدیل می کند. - -هنگامی که `MATERIALIZED VIEW` به موتور می پیوندد و شروع به جمع کردن داده ها در پس زمینه می کند. این اجازه می دهد تا شما را به طور مستمر دریافت پیام از کافکا و تبدیل به فرمت مورد نیاز با استفاده از `SELECT`. -یک جدول کافکا می تواند به عنوان بسیاری از دیدگاه های تحقق به عنوان دوست دارید, اطلاعات از جدول کافکا به طور مستقیم به عنوان خوانده شده, اما دریافت پرونده های جدید (در بلوک), به این ترتیب شما می توانید به چند جدول با سطح جزییات مختلف ارسال (با گروه بندی - تجمع و بدون). - -مثال: - -``` sql - CREATE TABLE queue ( - timestamp UInt64, - level String, - message String - ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); - - CREATE TABLE daily ( - day Date, - level String, - total UInt64 - ) ENGINE = SummingMergeTree(day, (day, level), 8192); - - CREATE MATERIALIZED VIEW consumer TO daily - AS SELECT toDate(toDateTime(timestamp)) AS day, level, count() as total - FROM queue GROUP BY day, level; - - SELECT level, sum(total) FROM daily GROUP BY level; -``` - -برای بهبود عملکرد, پیام های دریافت شده را به بلوک های اندازه گروه بندی می شوند [ا_فزونهها](../../../operations/server-configuration-parameters/settings.md#settings-max_insert_block_size). اگر بلوک در داخل تشکیل نشده است [_خاله جریان](../../../operations/server-configuration-parameters/settings.md) میلی ثانیه, داده خواهد شد به جدول بدون در نظر گرفتن کامل از بلوک سرخ. - -برای جلوگیری از دریافت داده های موضوع و یا تغییر منطق تبدیل جدا مشاهده محقق: - -``` sql - DETACH TABLE consumer; - ATTACH TABLE consumer; -``` - -اگر شما می خواهید به تغییر جدول هدف با استفاده از `ALTER` توصیه می کنیم دیدگاه مادی را غیرفعال کنید تا از اختلاف بین جدول هدف و داده ها از نظر جلوگیری شود. - -## پیکربندی {#configuration} - -شبیه به GraphiteMergeTree های کافکا پشتیبانی از موتور تمدید پیکربندی با استفاده از ClickHouse فایل پیکربندی. دو کلید پیکربندی است که شما می توانید استفاده کنید وجود دارد: جهانی (`kafka`) و سطح موضوع (`kafka_*`). پیکربندی جهانی برای اولین بار اعمال می شود و سپس پیکربندی سطح موضوع اعمال می شود (در صورت وجود). - -``` xml - - - cgrp - smallest - - - - - 250 - 100000 - -``` - -برای یک لیست از گزینه های پیکربندی ممکن, دیدن [مرجع پیکربندی کتابدار](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md). استفاده از تاکید (`_`) به جای یک نقطه در پیکربندی کلیک. به عنوان مثال, `check.crcs=true` خواهد بود `true`. - -## ستونهای مجازی {#virtual-columns} - -- `_topic` — Kafka topic. -- `_key` — Key of the message. -- `_offset` — Offset of the message. -- `_timestamp` — Timestamp of the message. -- `_partition` — Partition of Kafka topic. - -**همچنین نگاه کنید به** - -- [ستونهای مجازی](../index.md#table_engines-virtual_columns) - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/kafka/) diff --git a/docs/fa/engines/table-engines/integrations/mysql.md b/docs/fa/engines/table-engines/integrations/mysql.md deleted file mode 100644 index 9c7af095d04..00000000000 --- a/docs/fa/engines/table-engines/integrations/mysql.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 33 -toc_title: MySQL ---- - -# Mysql {#mysql} - -موتور خروجی زیر اجازه می دهد تا شما را به انجام `SELECT` نمایش داده شد در داده است که بر روی یک سرور خروجی از راه دور ذخیره می شود. - -## ایجاد یک جدول {#creating-a-table} - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2], - ... -) ENGINE = MySQL('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']); -``` - -مشاهده شرح مفصلی از [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) پرس و جو. - -ساختار جدول می تواند از ساختار جدول خروجی زیر اصلی متفاوت است: - -- نام ستون باید همان است که در جدول خروجی زیر اصلی باشد, اما شما می توانید تنها برخی از این ستون ها و در هر جهت استفاده. -- انواع ستون ممکن است از کسانی که در جدول خروجی زیر اصلی متفاوت است. فاحشه خانه تلاش می کند تا [بازیگران](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) ارزش ها را به انواع داده های کلیک. - -**پارامترهای موتور** - -- `host:port` — MySQL server address. - -- `database` — Remote database name. - -- `table` — Remote table name. - -- `user` — MySQL user. - -- `password` — User password. - -- `replace_query` — Flag that converts `INSERT INTO` نمایش داده شد به `REPLACE INTO`. اگر `replace_query=1`, پرس و جو جایگزین شده است. - -- `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` بیان است که به اضافه `INSERT` پرس و جو. - - مثال: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1` کجا `on_duplicate_clause` هست `UPDATE c2 = c2 + 1`. دیدن [مستندات خروجی زیر](https://dev.mysql.com/doc/refman/8.0/en/insert-on-duplicate.html) برای پیدا کردن که `on_duplicate_clause` شما می توانید با استفاده از `ON DUPLICATE KEY` بند بند. - - برای مشخص کردن `on_duplicate_clause` شما نیاز به تصویب `0` به `replace_query` پارامتر. اگر شما به طور همزمان عبور `replace_query = 1` و `on_duplicate_clause`, تاتر تولید یک استثنا. - -ساده `WHERE` بند هایی مانند `=, !=, >, >=, <, <=` بر روی سرور خروجی زیر اجرا شده است. - -بقیه شرایط و `LIMIT` محدودیت نمونه برداری در محل کلیک تنها پس از پرس و جو به پس از اتمام خروجی زیر اجرا شده است. - -## مثال طریقه استفاده {#usage-example} - -جدول در خروجی زیر: - -``` text -mysql> CREATE TABLE `test`.`test` ( - -> `int_id` INT NOT NULL AUTO_INCREMENT, - -> `int_nullable` INT NULL DEFAULT NULL, - -> `float` FLOAT NOT NULL, - -> `float_nullable` FLOAT NULL DEFAULT NULL, - -> PRIMARY KEY (`int_id`)); -Query OK, 0 rows affected (0,09 sec) - -mysql> insert into test (`int_id`, `float`) VALUES (1,2); -Query OK, 1 row affected (0,00 sec) - -mysql> select * from test; -+------+----------+-----+----------+ -| int_id | int_nullable | float | float_nullable | -+------+----------+-----+----------+ -| 1 | NULL | 2 | NULL | -+------+----------+-----+----------+ -1 row in set (0,00 sec) -``` - -جدول در تاتر, بازیابی داده ها از جدول خروجی زیر ایجاد شده در بالا: - -``` sql -CREATE TABLE mysql_table -( - `float_nullable` Nullable(Float32), - `int_id` Int32 -) -ENGINE = MySQL('localhost:3306', 'test', 'test', 'bayonet', '123') -``` - -``` sql -SELECT * FROM mysql_table -``` - -``` text -┌─float_nullable─┬─int_id─┐ -│ ᴺᵁᴸᴸ │ 1 │ -└────────────────┴────────┘ -``` - -## همچنین نگاه کنید به {#see-also} - -- [این ‘mysql’ تابع جدول](../../../sql-reference/table-functions/mysql.md) -- [با استفاده از خروجی زیر به عنوان منبع فرهنگ لغت خارجی](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-mysql) - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/mysql/) diff --git a/docs/fa/engines/table-engines/integrations/odbc.md b/docs/fa/engines/table-engines/integrations/odbc.md deleted file mode 100644 index 94299a5ebf0..00000000000 --- a/docs/fa/engines/table-engines/integrations/odbc.md +++ /dev/null @@ -1,132 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 35 -toc_title: ODBC ---- - -# ODBC {#table-engine-odbc} - -اجازه می دهد تا تاتر برای اتصال به پایگاه داده های خارجی از طریق [ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity). - -با خیال راحت پیاده سازی اتصالات ان بی سی, تاتر با استفاده از یک برنامه جداگانه `clickhouse-odbc-bridge`. اگر راننده او بی سی به طور مستقیم از لود `clickhouse-server`, مشکلات راننده می تواند سرور تاتر سقوط. تاتر به طور خودکار شروع می شود `clickhouse-odbc-bridge` هنگامی که مورد نیاز است. برنامه پل او بی سی از همان بسته به عنوان نصب `clickhouse-server`. - -این موتور از [Nullable](../../../sql-reference/data-types/nullable.md) نوع داده. - -## ایجاد یک جدول {#creating-a-table} - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1], - name2 [type2], - ... -) -ENGINE = ODBC(connection_settings, external_database, external_table) -``` - -مشاهده شرح مفصلی از [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) پرس و جو. - -ساختار جدول می تواند از ساختار جدول منبع متفاوت باشد: - -- نام ستون باید همان است که در جدول منبع باشد, اما شما می توانید تنها برخی از این ستون ها و در هر جهت استفاده. -- انواع ستون ممکن است از کسانی که در جدول منبع متفاوت. فاحشه خانه تلاش می کند تا [بازیگران](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) ارزش ها را به انواع داده های کلیک. - -**پارامترهای موتور** - -- `connection_settings` — Name of the section with connection settings in the `odbc.ini` پرونده. -- `external_database` — Name of a database in an external DBMS. -- `external_table` — Name of a table in the `external_database`. - -## مثال طریقه استفاده {#usage-example} - -**بازیابی داده ها از نصب و راه اندازی خروجی زیر محلی از طریق ان بی سی** - -این مثال برای لینوکس اوبونتو 18.04 و سرور خروجی زیر 5.7 بررسی می شود. - -اطمینان حاصل شود که unixODBC و MySQL اتصال نصب شده است. - -به طور پیش فرض (در صورت نصب از بسته), کلیک خانه شروع می شود به عنوان کاربر `clickhouse`. بدین ترتیب, شما نیاز به ایجاد و پیکربندی این کاربر در سرور خروجی زیر. - -``` bash -$ sudo mysql -``` - -``` sql -mysql> CREATE USER 'clickhouse'@'localhost' IDENTIFIED BY 'clickhouse'; -mysql> GRANT ALL PRIVILEGES ON *.* TO 'clickhouse'@'clickhouse' WITH GRANT OPTION; -``` - -سپس اتصال را پیکربندی کنید `/etc/odbc.ini`. - -``` bash -$ cat /etc/odbc.ini -[mysqlconn] -DRIVER = /usr/local/lib/libmyodbc5w.so -SERVER = 127.0.0.1 -PORT = 3306 -DATABASE = test -USERNAME = clickhouse -PASSWORD = clickhouse -``` - -شما می توانید اتصال با استفاده از بررسی `isql` ابزار از unixODBC نصب و راه اندازی. - -``` bash -$ isql -v mysqlconn -+-------------------------+ -| Connected! | -| | -... -``` - -جدول در خروجی زیر: - -``` text -mysql> CREATE TABLE `test`.`test` ( - -> `int_id` INT NOT NULL AUTO_INCREMENT, - -> `int_nullable` INT NULL DEFAULT NULL, - -> `float` FLOAT NOT NULL, - -> `float_nullable` FLOAT NULL DEFAULT NULL, - -> PRIMARY KEY (`int_id`)); -Query OK, 0 rows affected (0,09 sec) - -mysql> insert into test (`int_id`, `float`) VALUES (1,2); -Query OK, 1 row affected (0,00 sec) - -mysql> select * from test; -+------+----------+-----+----------+ -| int_id | int_nullable | float | float_nullable | -+------+----------+-----+----------+ -| 1 | NULL | 2 | NULL | -+------+----------+-----+----------+ -1 row in set (0,00 sec) -``` - -جدول در تاتر بازیابی داده ها از جدول خروجی زیر: - -``` sql -CREATE TABLE odbc_t -( - `int_id` Int32, - `float_nullable` Nullable(Float32) -) -ENGINE = ODBC('DSN=mysqlconn', 'test', 'test') -``` - -``` sql -SELECT * FROM odbc_t -``` - -``` text -┌─int_id─┬─float_nullable─┐ -│ 1 │ ᴺᵁᴸᴸ │ -└────────┴────────────────┘ -``` - -## همچنین نگاه کنید به {#see-also} - -- [لغت نامه های خارجی ان بی سی](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-odbc) -- [تابع جدول ان بی سی](../../../sql-reference/table-functions/odbc.md) - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/odbc/) diff --git a/docs/fa/engines/table-engines/log-family/index.md b/docs/fa/engines/table-engines/log-family/index.md deleted file mode 100644 index e7d22c37912..00000000000 --- a/docs/fa/engines/table-engines/log-family/index.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u062B\u0628\u062A \u062E\u0627\u0646\u0648\u0627\u062F\u0647" -toc_priority: 29 -toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" ---- - -# ورود خانواده موتور {#log-engine-family} - -هنگامی که شما نیاز به سرعت نوشتن بسیاری از جداول کوچک (تا حدود 1 میلیون ردیف) و بعد به عنوان یک کل خواندن این موتور برای حالات توسعه داده شد. - -موتورهای خانواده: - -- [خط زدن](stripelog.md) -- [ثبت](log.md) -- [جمع شدن](tinylog.md) - -## ویژگیهای مشترک {#common-properties} - -موتورها: - -- ذخیره داده ها بر روی یک دیسک. - -- اضافه کردن داده ها به پایان فایل هنگام نوشتن. - -- قفل پشتیبانی برای دسترسی همزمان داده ها. - - در طول `INSERT` نمایش داده شد, جدول قفل شده است, و دیگر نمایش داده شد برای خواندن و نوشتن داده ها هر دو منتظر جدول برای باز کردن. اگر هیچ نمایش داده شد نوشتن داده ها وجود دارد, هر تعداد از نمایش داده شد خواندن داده ها را می توان به صورت همزمان انجام. - -- پشتیبانی نمی کند [جهش](../../../sql-reference/statements/alter.md#alter-mutations) عملیات. - -- هنوز شاخص را پشتیبانی نمی کند. - - این به این معنی است که `SELECT` نمایش داده شد برای محدوده داده ها موثر نیست. - -- هنوز داده نوشتن نیست اتمی. - - شما می توانید یک جدول با داده های خراب اگر چیزی می شکند عملیات نوشتن, مثلا, خاموش کردن سرور غیر طبیعی. - -## تفاوت {#differences} - -این `TinyLog` موتور ساده ترین در خانواده است و فقیرترین قابلیت ها و کمترین بهره وری را فراهم می کند. این `TinyLog` موتور از خواندن داده های موازی با چندین موضوع پشتیبانی نمی کند. این اطلاعات کندتر از موتورهای دیگر در خانواده است که خواندن موازی را پشتیبانی می کند و تقریبا به عنوان بسیاری از توصیفگرها به عنوان `Log` موتور به دلیل ذخیره هر ستون در یک فایل جداگانه. در حالات کم بار ساده استفاده کنید. - -این `Log` و `StripeLog` موتورهای پشتیبانی خواندن داده های موازی. هنگام خواندن داده ها, تاتر با استفاده از موضوعات متعدد. هر موضوع یک بلوک داده جداگانه را پردازش می کند. این `Log` موتور با استفاده از یک فایل جداگانه برای هر ستون از جدول. `StripeLog` ذخیره تمام داده ها در یک فایل. در نتیجه `StripeLog` موتور با استفاده از توصیف کمتر در سیستم عامل, اما `Log` موتور فراهم می کند بهره وری بالاتر در هنگام خواندن داده ها. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/log_family/) - - diff --git a/docs/fa/engines/table-engines/log-family/log.md b/docs/fa/engines/table-engines/log-family/log.md deleted file mode 100644 index a8a871171db..00000000000 --- a/docs/fa/engines/table-engines/log-family/log.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 33 -toc_title: "\u062B\u0628\u062A" ---- - -# ثبت {#log} - -موتور متعلق به خانواده از موتورهای ورود به سیستم. مشاهده خواص مشترک از موتورهای ورود به سیستم و تفاوت های خود را در [ورود خانواده موتور](index.md) مقاله. - -ورود متفاوت از [جمع شدن](tinylog.md) در این فایل کوچک “marks” ساکن با فایل های ستون. این علامت ها در هر بلوک داده نوشته شده است و شامل شیپور خاموشی که نشان می دهد از کجا شروع به خواندن فایل به منظور جست و خیز تعداد مشخصی از ردیف. این باعث می شود امکان خواندن داده های جدول در موضوعات مختلف. -برای همزمان دسترسی به داده ها, عملیات خواندن را می توان به طور همزمان انجام, در حالی که ارسال عملیات بلوک می خواند و هر یک از دیگر. -موتور ورود به سیستم می کند شاخص را پشتیبانی نمی کند. به طور مشابه, اگر نوشتن به یک جدول شکست خورده, جدول شکسته است, و خواندن از این خطا را برمی گرداند. موتور ورود به سیستم مناسب برای داده های موقت است, نوشتن یک بار جداول, و برای تست و یا تظاهرات اهداف. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/log/) diff --git a/docs/fa/engines/table-engines/log-family/stripelog.md b/docs/fa/engines/table-engines/log-family/stripelog.md deleted file mode 100644 index 5c6fe14994e..00000000000 --- a/docs/fa/engines/table-engines/log-family/stripelog.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 32 -toc_title: "\u062E\u0637 \u0632\u062F\u0646" ---- - -# خط زدن {#stripelog} - -این موتور متعلق به خانواده از موتورهای ورود به سیستم. مشاهده خواص مشترک از موتورهای ورود به سیستم و تفاوت های خود را در [ورود خانواده موتور](index.md) مقاله. - -با استفاده از این موتور در حالات زمانی که شما نیاز به نوشتن بسیاری از جداول با مقدار کمی از داده ها (کمتر از 1 میلیون ردیف). - -## ایجاد یک جدول {#table_engines-stripelog-creating-a-table} - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - column1_name [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - column2_name [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... -) ENGINE = StripeLog -``` - -شرح مفصلی از [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) پرس و جو. - -## نوشتن داده ها {#table_engines-stripelog-writing-the-data} - -این `StripeLog` موتور فروشگاه تمام ستون ها در یک فایل. برای هر `INSERT` پرس و جو, خانه رعیتی بلوک داده ها به پایان یک فایل جدول, نوشتن ستون یک به یک. - -برای هر کلیک جدول فایل ها را می نویسد: - -- `data.bin` — Data file. -- `index.mrk` — File with marks. Marks contain offsets for each column of each data block inserted. - -این `StripeLog` موتور را پشتیبانی نمی کند `ALTER UPDATE` و `ALTER DELETE` عملیات. - -## خواندن داده ها {#table_engines-stripelog-reading-the-data} - -فایل را با نشانه اجازه می دهد تا ClickHouse به parallelize خواندن داده ها. این به این معنی است که یک `SELECT` پرس و جو ردیف در جهت غیر قابل پیش بینی می گرداند. استفاده از `ORDER BY` بند برای مرتب کردن ردیف. - -## مثال استفاده {#table_engines-stripelog-example-of-use} - -ایجاد یک جدول: - -``` sql -CREATE TABLE stripe_log_table -( - timestamp DateTime, - message_type String, - message String -) -ENGINE = StripeLog -``` - -درج داده: - -``` sql -INSERT INTO stripe_log_table VALUES (now(),'REGULAR','The first regular message') -INSERT INTO stripe_log_table VALUES (now(),'REGULAR','The second regular message'),(now(),'WARNING','The first warning message') -``` - -ما با استفاده از دو `INSERT` نمایش داده شد برای ایجاد دو بلوک داده ها در داخل `data.bin` پرونده. - -خانه رعیتی با استفاده از موضوعات متعدد در هنگام انتخاب داده ها. هر موضوع یک بلوک داده جداگانه را می خواند و ردیف ها را به طور مستقل به پایان می رساند. در نتیجه, منظور از بلوک های ردیف در خروجی می کند منظور از بلوک های مشابه در ورودی در اکثر موارد مطابقت ندارد. به عنوان مثال: - -``` sql -SELECT * FROM stripe_log_table -``` - -``` text -┌───────────timestamp─┬─message_type─┬─message────────────────────┐ -│ 2019-01-18 14:27:32 │ REGULAR │ The second regular message │ -│ 2019-01-18 14:34:53 │ WARNING │ The first warning message │ -└─────────────────────┴──────────────┴────────────────────────────┘ -┌───────────timestamp─┬─message_type─┬─message───────────────────┐ -│ 2019-01-18 14:23:43 │ REGULAR │ The first regular message │ -└─────────────────────┴──────────────┴───────────────────────────┘ -``` - -مرتب سازی نتایج (صعودی با ترتیب به طور پیش فرض): - -``` sql -SELECT * FROM stripe_log_table ORDER BY timestamp -``` - -``` text -┌───────────timestamp─┬─message_type─┬─message────────────────────┐ -│ 2019-01-18 14:23:43 │ REGULAR │ The first regular message │ -│ 2019-01-18 14:27:32 │ REGULAR │ The second regular message │ -│ 2019-01-18 14:34:53 │ WARNING │ The first warning message │ -└─────────────────────┴──────────────┴────────────────────────────┘ -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/stripelog/) diff --git a/docs/fa/engines/table-engines/log-family/tinylog.md b/docs/fa/engines/table-engines/log-family/tinylog.md deleted file mode 100644 index ea06ba84c96..00000000000 --- a/docs/fa/engines/table-engines/log-family/tinylog.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 34 -toc_title: "\u062C\u0645\u0639 \u0634\u062F\u0646" ---- - -# جمع شدن {#tinylog} - -موتور متعلق به خانواده موتور ورود به سیستم. ببینید [ورود خانواده موتور](index.md) برای خواص مشترک موتورهای ورود به سیستم و تفاوت های خود را. - -این موتور جدول معمولا با روش نوشتن یک بار استفاده می شود: نوشتن داده ها یک بار و سپس خواندن هر چند بار که لازم است. مثلا, شما می توانید استفاده کنید `TinyLog`- نوع جداول برای داده های واسطه است که در دسته های کوچک پردازش شده است. توجه داشته باشید که ذخیره سازی داده ها در تعداد زیادی از جداول کوچک بی اثر است. - -نمایش داده شد در یک جریان واحد اجرا شده است. به عبارت دیگر این موتور برای جداول نسبتا کوچک (تا حدود 1000000 ردیف) در نظر گرفته شده است. این را حس می کند به استفاده از این موتور جدول اگر شما بسیاری از جداول کوچک, از ساده تر از [ثبت](log.md) موتور (فایل های کمتر نیاز به باز شود). - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/tinylog/) diff --git a/docs/fa/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/fa/engines/table-engines/mergetree-family/aggregatingmergetree.md deleted file mode 100644 index c0edee50205..00000000000 --- a/docs/fa/engines/table-engines/mergetree-family/aggregatingmergetree.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 35 -toc_title: "\u0631\u06CC\u0632\u062F\u0627\u0646\u0647" ---- - -# ریزدانه {#aggregatingmergetree} - -موتور به ارث می برد از [ادغام](mergetree.md#table_engines-mergetree), تغییر منطق برای ادغام قطعات داده. تاتر جایگزین تمام ردیف با کلید اصلی همان (یا با دقت بیشتر, با همان [کلید مرتب سازی](mergetree.md)) با یک ردیف (در یک بخش یک داده) که ترکیبی از ایالت های توابع کل را ذخیره می کند. - -شما می توانید استفاده کنید `AggregatingMergeTree` جداول برای تجمع داده افزایشی, از جمله برای نمایش محقق جمع. - -موتور تمام ستون ها را با انواع زیر پردازش می کند: - -- [کارکرد](../../../sql-reference/data-types/aggregatefunction.md) -- [عملکرد پلاکتی](../../../sql-reference/data-types/simpleaggregatefunction.md) - -مناسب برای استفاده است `AggregatingMergeTree` اگر تعداد ردیف ها را با دستور کاهش دهد. - -## ایجاد یک جدول {#creating-a-table} - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... -) ENGINE = AggregatingMergeTree() -[PARTITION BY expr] -[ORDER BY expr] -[SAMPLE BY expr] -[TTL expr] -[SETTINGS name=value, ...] -``` - -برای شرح پارامترهای درخواست را ببینید [درخواست توضیحات](../../../sql-reference/statements/create.md). - -**بندهای پرسوجو** - -هنگام ایجاد یک `AggregatingMergeTree` جدول همان [بند](mergetree.md) در هنگام ایجاد یک مورد نیاز است `MergeTree` جدول - -
- -روش منسوخ برای ایجاد یک جدول - -!!! attention "توجه" - هنوز این روش در پروژه های جدید استفاده کنید و, در صورت امکان, تغییر پروژه های قدیمی به روش بالا توضیح. - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... -) ENGINE [=] AggregatingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity) -``` - -همه پارامترها همان معنی را دارند `MergeTree`. -
- -## انتخاب و درج {#select-and-insert} - -برای وارد کردن داده ها استفاده کنید [INSERT SELECT](../../../sql-reference/statements/insert-into.md) پرس و جو با کل دولت توابع. -هنگام انتخاب داده ها از `AggregatingMergeTree` جدول استفاده کنید `GROUP BY` بند و توابع کل همان هنگام قرار دادن داده, اما با استفاده از `-Merge` پسوند. - -در نتایج `SELECT` پرس و جو, ارزش `AggregateFunction` نوع اجرای خاص نمایندگی دودویی برای همه فرمت های خروجی کلیک کنید. اگر کمپرسی داده ها به, مثلا, `TabSeparated` قالب با `SELECT` پرس و جو و سپس این روگرفت را می توان با استفاده از لود `INSERT` پرس و جو. - -## به عنوان مثال از یک مشاهده محقق جمع {#example-of-an-aggregated-materialized-view} - -`AggregatingMergeTree` مشاهده تحقق است که به تماشای `test.visits` جدول: - -``` sql -CREATE MATERIALIZED VIEW test.basic -ENGINE = AggregatingMergeTree() PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate) -AS SELECT - CounterID, - StartDate, - sumState(Sign) AS Visits, - uniqState(UserID) AS Users -FROM test.visits -GROUP BY CounterID, StartDate; -``` - -درج داده به `test.visits` جدول - -``` sql -INSERT INTO test.visits ... -``` - -داده ها در هر دو جدول و مشخصات قرار داده شده `test.basic` که تجمع انجام خواهد شد. - -برای دریافت اطلاعات جمع, ما نیاز به اجرای یک پرس و جو مانند `SELECT ... GROUP BY ...` از نظر `test.basic`: - -``` sql -SELECT - StartDate, - sumMerge(Visits) AS Visits, - uniqMerge(Users) AS Users -FROM test.basic -GROUP BY StartDate -ORDER BY StartDate; -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/aggregatingmergetree/) diff --git a/docs/fa/engines/table-engines/mergetree-family/collapsingmergetree.md b/docs/fa/engines/table-engines/mergetree-family/collapsingmergetree.md deleted file mode 100644 index d0d5b4323fb..00000000000 --- a/docs/fa/engines/table-engines/mergetree-family/collapsingmergetree.md +++ /dev/null @@ -1,306 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 36 -toc_title: "\u0633\u0642\u0648\u0637 \u063A\u0630\u0627\u06CC \u0627\u0635\u0644\u06CC" ---- - -# سقوط غذای اصلی {#table_engine-collapsingmergetree} - -موتور به ارث می برد از [ادغام](mergetree.md) و می افزاید: منطق ردیف سقوط به قطعات داده الگوریتم ادغام. - -`CollapsingMergeTree` ناهمزمان حذف (فرو می ریزد) جفت ردیف اگر همه از زمینه ها در یک کلید مرتب سازی (`ORDER BY`) معادل به استثنای زمینه خاص است `Sign` که می تواند داشته باشد `1` و `-1` ارزشهای خبری عبارتند از: ردیف بدون یک جفت نگهداری می شوند. برای اطلاعات بیشتر نگاه کنید به [سقوط](#table_engine-collapsingmergetree-collapsing) بخش از سند. - -موتور ممکن است به طور قابل توجهی حجم ذخیره سازی را کاهش دهد و بهره وری را افزایش دهد `SELECT` پرس و جو به عنوان یک نتیجه. - -## ایجاد یک جدول {#creating-a-table} - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... -) ENGINE = CollapsingMergeTree(sign) -[PARTITION BY expr] -[ORDER BY expr] -[SAMPLE BY expr] -[SETTINGS name=value, ...] -``` - -برای شرح پارامترهای پرس و جو, دیدن [توضیحات پرس و جو](../../../sql-reference/statements/create.md). - -**پارامترهای پیش ساخته** - -- `sign` — Name of the column with the type of row: `1` یک “state” سطر, `-1` یک “cancel” پارو زدن. - - Column data type — `Int8`. - -**بندهای پرسوجو** - -هنگام ایجاد یک `CollapsingMergeTree` جدول, همان [بندهای پرسوجو](mergetree.md#table_engine-mergetree-creating-a-table) در هنگام ایجاد یک مورد نیاز است `MergeTree` جدول - -
- -روش منسوخ برای ایجاد یک جدول - -!!! attention "توجه" - هنوز این روش در پروژه های جدید استفاده کنید و, در صورت امکان, تغییر پروژه های قدیمی به روش بالا توضیح. - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... -) ENGINE [=] CollapsingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, sign) -``` - -همه پارامترها به استثنای `sign` همان معنی را در `MergeTree`. - -- `sign` — Name of the column with the type of row: `1` — “state” سطر, `-1` — “cancel” پارو زدن. - - Column Data Type — `Int8`. - -
- -## سقوط {#table_engine-collapsingmergetree-collapsing} - -### داده {#data} - -وضعیت جایی که شما نیاز به ذخیره به طور مداوم در حال تغییر داده ها برای برخی از شی را در نظر بگیرید. برای تلفن های موبایل منطقی به یک ردیف برای یک شی و به روز رسانی در هر تغییر, اما عملیات به روز رسانی گران و کند برای سندرم تونل کارپ است چرا که نیاز به بازنویسی از داده ها در ذخیره سازی. اگر شما نیاز به نوشتن داده ها به سرعت, به روز رسانی قابل قبول نیست, اما شما می توانید تغییرات یک شی پی در پی به شرح زیر ارسال. - -استفاده از ستون خاص `Sign`. اگر `Sign = 1` این بدان معنی است که ردیف دولت از یک شی است, اجازه دهید اسمش را “state” پارو زدن. اگر `Sign = -1` به این معنی لغو دولت از یک شی با ویژگی های مشابه, اجازه دهید اسمش را “cancel” پارو زدن. - -برای مثال ما می خواهیم برای محاسبه چقدر صفحات کاربران بررسی می شود در برخی از سایت و چه مدت وجود دارد. در برخی از لحظه ما ارسال ردیف زیر را با دولت از فعالیت های کاربر: - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ -│ 4324182021466249494 │ 5 │ 146 │ 1 │ -└─────────────────────┴───────────┴──────────┴──────┘ -``` - -در چند لحظه بعد ما تغییر فعالیت کاربر را ثبت می کنیم و با دو ردیف زیر می نویسیم. - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ -│ 4324182021466249494 │ 5 │ 146 │ -1 │ -│ 4324182021466249494 │ 6 │ 185 │ 1 │ -└─────────────────────┴───────────┴──────────┴──────┘ -``` - -ردیف اول لغو حالت قبلی از جسم (کاربر). این باید زمینه های کلیدی مرتب سازی دولت لغو به استثنای کپی کنید `Sign`. - -ردیف دوم شامل وضعیت فعلی. - -همانطور که ما نیاز به تنها دولت گذشته از فعالیت های کاربر, ردیف - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ -│ 4324182021466249494 │ 5 │ 146 │ 1 │ -│ 4324182021466249494 │ 5 │ 146 │ -1 │ -└─────────────────────┴───────────┴──────────┴──────┘ -``` - -می توان حذف سقوط نامعتبر (قدیمی) دولت از یک شی. `CollapsingMergeTree` این کار در حالی که ادغام قطعات داده. - -چرا ما نیاز به 2 ردیف برای هر تغییر در خواندن [الگوریتم](#table_engine-collapsingmergetree-collapsing-algorithm) بند بند بند. - -**خواص عجیب و غریب چنین رویکردی** - -1. برنامه ای که می نویسد داده ها باید به یاد داشته باشید دولت از یک شی قادر به لغو. “Cancel” رشته باید نسخه هایی از زمینه های کلیدی مرتب سازی شامل “state” رشته و مخالف `Sign`. این افزایش اندازه اولیه ذخیره سازی اما اجازه می دهد تا به نوشتن داده ها به سرعت. -2. در حال رشد طولانی در ستون کاهش بهره وری از موتور با توجه به بار برای نوشتن. داده های ساده تر, بالاتر بهره وری. -3. این `SELECT` نتایج به شدت بستگی به قوام شی تغییر تاریخ. هنگام تهیه داده ها برای قرار دادن دقیق باشید. شما می توانید نتایج غیر قابل پیش بینی در اطلاعات متناقض برای مثال مقادیر منفی برای معیارهای غیر منفی مانند جلسه عمق. - -### الگوریتم {#table_engine-collapsingmergetree-collapsing-algorithm} - -هنگامی که تاتر ادغام قطعات داده, هر گروه از ردیف متوالی با کلید مرتب سازی همان (`ORDER BY`) به بیش از دو ردیف کاهش می یابد, یکی با `Sign = 1` (“state” ردیف) و دیگری با `Sign = -1` (“cancel” ردیف). به عبارت دیگر, سقوط نوشته. - -برای هر یک از داده ها در نتیجه بخشی تاتر موجب صرفه جویی در: - -1. اولین “cancel” و گذشته “state” ردیف, اگر تعداد “state” و “cancel” ردیف مسابقات و ردیف گذشته است “state” پارو زدن. -2. گذشته “state” ردیف, اگر بیشتر وجود دارد “state” سطر از “cancel” ردیف -3. اولین “cancel” ردیف, اگر بیشتر وجود دارد “cancel” سطر از “state” ردیف -4. هیچ یک از ردیف, در تمام موارد دیگر. - -همچنین زمانی که حداقل وجود دارد 2 بیشتر “state” سطر از “cancel” ردیف یا حداقل 2 بیشتر “cancel” سپس سطرها “state” ردیف, ادغام ادامه, اما تاتر این وضعیت رفتار به عنوان یک خطای منطقی و ثبت در ورود به سیستم سرور. این خطا می تواند رخ دهد اگر داده های مشابه بیش از یک بار قرار داده شد. - -بدین ترتیب, سقوط باید نتایج حاصل از محاسبه ارقام تغییر نمی. -تغییرات به تدریج فرو ریخت به طوری که در پایان تنها دولت گذشته تقریبا در هر شی را ترک کرد. - -این `Sign` لازم است زیرا الگوریتم ادغام تضمین نمی کند که تمام ردیف ها با کلید مرتب سازی مشابه در بخش داده های مشابه و حتی در همان سرور فیزیکی باشد. روند کلیک `SELECT` نمایش داده شد با موضوعات مختلف و می تواند منظور از ردیف در نتیجه پیش بینی نیست. تجمع مورد نیاز است اگر نیاز به طور کامل وجود دارد “collapsed” داده ها از `CollapsingMergeTree` جدول - -برای نهایی سقوط, نوشتن یک پرس و جو با `GROUP BY` بند و مجموع توابع است که برای ثبت نام حساب. برای مثال برای محاسبه مقدار استفاده کنید `sum(Sign)` به جای `count()`. برای محاسبه مجموع چیزی استفاده کنید `sum(Sign * x)` به جای `sum(x)` و به همین ترتیب و همچنین اضافه کنید `HAVING sum(Sign) > 0`. - -مصالح `count`, `sum` و `avg` می تواند محاسبه این راه. مجموع `uniq` می تواند محاسبه شود اگر یک شی حداقل یک دولت سقوط نیست. مصالح `min` و `max` محاسبه نشد زیرا `CollapsingMergeTree` می کند تاریخ ارزش از کشورهای سقوط را نجات دهد. - -اگر شما نیاز به استخراج داده ها بدون تجمع (مثلا, برای بررسی اینکه ردیف در حال حاضر که جدیدترین ارزش مطابقت با شرایط خاص هستند), شما می توانید با استفاده از `FINAL` تغییردهنده برای `FROM` بند بند. این رویکرد به طور قابل توجهی کمتر موثر است. - -## مثال استفاده {#example-of-use} - -اطلاعات نمونه: - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ -│ 4324182021466249494 │ 5 │ 146 │ 1 │ -│ 4324182021466249494 │ 5 │ 146 │ -1 │ -│ 4324182021466249494 │ 6 │ 185 │ 1 │ -└─────────────────────┴───────────┴──────────┴──────┘ -``` - -ایجاد جدول: - -``` sql -CREATE TABLE UAct -( - UserID UInt64, - PageViews UInt8, - Duration UInt8, - Sign Int8 -) -ENGINE = CollapsingMergeTree(Sign) -ORDER BY UserID -``` - -درج داده ها: - -``` sql -INSERT INTO UAct VALUES (4324182021466249494, 5, 146, 1) -``` - -``` sql -INSERT INTO UAct VALUES (4324182021466249494, 5, 146, -1),(4324182021466249494, 6, 185, 1) -``` - -ما با استفاده از دو `INSERT` نمایش داده شد برای ایجاد دو بخش داده های مختلف. اگر ما وارد کردن داده ها با یک پرس و جو تاتر ایجاد یک بخش داده ها و هر گونه ادغام تا کنون انجام نمی. - -گرفتن داده ها: - -``` sql -SELECT * FROM UAct -``` - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ -│ 4324182021466249494 │ 5 │ 146 │ -1 │ -│ 4324182021466249494 │ 6 │ 185 │ 1 │ -└─────────────────────┴───────────┴──────────┴──────┘ -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ -│ 4324182021466249494 │ 5 │ 146 │ 1 │ -└─────────────────────┴───────────┴──────────┴──────┘ -``` - -چه ما را ببینید و جایی که در حال سقوط است? - -با دو `INSERT` نمایش داده شد, ما ایجاد 2 قطعات داده. این `SELECT` پرس و جو در انجام شد 2 موضوعات, و ما یک نظم تصادفی از ردیف کردم. سقوط رخ داده است چرا که هیچ ادغام از قطعات داده وجود دارد و در عین حال. تاتر ادغام بخش داده ها در یک لحظه ناشناخته که ما نمی توانیم پیش بینی. - -بنابراین ما نیاز به تجمع: - -``` sql -SELECT - UserID, - sum(PageViews * Sign) AS PageViews, - sum(Duration * Sign) AS Duration -FROM UAct -GROUP BY UserID -HAVING sum(Sign) > 0 -``` - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┐ -│ 4324182021466249494 │ 6 │ 185 │ -└─────────────────────┴───────────┴──────────┘ -``` - -اگر ما تجمع نیاز ندارد و می خواهید به زور سقوط, ما می توانیم با استفاده از `FINAL` تغییردهنده برای `FROM` بند بند. - -``` sql -SELECT * FROM UAct FINAL -``` - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ -│ 4324182021466249494 │ 6 │ 185 │ 1 │ -└─────────────────────┴───────────┴──────────┴──────┘ -``` - -این روش انتخاب داده ها بسیار کم است. برای میزهای بزرگ ازش استفاده نکن - -## نمونه ای از روش دیگری {#example-of-another-approach} - -اطلاعات نمونه: - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ -│ 4324182021466249494 │ 5 │ 146 │ 1 │ -│ 4324182021466249494 │ -5 │ -146 │ -1 │ -│ 4324182021466249494 │ 6 │ 185 │ 1 │ -└─────────────────────┴───────────┴──────────┴──────┘ -``` - -ایده این است که ادغام را به حساب تنها زمینه های کلیدی. و در “Cancel” خط ما می توانیم مقادیر منفی که برابر نسخه های قبلی از ردیف در هنگام جمع بدون استفاده از ستون نشانه را مشخص کنید. برای این روش لازم است نوع داده را تغییر دهید `PageViews`,`Duration` برای ذخیره مقادیر منفی از UInt8 -\> Int16. - -``` sql -CREATE TABLE UAct -( - UserID UInt64, - PageViews Int16, - Duration Int16, - Sign Int8 -) -ENGINE = CollapsingMergeTree(Sign) -ORDER BY UserID -``` - -بیایید روش را تست کنیم: - -``` sql -insert into UAct values(4324182021466249494, 5, 146, 1); -insert into UAct values(4324182021466249494, -5, -146, -1); -insert into UAct values(4324182021466249494, 6, 185, 1); - -select * from UAct final; // avoid using final in production (just for a test or small tables) -``` - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ -│ 4324182021466249494 │ 6 │ 185 │ 1 │ -└─────────────────────┴───────────┴──────────┴──────┘ -``` - -``` sql -SELECT - UserID, - sum(PageViews) AS PageViews, - sum(Duration) AS Duration -FROM UAct -GROUP BY UserID -```text -┌──────────────UserID─┬─PageViews─┬─Duration─┐ -│ 4324182021466249494 │ 6 │ 185 │ -└─────────────────────┴───────────┴──────────┘ -``` - -``` sqk -select count() FROM UAct -``` - -``` text -┌─count()─┐ -│ 3 │ -└─────────┘ -``` - -``` sql -optimize table UAct final; - -select * FROM UAct -``` - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ -│ 4324182021466249494 │ 6 │ 185 │ 1 │ -└─────────────────────┴───────────┴──────────┴──────┘ -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/collapsingmergetree/) diff --git a/docs/fa/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/fa/engines/table-engines/mergetree-family/custom-partitioning-key.md deleted file mode 100644 index 825535c9c95..00000000000 --- a/docs/fa/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ /dev/null @@ -1,128 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 32 -toc_title: "\u06A9\u0644\u06CC\u062F \u067E\u0627\u0631\u062A\u06CC\u0634\u0646 \u0628\ - \u0646\u062F\u06CC \u0633\u0641\u0627\u0631\u0634\u06CC" ---- - -# کلید پارتیشن بندی سفارشی {#custom-partitioning-key} - -پارتیشن بندی برای [ادغام](mergetree.md) جداول خانواده (شامل [تکرار](replication.md) جدول). [نمایش محقق](../special/materializedview.md#materializedview) بر اساس جداول ادغام پشتیبانی پارتیشن بندی, همچنین. - -پارتیشن ترکیبی منطقی از سوابق در یک جدول توسط یک معیار مشخص شده است. شما می توانید یک پارتیشن توسط معیار دلخواه تنظیم, مانند ماه, به روز, و یا بر اساس نوع رویداد. هر پارتیشن به طور جداگانه ذخیره می شود به ساده دستکاری این داده ها. هنگام دسترسی به داده ها, تاتر با استفاده از کوچکترین زیر مجموعه از پارتیشن ممکن. - -پارتیشن در مشخص `PARTITION BY expr` بند زمانی که [ایجاد یک جدول](mergetree.md#table_engine-mergetree-creating-a-table). کلید پارتیشن می تواند هر عبارت از ستون های جدول باشد. برای مثال برای مشخص کردن پارتیشن بندی توسط ماه با استفاده از بیان `toYYYYMM(date_column)`: - -``` sql -CREATE TABLE visits -( - VisitDate Date, - Hour UInt8, - ClientID UUID -) -ENGINE = MergeTree() -PARTITION BY toYYYYMM(VisitDate) -ORDER BY Hour; -``` - -کلید پارتیشن همچنین می تواند یک تاپل از عبارات (شبیه به [کلید اصلی](mergetree.md#primary-keys-and-indexes-in-queries)). به عنوان مثال: - -``` sql -ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/name', 'replica1', Sign) -PARTITION BY (toMonday(StartDate), EventType) -ORDER BY (CounterID, StartDate, intHash32(UserID)); -``` - -در این مثال ما مجموعه پارتیشن بندی توسط انواع رویداد رخ داده است که در طول هفته جاری. - -هنگام قرار دادن داده های جدید به یک جدول, این داده ها به عنوان یک بخش جداگانه ذخیره می شود (تکه) مرتب شده بر اساس کلید اصلی. در 10-15 دقیقه پس از قرار دادن, بخش هایی از پارتیشن همان به کل بخش با هم ادغام شدند. - -!!! info "اطلاعات" - ادغام تنها برای قطعات داده که همان مقدار برای بیان پارتیشن بندی کار می کند. این به این معنی است **شما باید پارتیشن بیش از حد دانه را ندارد** (بیش از حدود یک هزار پارتیشن). در غیر این صورت `SELECT` پرس و جو انجام ضعیف به دلیل تعداد نامعقول زیادی از فایل ها در سیستم فایل و توصیف باز کردن فایل. - -استفاده از [سیستم.قطعات](../../../operations/system-tables.md#system_tables-parts) جدول برای مشاهده قطعات جدول و پارتیشن. مثلا, اجازه دهید فرض کنیم که ما یک `visits` جدول با پارتیشن بندی در ماه. بیایید انجام دهیم `SELECT` پرسوجو برای `system.parts` جدول: - -``` sql -SELECT - partition, - name, - active -FROM system.parts -WHERE table = 'visits' -``` - -``` text -┌─partition─┬─name───────────┬─active─┐ -│ 201901 │ 201901_1_3_1 │ 0 │ -│ 201901 │ 201901_1_9_2 │ 1 │ -│ 201901 │ 201901_8_8_0 │ 0 │ -│ 201901 │ 201901_9_9_0 │ 0 │ -│ 201902 │ 201902_4_6_1 │ 1 │ -│ 201902 │ 201902_10_10_0 │ 1 │ -│ 201902 │ 201902_11_11_0 │ 1 │ -└───────────┴────────────────┴────────┘ -``` - -این `partition` ستون شامل نام پارتیشن. دو پارتیشن در این مثال وجود دارد: `201901` و `201902`. شما می توانید از این مقدار ستون برای مشخص کردن نام پارتیشن در استفاده کنید [ALTER … PARTITION](#alter_manipulations-with-partitions) نمایش داده شد. - -این `name` ستون شامل نام قطعات داده پارتیشن. شما می توانید از این ستون برای مشخص کردن نام شرکت در [ALTER ATTACH PART](#alter_attach-partition) پرس و جو. - -بیایید شکستن نام بخش اول: `201901_1_3_1`: - -- `201901` نام پارتیشن است. -- `1` حداقل تعداد بلوک داده است. -- `3` حداکثر تعداد بلوک داده است. -- `1` سطح تکه (عمق درخت ادغام از تشکیل شده است). - -!!! info "اطلاعات" - بخش هایی از جداول قدیمی از نوع نام: `20190117_20190123_2_2_0` (حداقل تاریخ - حداکثر تاریخ - حداقل تعداد بلوک - حداکثر تعداد بلوک - سطح). - -این `active` ستون وضعیت بخش را نشان می دهد. `1` فعال است; `0` غیر فعال است. قطعات غیر فعال هستند, مثلا, قطعات منبع باقی مانده پس از ادغام به بخش بزرگتر. قطعات داده خراب نیز به عنوان غیر فعال نشان داد. - -همانطور که شما می توانید در مثال ببینید, چندین بخش از هم جدا از پارتیشن های مشابه وجود دارد (مثلا, `201901_1_3_1` و `201901_1_9_2`). این به این معنی است که این قطعات با هم ادغام شدند و در عین حال. تاتر بخش های داده شده داده ها را به صورت دوره ای در حدود 15 دقیقه پس از قرار دادن ادغام می کند. علاوه بر این, شما می توانید یک ادغام غیر برنامه ریزی شده با استفاده از انجام [OPTIMIZE](../../../sql-reference/statements/misc.md#misc_operations-optimize) پرس و جو. مثال: - -``` sql -OPTIMIZE TABLE visits PARTITION 201902; -``` - -``` text -┌─partition─┬─name───────────┬─active─┐ -│ 201901 │ 201901_1_3_1 │ 0 │ -│ 201901 │ 201901_1_9_2 │ 1 │ -│ 201901 │ 201901_8_8_0 │ 0 │ -│ 201901 │ 201901_9_9_0 │ 0 │ -│ 201902 │ 201902_4_6_1 │ 0 │ -│ 201902 │ 201902_4_11_2 │ 1 │ -│ 201902 │ 201902_10_10_0 │ 0 │ -│ 201902 │ 201902_11_11_0 │ 0 │ -└───────────┴────────────────┴────────┘ -``` - -قطعات غیر فعال خواهد شد حدود حذف 10 دقیقه پس از ادغام. - -راه دیگر برای مشاهده مجموعه ای از قطعات و پارتیشن ها این است که به دایرکتوری جدول بروید: `/var/lib/clickhouse/data///`. به عنوان مثال: - -``` bash -/var/lib/clickhouse/data/default/visits$ ls -l -total 40 -drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 201901_1_3_1 -drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201901_1_9_2 -drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 15:52 201901_8_8_0 -drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 15:52 201901_9_9_0 -drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201902_10_10_0 -drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201902_11_11_0 -drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:19 201902_4_11_2 -drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 12:09 201902_4_6_1 -drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 detached -``` - -پوشهها ‘201901_1_1_0’, ‘201901_1_7_1’ و به همین ترتیب دایرکتوری از قطعات هستند. هر بخش مربوط به پارتیشن مربوطه و شامل داده ها فقط برای یک ماه خاص (جدول در این مثال پارتیشن بندی توسط ماه). - -این `detached` دایرکتوری شامل قطعات است که از جدول با استفاده از جدا شد [DETACH](../../../sql-reference/statements/alter.md#alter_detach-partition) پرس و جو. قطعات خراب نیز به این دایرکتوری منتقل, به جای اینکه حذف. سرور از قطعات از `detached` directory. You can add, delete, or modify the data in this directory at any time – the server will not know about this until you run the [ATTACH](../../../sql-reference/statements/alter.md#alter_attach-partition) پرس و جو. - -توجه داشته باشید که در سرور عامل شما نمی توانید به صورت دستی مجموعه ای از قطعات یا داده های خود را بر روی سیستم فایل تغییر دهید زیرا سرور در این مورد نمی داند. برای جداول غیر تکرار, شما می توانید این کار را انجام زمانی که سرور متوقف شده است, اما توصیه نمی شود. برای جداول تکرار, مجموعه ای از قطعات را نمی توان در هر صورت تغییر. - -کلیک هاوس اجازه می دهد تا شما را به انجام عملیات با پارتیشن: حذف, کپی از یک جدول به دیگری, و یا ایجاد یک نسخه پشتیبان تهیه. مشاهده لیست تمام عملیات در بخش [دستکاری با پارتیشن ها و قطعات](../../../sql-reference/statements/alter.md#alter_manipulations-with-partitions). - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/custom_partitioning_key/) diff --git a/docs/fa/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/fa/engines/table-engines/mergetree-family/graphitemergetree.md deleted file mode 100644 index 2b2ed45152d..00000000000 --- a/docs/fa/engines/table-engines/mergetree-family/graphitemergetree.md +++ /dev/null @@ -1,174 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 38 -toc_title: "\u0646\u0645\u0648\u062F\u0627\u0631" ---- - -# نمودار {#graphitemergetree} - -این موتور طراحی شده است برای نازک شدن و جمع/متوسط (خلاصه) [گرافیت](http://graphite.readthedocs.io/en/latest/index.html) داده ها. این ممکن است به توسعه دهندگان که می خواهند به استفاده از تاتر به عنوان یک فروشگاه داده ها برای گرافیت مفید است. - -شما می توانید هر موتور جدول کلیک برای ذخیره داده گرافیت اگر شما رولپ نیاز ندارد استفاده, اما اگر شما نیاز به یک استفاده خلاصه `GraphiteMergeTree`. موتور حجم ذخیره سازی را کاهش می دهد و بهره وری نمایش داده شد از گرافیت را افزایش می دهد. - -موتور خواص از ارث می برد [ادغام](mergetree.md). - -## ایجاد یک جدول {#creating-table} - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - Path String, - Time DateTime, - Value , - Version - ... -) ENGINE = GraphiteMergeTree(config_section) -[PARTITION BY expr] -[ORDER BY expr] -[SAMPLE BY expr] -[SETTINGS name=value, ...] -``` - -مشاهده شرح مفصلی از [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) پرس و جو. - -یک جدول برای داده های گرافیت باید ستون های زیر را برای داده های زیر داشته باشد: - -- نام متریک (سنسور گرافیت). نوع داده: `String`. - -- زمان اندازه گیری متریک. نوع داده: `DateTime`. - -- ارزش متریک. نوع داده: هر عددی. - -- نسخه از متریک. نوع داده: هر عددی. - - تاتر موجب صرفه جویی در ردیف با بالاترین نسخه و یا گذشته نوشته شده است اگر نسخه یکسان هستند. ردیف های دیگر در طول ادغام قطعات داده حذف می شوند. - -نام این ستون ها باید در پیکربندی خلاصه مجموعه. - -**پارامترهای نمودار** - -- `config_section` — Name of the section in the configuration file, where are the rules of rollup set. - -**بندهای پرسوجو** - -هنگام ایجاد یک `GraphiteMergeTree` جدول, همان [بند](mergetree.md#table_engine-mergetree-creating-a-table) در هنگام ایجاد یک مورد نیاز است `MergeTree` جدول - -
- -روش منسوخ برای ایجاد یک جدول - -!!! attention "توجه" - هنوز این روش در پروژه های جدید استفاده کنید و, در صورت امکان, تغییر پروژه های قدیمی به روش بالا توضیح. - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - EventDate Date, - Path String, - Time DateTime, - Value , - Version - ... -) ENGINE [=] GraphiteMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, config_section) -``` - -همه پارامترها به استثنای `config_section` همان معنی را در `MergeTree`. - -- `config_section` — Name of the section in the configuration file, where are the rules of rollup set. - -
- -## پیکربندی رولپ {#rollup-configuration} - -تنظیمات برای خلاصه توسط تعریف [لغزش _ نمودار](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-graphite) پارامتر در پیکربندی سرور. نام پارامتر می تواند هر. شما می توانید تنظیمات متعددی ایجاد کنید و برای جداول مختلف استفاده کنید. - -ساختار پیکربندی رولپ: - - required-columns - patterns - -### ستون های مورد نیاز {#required-columns} - -- `path_column_name` — The name of the column storing the metric name (Graphite sensor). Default value: `Path`. -- `time_column_name` — The name of the column storing the time of measuring the metric. Default value: `Time`. -- `value_column_name` — The name of the column storing the value of the metric at the time set in `time_column_name`. مقدار پیشفرض: `Value`. -- `version_column_name` — The name of the column storing the version of the metric. Default value: `Timestamp`. - -### الگوها {#patterns} - -ساختار `patterns` بخش: - -``` text -pattern - regexp - function -pattern - regexp - age + precision - ... -pattern - regexp - function - age + precision - ... -pattern - ... -default - function - age + precision - ... -``` - -!!! warning "توجه" - الگوها باید به شدت دستور داده شوند: - - 1. Patterns without `function` or `retention`. - 1. Patterns with both `function` and `retention`. - 1. Pattern `default`. - -هنگام پردازش یک ردیف, تاتر چک قوانین در `pattern` بخش. هر یک از `pattern` (شامل `default`) بخش می تواند شامل `function` پارامتر برای تجمع, `retention` پارامترها یا هر دو. اگر نام متریک با `regexp`, قوانین از `pattern` بخش (یا بخش) اعمال می شود; در غیر این صورت, قوانین از `default` بخش استفاده می شود. - -زمینه برای `pattern` و `default` بخش ها: - -- `regexp`– A pattern for the metric name. -- `age` – The minimum age of the data in seconds. -- `precision`– How precisely to define the age of the data in seconds. Should be a divisor for 86400 (seconds in a day). -- `function` – The name of the aggregating function to apply to data whose age falls within the range `[age, age + precision]`. - -### مثال پیکربندی {#configuration-example} - -``` xml - - Version - - click_cost - any - - 0 - 5 - - - 86400 - 60 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/graphitemergetree/) diff --git a/docs/fa/engines/table-engines/mergetree-family/index.md b/docs/fa/engines/table-engines/mergetree-family/index.md deleted file mode 100644 index c1d84083fea..00000000000 --- a/docs/fa/engines/table-engines/mergetree-family/index.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0627\u062F\u063A\u0627\u0645 \u062E\u0627\u0646\u0648\u0627\u062F\ - \u0647" -toc_priority: 28 ---- - - diff --git a/docs/fa/engines/table-engines/mergetree-family/mergetree.md b/docs/fa/engines/table-engines/mergetree-family/mergetree.md deleted file mode 100644 index 264c7379466..00000000000 --- a/docs/fa/engines/table-engines/mergetree-family/mergetree.md +++ /dev/null @@ -1,654 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 30 -toc_title: "\u0627\u062F\u063A\u0627\u0645" ---- - -# ادغام {#table_engines-mergetree} - -این `MergeTree` موتور و سایر موتورهای این خانواده (`*MergeTree`) موتورهای جدول کلیک قوی ترین. - -موتور در `MergeTree` خانواده برای قرار دادن مقدار بسیار زیادی از داده ها را به یک جدول طراحی شده است. داده ها به سرعت به بخش جدول توسط بخش نوشته شده است, سپس قوانین برای ادغام قطعات در پس زمینه اعمال. این روش بسیار موثرتر از به طور مستمر بازنویسی داده ها در ذخیره سازی در درج است. - -ویژگی های اصلی: - -- فروشگاه داده طبقه بندی شده اند توسط کلید اصلی. - - این اجازه می دهد تا به شما برای ایجاد یک شاخص پراکنده کوچک است که کمک می کند تا پیدا کردن اطلاعات سریع تر. - -- پارتیشن ها را می توان در صورت استفاده کرد [کلید پارتیشن بندی](custom-partitioning-key.md) مشخص شده است. - - تاتر پشتیبانی از عملیات خاص با پارتیشن که موثر تر از عملیات عمومی بر روی داده های مشابه با همان نتیجه. کلیک هاوس همچنین به طور خودکار داده های پارتیشن را که کلید پارتیشن بندی در پرس و جو مشخص شده است قطع می کند. این نیز باعث بهبود عملکرد پرس و جو. - -- پشتیبانی از تکرار داده ها. - - خانواده `ReplicatedMergeTree` جداول فراهم می کند تکرار داده ها. برای کسب اطلاعات بیشتر, دیدن [تکرار داده ها](replication.md). - -- پشتیبانی از نمونه برداری داده ها. - - در صورت لزوم می توانید روش نمونه گیری داده ها را در جدول تنظیم کنید. - -!!! info "اطلاعات" - این [ادغام](../special/merge.md#merge) موتور به تعلق ندارد `*MergeTree` خانواده - -## ایجاد یک جدول {#table_engine-mergetree-creating-a-table} - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2], - ... - INDEX index_name1 expr1 TYPE type1(...) GRANULARITY value1, - INDEX index_name2 expr2 TYPE type2(...) GRANULARITY value2 -) ENGINE = MergeTree() -[PARTITION BY expr] -[ORDER BY expr] -[PRIMARY KEY expr] -[SAMPLE BY expr] -[TTL expr [DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'], ...] -[SETTINGS name=value, ...] -``` - -برای شرح پارامترها [ایجاد توصیف پرسوجو](../../../sql-reference/statements/create.md). - -!!! note "یادداشت" - `INDEX` یک ویژگی تجربی است [شاخص های داده پرش](#table_engine-mergetree-data_skipping-indexes). - -### بندهای پرسوجو {#mergetree-query-clauses} - -- `ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. این `MergeTree` موتور پارامترهای ندارد. - -- `PARTITION BY` — The [کلید پارتیشن بندی](custom-partitioning-key.md). - - برای تقسیم ماه از `toYYYYMM(date_column)` عبارت, جایی که `date_column` یک ستون با تاریخ از نوع است [تاریخ](../../../sql-reference/data-types/date.md). نام پارتیشن در اینجا `"YYYYMM"` قالب. - -- `ORDER BY` — The sorting key. - - یک تاپل از ستون ها و یا عبارات دلخواه. مثال: `ORDER BY (CounterID, EventDate)`. - -- `PRIMARY KEY` — The primary key if it [متفاوت از کلید مرتب سازی](#choosing-a-primary-key-that-differs-from-the-sorting-key). - - به طور پیش فرض کلید اصلی همان کلید مرتب سازی است (که توسط مشخص شده است `ORDER BY` بند). بنابراین در اکثر موارد غیر ضروری است برای مشخص کردن یک جداگانه `PRIMARY KEY` بند بند. - -- `SAMPLE BY` — An expression for sampling. - - اگر یک عبارت نمونه برداری استفاده شده است, کلید اصلی باید باشد. مثال: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. - -- `TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [بین دیسک و حجم](#table_engine-mergetree-multiple-volumes). - - بیان باید یکی داشته باشد `Date` یا `DateTime` ستون به عنوان یک نتیجه. مثال: - `TTL date + INTERVAL 1 DAY` - - نوع قانون `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'` مشخص یک عمل با بخش انجام می شود در صورتی که بیان راضی است (می رسد زمان فعلی): حذف ردیف منقضی شده, در حال حرکت بخشی (اگر بیان برای تمام ردیف در یک بخش راضی است) به دیسک مشخص (`TO DISK 'xxx'`) یا به حجم (`TO VOLUME 'xxx'`). نوع پیش فرض قانون حذف است (`DELETE`). فهرست قوانین متعدد می توانید مشخص, اما باید بیش از یک وجود داشته باشد `DELETE` قانون. - - برای اطلاعات بیشتر, دیدن [ستون ها و جداول](#table_engine-mergetree-ttl) - -- `SETTINGS` — Additional parameters that control the behavior of the `MergeTree`: - - - `index_granularity` — Maximum number of data rows between the marks of an index. Default value: 8192. See [ذخیره سازی داده ها](#mergetree-data-storage). - - `index_granularity_bytes` — Maximum size of data granules in bytes. Default value: 10Mb. To restrict the granule size only by number of rows, set to 0 (not recommended). See [ذخیره سازی داده ها](#mergetree-data-storage). - - `enable_mixed_granularity_parts` — Enables or disables transitioning to control the granule size with the `index_granularity_bytes` تنظیمات. قبل از نسخه 19.11, تنها وجود دارد `index_granularity` تنظیم برای محدود کردن اندازه گرانول. این `index_granularity_bytes` تنظیم را بهبود می بخشد عملکرد کلیک در هنگام انتخاب داده ها از جداول با ردیف بزرگ (ده ها و صدها مگابایت). اگر شما جداول با ردیف بزرگ, شما می توانید این تنظیمات را برای جداول را قادر به بهبود بهره وری از `SELECT` نمایش داده شد. - - `use_minimalistic_part_header_in_zookeeper` — Storage method of the data parts headers in ZooKeeper. If `use_minimalistic_part_header_in_zookeeper=1`, سپس باغ وحش ذخیره داده های کمتر. برای کسب اطلاعات بیشتر, دیدن [تنظیم توضیحات](../../../operations/server-configuration-parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) داخل “Server configuration parameters”. - - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` بایت, تاتر می خواند و می نویسد داده ها به دیسک ذخیره سازی با استفاده از رابط من/ای مستقیم (`O_DIRECT` گزینه). اگر `min_merge_bytes_to_use_direct_io = 0`, سپس مستقیم من / ای غیر فعال است. مقدار پیشفرض: `10 * 1024 * 1024 * 1024` بایت - - - `merge_with_ttl_timeout` — Minimum delay in seconds before repeating a merge with TTL. Default value: 86400 (1 day). - - `write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don't turn it off. - - `merge_max_block_size` — Maximum number of rows in block for merge operations. Default value: 8192. - - `storage_policy` — Storage policy. See [با استفاده از دستگاه های بلوک های متعدد برای ذخیره سازی داده ها](#table_engine-mergetree-multiple-volumes). - -**مثال تنظیمات بخش** - -``` sql -ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity=8192 -``` - -در مثال ما مجموعه پارتیشن بندی توسط ماه. - -ما همچنین یک عبارت برای نمونه برداری به عنوان یک هش توسط شناسه کاربر تنظیم شده است. این اجازه می دهد تا شما را به نام مستعار داده ها در جدول برای هر `CounterID` و `EventDate`. اگر یک تعریف می کنید [SAMPLE](../../../sql-reference/statements/select/sample.md#select-sample-clause) بند هنگام انتخاب داده ClickHouse را یک به طور مساوی pseudorandom داده های نمونه به صورت زیر مجموعه ای از کاربران است. - -این `index_granularity` تنظیم می تواند حذف شود زیرا 8192 مقدار پیش فرض است. - -
- -روش منسوخ برای ایجاد یک جدول - -!!! attention "توجه" - از این روش در پروژه های جدید استفاده نکنید. در صورت امکان, تغییر پروژه های قدیمی به روش بالا توضیح. - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... -) ENGINE [=] MergeTree(date-column [, sampling_expression], (primary, key), index_granularity) -``` - -**پارامترهای ادغام() ** - -- `date-column` — The name of a column of the [تاریخ](../../../sql-reference/data-types/date.md) نوع. تاتر به طور خودکار ایجاد پارتیشن های ماه بر اساس این ستون. نام پارتیشن در `"YYYYMM"` قالب. -- `sampling_expression` — An expression for sampling. -- `(primary, key)` — Primary key. Type: [تاپل()](../../../sql-reference/data-types/tuple.md) -- `index_granularity` — The granularity of an index. The number of data rows between the “marks” از یک شاخص. ارزش 8192 برای بسیاری از وظایف مناسب است. - -**مثال** - -``` sql -MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID)), 8192) -``` - -این `MergeTree` موتور در همان راه به عنوان مثال بالا برای روش پیکربندی موتور اصلی پیکربندی شده است. -
- -## ذخیره سازی داده ها {#mergetree-data-storage} - -جدول شامل قطعات داده مرتب شده بر اساس کلید اصلی. - -هنگامی که داده ها در یک جدول قرار داده, قطعات داده های جداگانه ایجاد می شوند و هر یک از این واژه ها از لحاظ واژگان توسط کلید اصلی طبقه بندی شده اند. برای مثال اگر کلید اصلی است `(CounterID, Date)` داده ها در بخش طبقه بندی شده اند `CounterID` و در هر `CounterID`, این است که توسط دستور داد `Date`. - -داده های متعلق به پارتیشن های مختلف به بخش های مختلف جدا می شوند. در پس زمینه, کلیکهاوس ادغام قطعات داده ها برای ذخیره سازی موثر تر. قطعات متعلق به پارتیشن های مختلف با هم ادغام شدند. مکانیزم ادغام تضمین نمی کند که تمام ردیف ها با همان کلید اصلی در بخش داده های مشابه باشد. - -هر بخش داده منطقی به گرانول تقسیم شده است. گرانول کوچکترین مجموعه داده های تفکیک پذیر است که خانه می خواند در هنگام انتخاب داده ها است. خانه را کلیک می کند ردیف یا ارزش تقسیم نمی, بنابراین هر گرانول همیشه شامل یک عدد صحیح از ردیف. ردیف اول یک گرانول با ارزش کلید اصلی برای ردیف مشخص شده است. برای هر بخش داده, تاتر ایجاد یک فایل شاخص است که فروشگاه علامت. برای هر ستون, چه در کلید اصلی است یا نه, خانه رعیتی نیز علامت همان فروشگاه. این علامت به شما اجازه داده پیدا کردن به طور مستقیم در فایل های ستون. - -اندازه گرانول توسط `index_granularity` و `index_granularity_bytes` تنظیمات موتور جدول. تعداد ردیف ها در یک گرانول در `[1, index_granularity]` محدوده, بسته به اندازه ردیف. اندازه گرانول می تواند بیش از `index_granularity_bytes` اگر اندازه یک ردیف بیشتر از ارزش تنظیم است. در این مورد, اندازه گرانول برابر اندازه ردیف. - -## کلید های اولیه و شاخص ها در نمایش داده شد {#primary-keys-and-indexes-in-queries} - -نگاهی به `(CounterID, Date)` کلید اصلی به عنوان مثال. در این مورد, مرتب سازی و شاخص را می توان به شرح زیر نشان داده شده: - - Whole data: [---------------------------------------------] - CounterID: [aaaaaaaaaaaaaaaaaabbbbcdeeeeeeeeeeeeefgggggggghhhhhhhhhiiiiiiiiikllllllll] - Date: [1111111222222233331233211111222222333211111112122222223111112223311122333] - Marks: | | | | | | | | | | | - a,1 a,2 a,3 b,3 e,2 e,3 g,1 h,2 i,1 i,3 l,3 - Marks numbers: 0 1 2 3 4 5 6 7 8 9 10 - -اگر پرسوجوی داده مشخص شود: - -- `CounterID in ('a', 'h')`, سرور بار خوانده شده داده ها در محدوده علامت `[0, 3)` و `[6, 8)`. -- `CounterID IN ('a', 'h') AND Date = 3`, سرور بار خوانده شده داده ها در محدوده علامت `[1, 3)` و `[7, 8)`. -- `Date = 3`, سرور می خواند داده ها در طیف وسیعی از علامت `[1, 10]`. - -نمونه های فوق نشان می دهد که همیشه بیشتر موثر برای استفاده از شاخص از اسکن کامل است. - -شاخص پراکنده اجازه می دهد تا داده های اضافی به عنوان خوانده شده. هنگام خواندن یک طیف وسیعی از کلید اصلی, تا `index_granularity * 2` ردیف اضافی در هر بلوک داده را می توان به عنوان خوانده شده. - -شاخص پراکنده اجازه می دهد شما را به کار با تعداد بسیار زیادی از ردیف جدول, چرا که در اکثر موارد, چنین شاخص در رم کامپیوتر مناسب. - -کلیک یک کلید اصلی منحصر به فرد نیاز ندارد. شما می توانید ردیف های متعدد را با همان کلید اولیه وارد کنید. - -### انتخاب کلید اصلی {#selecting-the-primary-key} - -تعداد ستون ها در کلید اصلی به صراحت محدود نمی شود. بسته به ساختار داده ها, شما می توانید ستون های بیشتر یا کمتر در کلید اصلی شامل. این ممکن است: - -- بهبود عملکرد یک شاخص. - - اگر کلید اصلی است `(a, b)` سپس یک ستون دیگر اضافه کنید `c` عملکرد را بهبود می بخشد اگر شرایط زیر رعایت شود: - - - نمایش داده شد با یک شرط در ستون وجود دارد `c`. - - محدوده داده های طولانی (چندین بار طولانی تر از `index_granularity`) با مقادیر یکسان برای `(a, b)` شایع هستند. به عبارت دیگر, در هنگام اضافه کردن یک ستون دیگر اجازه می دهد تا شما را به جست و خیز محدوده داده بسیار طولانی. - -- بهبود فشرده سازی داده ها. - - خانه را کلیک کنید انواع داده ها توسط کلید اصلی, بنابراین بالاتر از ثبات, بهتر فشرده سازی. - -- فراهم می کند که منطق اضافی در هنگام ادغام قطعات داده در [سقوط غذای اصلی](collapsingmergetree.md#table_engine-collapsingmergetree) و [سامینگمرگتری](summingmergetree.md) موتورها. - - در این مورد منطقی است که مشخص شود *کلید مرتب سازی* که متفاوت از کلید اصلی است. - -یک کلید اولیه طولانی منفی عملکرد درج و مصرف حافظه تاثیر می گذارد, اما ستون های اضافی در کلید اصلی انجام عملکرد تاتر در طول تاثیر نمی گذارد `SELECT` نمایش داده شد. - -### انتخاب کلید اصلی است که متفاوت از کلید مرتب سازی {#choosing-a-primary-key-that-differs-from-the-sorting-key} - -ممکن است که به مشخص کردن یک کلید اولیه (بیان با ارزش هایی که در فایل شاخص برای هر علامت نوشته شده است) که متفاوت از کلید مرتب سازی (بیان برای مرتب سازی ردیف در بخش های داده). در این مورد تاپل عبارت کلیدی اولیه باید یک پیشوند از تاپل عبارت کلیدی مرتب سازی شود. - -این ویژگی در هنگام استفاده از مفید است [سامینگمرگتری](summingmergetree.md) و -[ریزدانه](aggregatingmergetree.md) موتورهای جدول. در یک مورد مشترک در هنگام استفاده از این موتور جدول دو نوع ستون است: *ابعاد* و *اقدامات*. نمایش داده شد نمونه مقادیر کل ستون اندازه گیری با دلخواه `GROUP BY` و فیلتر بر اساس ابعاد. چون SummingMergeTree و AggregatingMergeTree جمع ردیف با همان مقدار از مرتب سازی کلیدی است برای اضافه کردن همه ابعاد آن است. در نتیجه, بیان کلیدی شامل یک لیست طولانی از ستون ها و این لیست باید اغلب با ابعاد تازه اضافه شده به روز. - -در این مورد منطقی است که تنها چند ستون در کلید اصلی را ترک کنید که اسکن های محدوده ای موثر را فراهم می کند و ستون های بعد باقی مانده را به دسته کلید مرتب سازی اضافه می کند. - -[ALTER](../../../sql-reference/statements/alter.md) از کلید مرتب سازی یک عملیات سبک وزن است چرا که زمانی که یک ستون جدید به طور همزمان به جدول و به کلید مرتب سازی اضافه, قطعات داده های موجود لازم نیست به تغییر. از کلید مرتب سازی قدیمی یک پیشوند از کلید مرتب سازی جدید است و هیچ داده در ستون به تازگی اضافه شده وجود دارد, داده ها توسط هر دو کلید مرتب سازی قدیمی و جدید در لحظه اصلاح جدول طبقه بندی شده اند. - -### استفاده از شاخص ها و پارتیشن ها در نمایش داده شد {#use-of-indexes-and-partitions-in-queries} - -برای `SELECT` نمایش داده شد, فاحشه خانه تجزیه و تحلیل اینکه یک شاخص می تواند مورد استفاده قرار گیرد. شاخص می تواند مورد استفاده قرار گیرد در صورتی که `WHERE/PREWHERE` بند بیان (به عنوان یکی از عناصر رابطه یا به طور کامل) است که نشان دهنده برابری یا نابرابری عملیات مقایسه و یا اگر `IN` یا `LIKE` با یک پیشوند ثابت در ستون ها و یا عبارات که در کلید اصلی و یا پارتیشن بندی هستند, و یا در برخی از توابع تا حدی تکراری از این ستون ها, و یا روابط منطقی از این عبارات. - -بدین ترتیب, ممکن است به سرعت اجرا نمایش داده شد در یک یا بسیاری از محدوده کلید اصلی. در این مثال, نمایش داده شد سریع خواهد بود که برای یک تگ ردیابی خاص اجرا, برای یک برچسب خاص و محدوده تاریخ, برای یک تگ و تاریخ خاص, برای برچسب های متعدد با محدوده تاریخ, و غیره. - -بیایید به موتور پیکربندی شده به شرح زیر نگاه کنیم: - - ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate) SETTINGS index_granularity=8192 - -در این مورد در نمایش داده شد: - -``` sql -SELECT count() FROM table WHERE EventDate = toDate(now()) AND CounterID = 34 -SELECT count() FROM table WHERE EventDate = toDate(now()) AND (CounterID = 34 OR CounterID = 42) -SELECT count() FROM table WHERE ((EventDate >= toDate('2014-01-01') AND EventDate <= toDate('2014-01-31')) OR EventDate = toDate('2014-05-01')) AND CounterID IN (101500, 731962, 160656) AND (CounterID = 101500 OR EventDate != toDate('2014-05-01')) -``` - -خانه رعیتی خواهد شاخص کلید اصلی به تر و تمیز داده های نامناسب و کلید پارتیشن بندی ماهانه به تر و تمیز پارتیشن که در محدوده تاریخ نامناسب هستند استفاده کنید. - -نمایش داده شد بالا نشان می دهد که شاخص حتی برای عبارات پیچیده استفاده می شود. خواندن از جدول سازمان یافته است به طوری که با استفاده از شاخص نمی تواند کندتر از اسکن کامل. - -در مثال زیر شاخص نمی تواند مورد استفاده قرار گیرد. - -``` sql -SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%' -``` - -برای بررسی اینکه تاتر می توانید شاخص زمانی که در حال اجرا یک پرس و جو استفاده, استفاده از تنظیمات [اجبار](../../../operations/settings/settings.md#settings-force_index_by_date) و [اجبار](../../../operations/settings/settings.md). - -کلید پارتیشن بندی توسط ماه اجازه می دهد تا خواندن تنها کسانی که بلوک های داده که حاوی تاریخ از محدوده مناسب. در این مورد, بلوک داده ها ممکن است حاوی داده ها برای بسیاری از تاریخ (تا یک ماه کامل). در یک بلوک, داده ها توسط کلید اصلی طبقه بندی شده اند, که ممکن است حاوی تاریخ به عنوان ستون اول نیست. به خاطر همین, با استفاده از یک پرس و جو تنها با یک وضعیت تاریخ که پیشوند کلید اصلی مشخص نیست باعث می شود اطلاعات بیشتر از یک تاریخ به عنوان خوانده شود. - -### استفاده از شاخص برای کلید های اولیه تا حدی یکنواخت {#use-of-index-for-partially-monotonic-primary-keys} - -در نظر بگیرید, مثلا, روز از ماه. یک فرم [توالی یکنواختی](https://en.wikipedia.org/wiki/Monotonic_function) برای یک ماه, اما برای مدت طولانی تر یکنواخت نیست. این یک توالی نیمه یکنواخت است. اگر یک کاربر ایجاد جدول با نیمه یکنواخت کلید اولیه, خانه را ایجاد یک شاخص پراکنده به طور معمول. هنگامی که یک کاربر داده ها را انتخاب از این نوع از جدول, تاتر تجزیه و تحلیل شرایط پرس و جو. اگر کاربر می خواهد برای دریافت اطلاعات بین دو علامت از شاخص و هر دو این علامت در عرض یک ماه سقوط, خانه رعیتی می توانید شاخص در این مورد خاص استفاده کنید زیرا می تواند فاصله بین پارامترهای یک پرس و جو و شاخص محاسبه. - -کلیک خانه می تواند یک شاخص استفاده کنید اگر ارزش های کلید اصلی در محدوده پارامتر پرس و جو یک توالی یکنواخت نشان دهنده نیست. در این مورد, تاتر با استفاده از روش اسکن کامل. - -تاتر با استفاده از این منطق نه تنها برای روز از توالی ماه, اما برای هر کلید اصلی است که نشان دهنده یک توالی نیمه یکنواخت. - -### شاخص های پرش داده (تجربی) {#table_engine-mergetree-data_skipping-indexes} - -اعلامیه شاخص در بخش ستون ها از `CREATE` پرس و جو. - -``` sql -INDEX index_name expr TYPE type(...) GRANULARITY granularity_value -``` - -برای جداول از `*MergeTree` خانواده, شاخص پرش داده را می توان مشخص. - -این شاخص جمع برخی از اطلاعات در مورد بیان مشخص شده بر روی بلوک, که شامل `granularity_value` گرانول (اندازه گرانول با استفاده از `index_granularity` تنظیم در موتور جدول). سپس این دانه ها در استفاده می شود `SELECT` نمایش داده شد برای کاهش مقدار داده ها به خواندن از روی دیسک با پرش بلوک های بزرگ از داده ها که `where` پرس و جو نمی تواند راضی باشد. - -**مثال** - -``` sql -CREATE TABLE table_name -( - u64 UInt64, - i32 Int32, - s String, - ... - INDEX a (u64 * i32, s) TYPE minmax GRANULARITY 3, - INDEX b (u64 * length(s)) TYPE set(1000) GRANULARITY 4 -) ENGINE = MergeTree() -... -``` - -شاخص ها از مثال می توانند توسط کلیک خانه استفاده شوند تا میزان داده ها را برای خواندن از دیسک در موارد زیر کاهش دهند: - -``` sql -SELECT count() FROM table WHERE s < 'z' -SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 -``` - -#### انواع شاخص های موجود {#available-types-of-indices} - -- `minmax` - - فروشگاه افراط و بیان مشخص شده (در صورتی که بیان شده است `tuple` سپس افراط را برای هر عنصر ذخیره می کند `tuple`), با استفاده از اطلاعات ذخیره شده برای پرش بلوک از داده ها مانند کلید اصلی. - -- `set(max_rows)` - - ارزش های منحصر به فرد بیان مشخص شده را ذخیره می کند (بیش از `max_rows` سطرها, `max_rows=0` یعنی “no limits”). با استفاده از مقادیر برای بررسی در صورتی که `WHERE` بیان رضایت بخش در یک بلوک از داده ها نیست. - -- `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` - - فروشگاه ها [فیلتر بلوم](https://en.wikipedia.org/wiki/Bloom_filter) که شامل تمام نمگرام از یک بلوک از داده ها. این نسخهها کار میکند تنها با رشته. می توان برای بهینه سازی استفاده کرد `equals`, `like` و `in` عبارات. - - - `n` — ngram size, - - `size_of_bloom_filter_in_bytes` — Bloom filter size in bytes (you can use large values here, for example, 256 or 512, because it can be compressed well). - - `number_of_hash_functions` — The number of hash functions used in the Bloom filter. - - `random_seed` — The seed for Bloom filter hash functions. - -- `tokenbf_v1(size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` - - همان `ngrambf_v1`, اما فروشگاه نشانه به جای نمرگرام. نشانه ها توالی هایی هستند که توسط شخصیت های غیر عددی جدا شده اند. - -- `bloom_filter([false_positive])` — Stores a [فیلتر بلوم](https://en.wikipedia.org/wiki/Bloom_filter) برای ستون مشخص. - - اختیاری `false_positive` پارامتر احتمال دریافت پاسخ مثبت کاذب از فیلتر است. مقادیر ممکن: (0, 1). مقدار پیش فرض: 0.025. - - انواع داده های پشتیبانی شده: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`, `Array`, `LowCardinality`, `Nullable`. - - توابع زیر می توانند از این استفاده کنند: [برابر](../../../sql-reference/functions/comparison-functions.md), [نقلقولها](../../../sql-reference/functions/comparison-functions.md), [داخل](../../../sql-reference/functions/in-functions.md), [notIn](../../../sql-reference/functions/in-functions.md), [دارد](../../../sql-reference/functions/array-functions.md). - - - -``` sql -INDEX sample_index (u64 * length(s)) TYPE minmax GRANULARITY 4 -INDEX sample_index2 (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100) GRANULARITY 4 -INDEX sample_index3 (lower(str), str) TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 4 -``` - -#### توابع پشتیبانی {#functions-support} - -شرایط در `WHERE` بند شامل تماس از توابع است که با ستون کار. اگر ستون بخشی از یک شاخص است, خانه رعیتی تلاش می کند تا استفاده از این شاخص در هنگام انجام توابع. تاتر از زیر مجموعه های مختلف از توابع برای استفاده از شاخص. - -این `set` شاخص را می توان با تمام توابع استفاده می شود. زیر مجموعه های تابع برای شاخص های دیگر در جدول زیر نشان داده شده است. - -| تابع (اپراتور) / شاخص | کلید اصلی | مینمکس | نمرمبف1 | توکنبف1 | ت_ضعیت | -|----------------------------------------------------------------------------------------------------------------------|-----------|--------|---------|---------|---------| -| [اطلاعات دقیق)](../../../sql-reference/functions/comparison-functions.md#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [نقلقولهای جدید از این نویسنده=, \<\>)](../../../sql-reference/functions/comparison-functions.md#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [مانند](../../../sql-reference/functions/string-search-functions.md#function-like) | ✔ | ✔ | ✔ | ✗ | ✗ | -| [notLike](../../../sql-reference/functions/string-search-functions.md#function-notlike) | ✔ | ✔ | ✔ | ✗ | ✗ | -| [startsWith](../../../sql-reference/functions/string-functions.md#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | -| [endsWith](../../../sql-reference/functions/string-functions.md#endswith) | ✗ | ✗ | ✔ | ✔ | ✗ | -| [چندزبانه](../../../sql-reference/functions/string-search-functions.md#function-multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | -| [داخل](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [notIn](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [کمتر (\<)](../../../sql-reference/functions/comparison-functions.md#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [بیشتر (\>)](../../../sql-reference/functions/comparison-functions.md#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [در حال بارگذاری)](../../../sql-reference/functions/comparison-functions.md#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [اطلاعات دقیق)](../../../sql-reference/functions/comparison-functions.md#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [خالی](../../../sql-reference/functions/array-functions.md#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [notEmpty](../../../sql-reference/functions/array-functions.md#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | -| شتابدهنده | ✗ | ✗ | ✗ | ✔ | ✗ | - -توابع با استدلال ثابت است که کمتر از اندازه نیگرام می تواند توسط استفاده نمی شود `ngrambf_v1` برای بهینه سازی پرس و جو. - -فیلتر بلوم می توانید مسابقات مثبت کاذب دارند, به طوری که `ngrambf_v1`, `tokenbf_v1` و `bloom_filter` شاخص ها نمی توانند برای بهینه سازی پرس و جو هایی که انتظار می رود نتیجه عملکرد نادرست باشد استفاده شوند: - -- می توان بهینه سازی کرد: - - `s LIKE '%test%'` - - `NOT s NOT LIKE '%test%'` - - `s = 1` - - `NOT s != 1` - - `startsWith(s, 'test')` -- نمی توان بهینه سازی کرد: - - `NOT s LIKE '%test%'` - - `s NOT LIKE '%test%'` - - `NOT s = 1` - - `s != 1` - - `NOT startsWith(s, 'test')` - -## دسترسی همزمان داده ها {#concurrent-data-access} - -برای دسترسی به جدول همزمان, ما با استفاده از چند نسخه. به عبارت دیگر, زمانی که یک جدول به طور همزمان خواندن و به روز, داده ها از مجموعه ای از قطعات است که در زمان پرس و جو در حال حاضر به عنوان خوانده شده. هیچ قفل طولانی وجود دارد. درج در راه عملیات خواندن نیست. - -خواندن از یک جدول به طور خودکار موازی. - -## ستون ها و جداول {#table_engine-mergetree-ttl} - -تعیین طول عمر ارزش. - -این `TTL` بند را می توان برای کل جدول و برای هر ستون فردی تنظیم شده است. همچنین منطق حرکت خودکار داده ها بین دیسک ها و حجم ها را مشخص می کند. - -عبارات باید به ارزیابی [تاریخ](../../../sql-reference/data-types/date.md) یا [DateTime](../../../sql-reference/data-types/datetime.md) نوع داده. - -مثال: - -``` sql -TTL time_column -TTL time_column + interval -``` - -برای تعریف `interval` استفاده [فاصله زمانی](../../../sql-reference/operators/index.md#operators-datetime) اپراتورها. - -``` sql -TTL date_time + INTERVAL 1 MONTH -TTL date_time + INTERVAL 15 HOUR -``` - -### ستون {#mergetree-column-ttl} - -هنگامی که مقادیر در ستون منقضی, خانه را جایگزین با مقادیر پیش فرض برای نوع داده ستون. اگر تمام مقادیر ستون در بخش داده منقضی, تاتر حذف این ستون از بخش داده ها در یک سیستم فایل. - -این `TTL` بند را نمی توان برای ستون های کلیدی استفاده کرد. - -مثالها: - -ایجاد یک جدول با تی ال - -``` sql -CREATE TABLE example_table -( - d DateTime, - a Int TTL d + INTERVAL 1 MONTH, - b Int TTL d + INTERVAL 1 MONTH, - c String -) -ENGINE = MergeTree -PARTITION BY toYYYYMM(d) -ORDER BY d; -``` - -اضافه کردن تی ال به یک ستون از یک جدول موجود - -``` sql -ALTER TABLE example_table - MODIFY COLUMN - c String TTL d + INTERVAL 1 DAY; -``` - -تغییر تعداد ستون - -``` sql -ALTER TABLE example_table - MODIFY COLUMN - c String TTL d + INTERVAL 1 MONTH; -``` - -### جدول {#mergetree-table-ttl} - -جدول می تواند بیان برای حذف ردیف منقضی شده و عبارات متعدد برای حرکت خودکار قطعات بین [دیسک یا حجم](#table_engine-mergetree-multiple-volumes). هنگامی که ردیف در جدول منقضی, تاتر حذف تمام ردیف مربوطه. برای قطعات در حال حرکت از ویژگی های, تمام ردیف از یک بخش باید معیارهای بیان جنبش را تامین کند. - -``` sql -TTL expr [DELETE|TO DISK 'aaa'|TO VOLUME 'bbb'], ... -``` - -نوع قانون کنترل هوشمند ممکن است هر عبارت را دنبال کند. این تاثیر می گذارد یک عمل است که باید انجام شود یک بار بیان راضی است (زمان فعلی می رسد): - -- `DELETE` - حذف ردیف منقضی شده (اقدام پیش فرض); -- `TO DISK 'aaa'` - انتقال بخشی به دیسک `aaa`; -- `TO VOLUME 'bbb'` - انتقال بخشی به دیسک `bbb`. - -مثالها: - -ایجاد یک جدول با تی ال - -``` sql -CREATE TABLE example_table -( - d DateTime, - a Int -) -ENGINE = MergeTree -PARTITION BY toYYYYMM(d) -ORDER BY d -TTL d + INTERVAL 1 MONTH [DELETE], - d + INTERVAL 1 WEEK TO VOLUME 'aaa', - d + INTERVAL 2 WEEK TO DISK 'bbb'; -``` - -تغییر تعداد جدول - -``` sql -ALTER TABLE example_table - MODIFY TTL d + INTERVAL 1 DAY; -``` - -**حذف داده ها** - -داده ها با یک حذف شده است زمانی که محل انتخابی ادغام قطعات داده. - -هنگامی که کلیک خانه را ببینید که داده تمام شده است, انجام یک ادغام خارج از برنامه. برای کنترل فرکانس چنین ادغام, شما می توانید مجموعه `merge_with_ttl_timeout`. اگر مقدار خیلی کم است, این بسیاری از ادغام خارج از برنامه است که ممکن است مقدار زیادی از منابع مصرف انجام. - -اگر شما انجام `SELECT` پرس و جو بین ادغام, شما ممکن است داده های منقضی شده. برای جلوگیری از استفاده از [OPTIMIZE](../../../sql-reference/statements/misc.md#misc_operations-optimize) پرسوجو قبل از `SELECT`. - -## با استفاده از دستگاه های بلوک های متعدد برای ذخیره سازی داده ها {#table_engine-mergetree-multiple-volumes} - -### معرفی شرکت {#introduction} - -`MergeTree` موتورهای جدول خانواده می تواند داده ها در دستگاه های بلوک های متعدد ذخیره کنید. مثلا, این می تواند مفید باشد زمانی که داده ها از یک جدول خاص به طور ضمنی به تقسیم “hot” و “cold”. داده های اخیر به طور منظم درخواست شده است اما نیاز به تنها مقدار کمی از فضای. برعکس, داده های تاریخی چربی دم به ندرت درخواست. اگر چندین دیسک در دسترس هستند “hot” داده ها ممکن است بر روی دیسک های سریع واقع (مثلا, اس اس اس اس بلوم و یا در حافظه), در حالی که “cold” داده ها بر روی موارد نسبتا کند (مثلا هارد). - -بخش داده ها حداقل واحد متحرک برای `MergeTree`- جدول موتور . داده های متعلق به یک بخش بر روی یک دیسک ذخیره می شود. قطعات داده را می توان بین دیسک در پس زمینه (با توجه به تنظیمات کاربر) و همچنین با استفاده از نقل مکان کرد [ALTER](../../../sql-reference/statements/alter.md#alter_move-partition) نمایش داده شد. - -### شرایط {#terms} - -- Disk — Block device mounted to the filesystem. -- Default disk — Disk that stores the path specified in the [مسیر](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-path) تنظیم سرور. -- Volume — Ordered set of equal disks (similar to [JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures)). -- Storage policy — Set of volumes and the rules for moving data between them. - -اسامی داده شده به اشخاص توصیف شده را می توان در جداول سیستم یافت می شود, [سیستم.داستان_یابی](../../../operations/system-tables.md#system_tables-storage_policies) و [سیستم.دیسکها](../../../operations/system-tables.md#system_tables-disks). برای اعمال یکی از سیاست های ذخیره سازی پیکربندی شده برای یک جدول از `storage_policy` تنظیم از `MergeTree`- جداول خانواده موتور . - -### پیکربندی {#table_engine-mergetree-multiple-volumes_configure} - -دیسک, حجم و سیاست های ذخیره سازی باید در داخل اعلام `` برچسب یا در فایل اصلی `config.xml` یا در یک فایل مجزا در `config.d` فهرست راهنما. - -ساختار پیکربندی: - -``` xml - - - - /mnt/fast_ssd/clickhouse/ - - - /mnt/hdd1/clickhouse/ - 10485760 - - - /mnt/hdd2/clickhouse/ - 10485760 - - - ... - - - ... - -``` - -برچسبها: - -- `` — Disk name. Names must be different for all disks. -- `path` — path under which a server will store data (`data` و `shadow` پوشه ها) باید با پایان ‘/’. -- `keep_free_space_bytes` — the amount of free disk space to be reserved. - -منظور از تعریف دیسک مهم نیست. - -نشانه گذاری پیکربندی سیاست های ذخیره سازی: - -``` xml - - ... - - - - - disk_name_from_disks_configuration - 1073741824 - - - - - - - 0.2 - - - - - - - - ... - -``` - -برچسبها: - -- `policy_name_N` — Policy name. Policy names must be unique. -- `volume_name_N` — Volume name. Volume names must be unique. -- `disk` — a disk within a volume. -- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume's disks. -- `move_factor` — when the amount of available space gets lower than this factor, data automatically start to move on the next volume if any (by default, 0.1). - -Cofiguration نمونه: - -``` xml - - ... - - - - - disk1 - disk2 - - - - - - - - fast_ssd - 1073741824 - - - disk1 - - - 0.2 - - - ... - -``` - -در مثال داده شده `hdd_in_order` سیاست پیاده سازی [گرد رابین](https://en.wikipedia.org/wiki/Round-robin_scheduling) نزدیک شو بنابراین این سیاست تنها یک جلد را تعریف می کند (`single`), قطعات داده ها بر روی تمام دیسک های خود را به ترتیب دایره ای ذخیره می شود. چنین سیاستی می تواند بسیار مفید اگر چندین دیسک مشابه به سیستم نصب شده وجود دارد, اما حمله پیکربندی نشده است. به خاطر داشته باشید که هر درایو دیسک منحصر به فرد قابل اعتماد نیست و شما ممکن است بخواهید با عامل تکرار 3 یا بیشتر جبران. - -اگر انواع مختلف دیسک های موجود در سیستم وجود دارد, `moving_from_ssd_to_hdd` سیاست را می توان به جای استفاده. حجم `hot` شامل یک دیسک اس اس دی (`fast_ssd`), و حداکثر اندازه یک بخش است که می تواند در این حجم ذخیره شده است 1گیگابایت. تمام قطعات با اندازه بزرگتر از 1 گیگابایت به طور مستقیم در `cold` حجم, که شامل یک دیسک هارد `disk1`. -همچنین هنگامی که دیسک `fast_ssd` می شود توسط بیش از پر 80%, داده خواهد شد به انتقال `disk1` توسط یک فرایند پس زمینه. - -منظور شمارش حجم در یک سیاست ذخیره سازی مهم است. هنگامی که یک حجم پر شده است, داده ها به یک بعدی منتقل. ترتیب شمارش دیسک نیز مهم است زیرا داده ها در نوبت ذخیره می شوند. - -هنگام ایجاد یک جدول می توان یکی از سیاست های ذخیره سازی پیکربندی شده را اعمال کرد: - -``` sql -CREATE TABLE table_with_non_default_policy ( - EventDate Date, - OrderID UInt64, - BannerID UInt64, - SearchPhrase String -) ENGINE = MergeTree -ORDER BY (OrderID, BannerID) -PARTITION BY toYYYYMM(EventDate) -SETTINGS storage_policy = 'moving_from_ssd_to_hdd' -``` - -این `default` سیاست ذخیره سازی نشان میدهد تنها با استفاده از یک حجم, که متشکل از تنها یک دیسک داده شده در ``. هنگامی که یک جدول ایجاد شده است, سیاست ذخیره سازی خود را نمی توان تغییر داد. - -### اطلاعات دقیق {#details} - -در مورد `MergeTree` جداول داده ها به دیسک در راه های مختلف گرفتن است: - -- به عنوان یک نتیجه از درج (`INSERT` پرسوجو). -- در طول پس زمینه ادغام و [جهشها](../../../sql-reference/statements/alter.md#alter-mutations). -- هنگام دانلود از ماکت دیگر. -- به عنوان یک نتیجه از انجماد پارتیشن [ALTER TABLE … FREEZE PARTITION](../../../sql-reference/statements/alter.md#alter_freeze-partition). - -در تمام این موارد به جز جهش و پارتیشن انجماد بخش ذخیره شده در حجم و دیسک با توجه به سیاست ذخیره سازی داده شده است: - -1. جلد اول (به ترتیب تعریف) که فضای دیسک به اندازه کافی برای ذخیره سازی یک بخش (`unreserved_space > current_part_size`) و اجازه می دهد تا برای ذخیره سازی بخش هایی از اندازه داده شده (`max_data_part_size_bytes > current_part_size`) انتخاب شده است . -2. در این حجم, که دیسک انتخاب شده است که به دنبال یکی, که برای ذخیره سازی تکه های قبلی از داده مورد استفاده قرار گرفت, و دارای فضای رایگان بیش از اندازه بخش (`unreserved_space - keep_free_space_bytes > current_part_size`). - -تحت هود جهش و پارتیشن انجماد استفاده از [لینک های سخت](https://en.wikipedia.org/wiki/Hard_link). لینک های سخت بین دیسک های مختلف پشتیبانی نمی شوند بنابراین در چنین مواردی قطعات حاصل شده بر روی دیسک های مشابه به عنوان اولیه ذخیره می شوند. - -در پس زمینه, قطعات بین حجم بر اساس مقدار فضای رایگان نقل مکان کرد (`move_factor` پارامتر) با توجه به سفارش حجم در فایل پیکربندی اعلام کرد. -داده ها هرگز از گذشته و به یکی از اولین منتقل شده است. ممکن است از جداول سیستم استفاده کنید [سیستم._خروج](../../../operations/system-tables.md#system_tables-part-log) (زمینه `type = MOVE_PART`) و [سیستم.قطعات](../../../operations/system-tables.md#system_tables-parts) (فیلدها `path` و `disk`) برای نظارت بر حرکت پس زمینه . همچنین, اطلاعات دقیق را می توان در سیاهههای مربوط به سرور پیدا شده است. - -کاربر می تواند نیروی حرکت بخشی یا پارتیشن از یک حجم به دیگری با استفاده از پرس و جو [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../sql-reference/statements/alter.md#alter_move-partition), تمام محدودیت برای عملیات پس زمینه در نظر گرفته شود. پرس و جو شروع یک حرکت به خودی خود و منتظر نیست برای عملیات پس زمینه به پایان خواهد رسید. کاربر یک پیام خطا اگر فضای رایگان به اندازه کافی در دسترس است و یا اگر هر یک از شرایط مورد نیاز را ملاقات کرد. - -داده های متحرک با تکرار داده ها دخالت نمی کنند. از این رو, سیاست های ذخیره سازی مختلف را می توان برای همان جدول در کپی های مختلف مشخص. - -پس از اتمام ادغام پس زمینه و جهش, قطعات قدیمی تنها پس از یک مقدار مشخصی از زمان حذف (`old_parts_lifetime`). -در طول این زمان به حجم یا دیسک های دیگر منتقل نمی شوند. از این رو, تا زمانی که قطعات در نهایت حذف, هنوز هم به حساب برای ارزیابی فضای دیسک اشغال گرفته. - -[مقاله اصلی](https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/) diff --git a/docs/fa/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/fa/engines/table-engines/mergetree-family/replacingmergetree.md deleted file mode 100644 index 0ace0e05afc..00000000000 --- a/docs/fa/engines/table-engines/mergetree-family/replacingmergetree.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 33 -toc_title: "\u062C\u0627\u06CC\u06AF\u0632\u06CC\u0646\u06CC" ---- - -# جایگزینی {#replacingmergetree} - -موتور متفاوت از [ادغام](mergetree.md#table_engines-mergetree) در که حذف نوشته های تکراری با همان مقدار اصلی کلید (یا دقیق تر, با همان [کلید مرتب سازی](mergetree.md) ارزش). - -تقسیم داده ها تنها در یک ادغام رخ می دهد. ادغام در پس زمینه در زمان ناشناخته رخ می دهد بنابراین شما نمی توانید برنامه ریزی کنید. برخی از داده ها ممکن است بدون پردازش باقی می ماند. اگر چه شما می توانید ادغام برنامه ریزی با استفاده از اجرا `OPTIMIZE` پرس و جو, در استفاده از این حساب نمی, به این دلیل که `OPTIMIZE` پرس و جو خواندن و نوشتن مقدار زیادی از داده ها. - -بدین ترتیب, `ReplacingMergeTree` مناسب برای پاک کردن داده های تکراری در پس زمینه برای صرفه جویی در فضا است اما عدم وجود تکراری را تضمین نمی کند. - -## ایجاد یک جدول {#creating-a-table} - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... -) ENGINE = ReplacingMergeTree([ver]) -[PARTITION BY expr] -[ORDER BY expr] -[PRIMARY KEY expr] -[SAMPLE BY expr] -[SETTINGS name=value, ...] -``` - -برای شرح پارامترهای درخواست را ببینید [درخواست توضیحات](../../../sql-reference/statements/create.md). - -**پارامترهای جایگزین** - -- `ver` — column with version. Type `UInt*`, `Date` یا `DateTime`. پارامتر اختیاری. - - هنگام ادغام, `ReplacingMergeTree` از تمام ردیف ها با همان کلید اصلی تنها یک برگ دارد: - - - گذشته در انتخاب, اگر `ver` تنظیم نشده است. - - با حداکثر نسخه, اگر `ver` مشخص. - -**بندهای پرسوجو** - -هنگام ایجاد یک `ReplacingMergeTree` جدول همان [بند](mergetree.md) در هنگام ایجاد یک مورد نیاز است `MergeTree` جدول - -
- -روش منسوخ برای ایجاد یک جدول - -!!! attention "توجه" - هنوز این روش در پروژه های جدید استفاده کنید و, در صورت امکان, تغییر پروژه های قدیمی به روش بالا توضیح. - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... -) ENGINE [=] ReplacingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, [ver]) -``` - -همه پارامترها به استثنای `ver` همان معنی را در `MergeTree`. - -- `ver` - ستون با نسخه . پارامتر اختیاری. برای شرح, متن بالا را ببینید. - -
- -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/replacingmergetree/) diff --git a/docs/fa/engines/table-engines/mergetree-family/replication.md b/docs/fa/engines/table-engines/mergetree-family/replication.md deleted file mode 100644 index be37a588968..00000000000 --- a/docs/fa/engines/table-engines/mergetree-family/replication.md +++ /dev/null @@ -1,218 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 31 -toc_title: "\u062A\u06A9\u0631\u0627\u0631 \u062F\u0627\u062F\u0647 \u0647\u0627" ---- - -# تکرار داده ها {#table_engines-replication} - -تکرار تنها برای جداول در خانواده ادغام پشتیبانی می شود: - -- تکرار غذای اصلی -- تکرار می شود -- جایگزینی تکراری -- تکرار پلاکتی مگرمرگتری -- تکرار تغییرات -- تکرار مجدد محصول -- تکرار کننده - -تکرار کار می کند در سطح یک جدول فردی, نه کل سرور. سرور می تواند هر دو جدول تکرار و غیر تکرار در همان زمان ذخیره کنید. - -تکرار بستگی ندارد sharding. هر سفال تکرار مستقل خود را دارد. - -داده های فشرده برای `INSERT` و `ALTER` نمایش داده شد تکرار شده است (برای اطلاعات بیشتر, اسناد و مدارک برای دیدن [ALTER](../../../sql-reference/statements/alter.md#query_language_queries_alter)). - -`CREATE`, `DROP`, `ATTACH`, `DETACH` و `RENAME` نمایش داده شد بر روی یک سرور اجرا و تکرار نیست: - -- این `CREATE TABLE` پرس و جو ایجاد یک جدول تکرار جدید بر روی سرور که پرس و جو اجرا شده است. اگر این جدول در حال حاضر بر روی سرور های دیگر وجود دارد, اضافه می کند یک ماکت جدید. -- این `DROP TABLE` پرس و جو حذف ماکت واقع در سرور که پرس و جو اجرا شده است. -- این `RENAME` پرس و جو تغییر نام جدول در یکی از کپی. به عبارت دیگر, جداول تکرار می توانید نام های مختلف در کپی های مختلف دارند. - -استفاده از کلیک [سرویس پرداخت درونبرنامهای پلی](https://zookeeper.apache.org) برای ذخیره سازی اطلاعات متا کپی. استفاده از باغ وحش نسخه 3.4.5 یا جدیدتر. - -برای استفاده از تکرار, پارامترهای مجموعه ای در [باغ وحش](../../../operations/server-configuration-parameters/settings.md#server-settings_zookeeper) بخش پیکربندی سرور. - -!!! attention "توجه" - هنوز تنظیمات امنیتی غفلت نیست. تاتر از `digest` [طرح اکل](https://zookeeper.apache.org/doc/current/zookeeperProgrammers.html#sc_ZooKeeperAccessControl) از زیر سیستم امنیتی باغ وحش. - -به عنوان مثال از تنظیم نشانی های خوشه باغ وحش: - -``` xml - - - example1 - 2181 - - - example2 - 2181 - - - example3 - 2181 - - -``` - -شما می توانید هر خوشه باغ وحش موجود را مشخص کنید و سیستم یک دایرکتوری را برای داده های خود استفاده می کند (دایرکتوری هنگام ایجاد یک جدول تکرار شده مشخص می شود). - -اگر باغ وحش در فایل پیکربندی تنظیم نشده, شما می توانید جداول تکرار ایجاد کنید, و هر جداول تکرار موجود خواهد شد فقط به عنوان خوانده شده. - -باغ وحش در استفاده نمی شود `SELECT` نمایش داده شد به دلیل تکرار می کند عملکرد تاثیر نمی گذارد `SELECT` و نمایش داده شد اجرا فقط به همان سرعتی که برای جداول غیر تکرار انجام می دهند. هنگامی که پرس و جو جداول تکرار توزیع, رفتار کلیک است که توسط تنظیمات کنترل [_شروع مجدد _شروع مجدد _شروع مجدد _کاربری](../../../operations/settings/settings.md#settings-max_replica_delay_for_distributed_queries) و [شناسه بسته:](../../../operations/settings/settings.md#settings-fallback_to_stale_replicas_for_distributed_queries). - -برای هر `INSERT` پرس و جو, حدود ده ورودی از طریق معاملات چند به باغ وحش دار اضافه. (به عبارت دقیق تر, این است که برای هر بلوک قرار داده شده از داده; پرس و جو درج شامل یک بلوک و یا یک بلوک در هر `max_insert_block_size = 1048576` ردیف) این منجر به زمان شروع کمی طولانی تر برای `INSERT` در مقایسه با جداول غیر تکرار. اما اگر شما به دنبال توصیه برای وارد کردن داده ها در دسته بیش از یک `INSERT` در هر ثانیه هیچ مشکلی ایجاد نمی کند. کل خوشه محل کلیک مورد استفاده برای هماهنگی یک خوشه باغ وحش در مجموع چند صد است `INSERTs` در هر ثانیه. توان در درج داده (تعداد ردیف در ثانیه) فقط به عنوان بالا به عنوان داده های غیر تکرار شده است. - -برای خوشه های بسیار بزرگ, شما می توانید خوشه باغ وحش های مختلف برای خرده ریز های مختلف استفاده کنید. با این حال, این لازم در یاندکس ثابت نشده.متریکا خوشه (تقریبا 300 سرور). - -تکرار ناهمزمان و چند استاد است. `INSERT` نمایش داده شد (و همچنین `ALTER`) را می توان به هر سرور در دسترس ارسال می شود. داده ها بر روی سرور قرار می گیرند که پرس و جو اجرا می شود و سپس به سرورهای دیگر کپی می شود. زیرا ناهمگام است, داده به تازگی قرار داده شده در کپی دیگر با برخی از تاخیر به نظر می رسد. اگر بخشی از کپی در دسترس نیست, داده ها نوشته شده است که در دسترس تبدیل. اگر یک ماکت در دسترس است, تاخیر مقدار زمان لازم برای انتقال بلوک از داده های فشرده بر روی شبکه است. - -به طور پیش فرض, پرس و جو درج منتظر تایید نوشتن داده ها از تنها یک ماکت. اگر داده ها با موفقیت به تنها یک ماکت نوشته شده بود و سرور با این ماکت متوقف به وجود, داده های ذخیره شده از دست خواهد رفت. برای فعال کردن گرفتن تایید داده ها می نویسد: از کپی های متعدد با استفاده از `insert_quorum` انتخاب - -هر بلوک از داده ها به صورت اتمی نوشته شده است. پرس و جو درج شده است را به بلوک تا تقسیم `max_insert_block_size = 1048576` ردیف به عبارت دیگر اگر `INSERT` پرس و جو کمتر از 1048576 ردیف, این است که به صورت اتمی ساخته شده. - -بلوک های داده تقسیم می شوند. برای چند می نویسد از بلوک داده های مشابه (بلوک های داده از همان اندازه حاوی ردیف در همان جهت) بلوک تنها یک بار نوشته شده است. دلیل این کار این است که در صورت شکست شبکه زمانی که نرم افزار سرویس گیرنده نمی داند که اگر داده ها به دسی بل نوشته شده بود, بنابراین `INSERT` پرس و جو به سادگی می تواند تکرار شود. مهم نیست که درج ماکت با داده های یکسان فرستاده شد. `INSERTs` ژولیده اند. پارامترهای تقسیم بندی توسط [ادغام](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-merge_tree) تنظیمات سرور. - -در طول تکرار, تنها داده های منبع برای وارد کردن بر روی شبکه منتقل. تحول داده های بیشتر (ادغام) هماهنگ و در تمام کپی در همان راه انجام. این به حداقل می رساند استفاده از شبکه, به این معنی که تکرار خوبی کار می کند زمانی که کپی در مراکز داده های مختلف اقامت. (توجه داشته باشید که تکثیر داده ها در مراکز داده های مختلف هدف اصلی از تکرار است.) - -شما می توانید هر تعداد از کپی از داده های مشابه داشته باشد. یاندکسمتریکا از تکرار دوگانه در تولید استفاده می کند. هر سرور با استفاده از حمله-5 و یا حمله-6, و حمله-10 در برخی موارد. این یک راه حل نسبتا قابل اعتماد و راحت است. - -سیستم نظارت بر هماهنگ سازی داده ها در کپی و قادر به بازیابی پس از شکست است. عدم موفقیت خودکار است (برای تفاوت های کوچک در داده ها) و یا نیمه اتوماتیک (زمانی که داده ها متفاوت بیش از حد, که ممکن است یک خطای پیکربندی نشان می دهد). - -## ایجاد جداول تکرار شده {#creating-replicated-tables} - -این `Replicated` پیشوند به نام موتور جدول اضافه شده است. به عنوان مثال:`ReplicatedMergeTree`. - -**تکرار \* پارامترهای ادغام** - -- `zoo_path` — The path to the table in ZooKeeper. -- `replica_name` — The replica name in ZooKeeper. - -مثال: - -``` sql -CREATE TABLE table_name -( - EventDate DateTime, - CounterID UInt32, - UserID UInt32 -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}') -PARTITION BY toYYYYMM(EventDate) -ORDER BY (CounterID, EventDate, intHash32(UserID)) -SAMPLE BY intHash32(UserID) -``` - -
- -به عنوان مثال در نحو توصیه - -``` sql -CREATE TABLE table_name -( - EventDate DateTime, - CounterID UInt32, - UserID UInt32 -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192) -``` - -
- -به عنوان مثال نشان می دهد, این پارامترها می تواند تعویض در براکت فرفری شامل. مقادیر جایگزین از گرفته ‘macros’ بخش از فایل پیکربندی. مثال: - -``` xml - - 05 - 02 - example05-02-1.yandex.ru - -``` - -مسیر به جدول در باغ وحش باید منحصر به فرد برای هر جدول تکرار شود. جداول در خرده ریز های مختلف باید مسیرهای مختلف داشته باشد. -در این مورد مسیر شامل قسمت های زیر است: - -`/clickhouse/tables/` پیشوند رایج است. ما توصیه می کنیم با استفاده از دقیقا این یکی. - -`{layer}-{shard}` شناسه سفال است. در این مثال شامل دو بخش از یاندکس.متریکا خوشه با استفاده از دو سطح شاردینگ. برای بسیاری از وظایف, شما می توانید فقط جایگزینی {سفال} ترک, خواهد شد که به شناسه سفال گسترش. - -`table_name` نام گره برای جدول در باغ وحش است. این یک ایده خوب را به همان نام جدول است. این است که به صراحت تعریف, چرا که در مقابل به نام جدول, این کار پس از یک پرس و جو تغییر نام نمی. -*HINT*: شما می توانید یک نام پایگاه داده در مقابل اضافه کنید `table_name` همینطور E. g. `db_name.table_name` - -نام ماکت شناسایی کپی های مختلف از همان جدول. شما می توانید نام سرور برای این استفاده, همانطور که در مثال. نام تنها نیاز به منحصر به فرد در هر سفال. - -شما می توانید پارامترهای صراحت به جای استفاده از تعویض تعریف کنیم. این ممکن است مناسب برای تست و برای پیکربندی خوشه های کوچک. با این حال, شما می توانید نمایش داده شد توزیع دی ال استفاده کنید (`ON CLUSTER`) در این مورد. - -هنگام کار با خوشه های بزرگ, توصیه می کنیم با استفاده از تعویض زیرا احتمال خطا را کاهش می دهد. - -اجرای `CREATE TABLE` پرس و جو در هر ماکت. این پرس و جو ایجاد یک جدول تکرار جدید, و یا می افزاید: یک ماکت جدید به یک موجود. - -اگر شما اضافه کردن یک ماکت جدید پس از جدول در حال حاضر شامل برخی از داده ها در کپی های دیگر کپی داده ها از کپی های دیگر کپی به یکی از جدید پس از اجرای پرس و جو. به عبارت دیگر, ماکت جدید خود را با دیگران همگام سازی. - -برای حذف یک ماکت, اجرا `DROP TABLE`. However, only one replica is deleted – the one that resides on the server where you run the query. - -## بازیابی پس از شکست {#recovery-after-failures} - -اگر باغ وحش در دسترس نیست که یک سرور شروع می شود, جداول تکرار تبدیل به حالت فقط خواندنی. این سیستم به صورت دوره ای تلاش برای اتصال به باغ وحش. - -اگر باغ وحش در طول یک در دسترس نیست `INSERT`, یا یک خطا رخ می دهد در هنگام تعامل با باغ وحش, یک استثنا پرتاب می شود. - -پس از اتصال به باغ وحش, سیستم چک چه مجموعه ای از داده ها در سیستم فایل محلی منطبق بر مجموعه مورد انتظار از داده ها (باغ وحش ذخیره این اطلاعات). اگر تناقضات کوچک وجود دارد, سیستم با همگام سازی داده ها با کپی حل. - -اگر سیستم تشخیص داده های شکسته قطعات (با اندازه اشتباه از فایل ها) و یا قطعات ناشناخته (قطعات نوشته شده به فایل سیستم اما ثبت نشده در باغ وحش) این حرکت را به `detached` دایرکتوری فرعی(حذف نمی شوند). هر بخش از دست رفته از کپی کپی کپی کپی. - -توجه داشته باشید که تاتر هیچ اقدامات مخرب مانند به طور خودکار حذف مقدار زیادی از داده ها را انجام نمی دهد. - -هنگامی که سرور شروع می شود (و یا ایجاد یک جلسه جدید با باغ وحش), این تنها چک مقدار و اندازه تمام فایل های. اگر اندازه فایل مطابقت اما بایت در جایی در وسط تغییر یافته است, این بلافاصله شناسایی نشده, اما تنها زمانی که تلاش برای خواندن داده ها برای یک `SELECT` پرس و جو. پرس و جو می اندازد یک استثنا در مورد کنترلی غیر تطبیق و یا اندازه یک بلوک فشرده. در این مورد, قطعات داده ها به صف تایید اضافه شده و کپی از کپی در صورت لزوم. - -اگر مجموعه ای محلی از داده های متفاوت بیش از حد از یک انتظار, یک مکانیزم ایمنی باعث شده است. سرور وارد این در ورود به سیستم و حاضر به راه اندازی. دلیل این کار این است که این مورد ممکن است یک خطای پیکربندی نشان می دهد, مانند اگر یک ماکت در سفال به طور تصادفی مانند یک ماکت در سفال های مختلف پیکربندی شده بود. با این حال, مانع برای این مکانیزم نسبتا کم, و این وضعیت ممکن است در طول بهبود شکست طبیعی رخ می دهد. در این مورد داده ها به صورت نیمه اتوماتیک بازسازی می شوند “pushing a button”. - -برای شروع بازیابی گره ایجاد کنید `/path_to_table/replica_name/flags/force_restore_data` در باغ وحش با هر یک از مطالب, و یا اجرای دستور برای بازگرداندن تمام جداول تکرار: - -``` bash -sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data -``` - -سپس سرور راه اندازی مجدد. در ابتدا سرور این پرچم ها را حذف می کند و شروع به بازیابی می کند. - -## بازیابی پس از از دست دادن اطلاعات کامل {#recovery-after-complete-data-loss} - -اگر تمام داده ها و ابرداده از یکی از سرورها ناپدید شد, این مراحل را برای بازیابی دنبال: - -1. نصب کلیک بر روی سرور. تعریف تعویض به درستی در فایل پیکربندی که شامل شناسه سفال و کپی, در صورت استفاده از. -2. اگر شما تا به حال جداول سه برابر است که باید به صورت دستی بر روی سرور تکرار, کپی اطلاعات خود را از یک ماکت (در دایرکتوری `/var/lib/clickhouse/data/db_name/table_name/`). -3. تعاریف جدول کپی واقع در `/var/lib/clickhouse/metadata/` از یک ماکت. اگر یک شناسه سفال یا ماکت به صراحت در تعاریف جدول تعریف, اصلاح به طوری که به این ماکت مربوط. (متناوبا, شروع سرور و تمام `ATTACH TABLE` نمایش داده شد که باید در شده .در حال بارگذاری `/var/lib/clickhouse/metadata/`.) -4. برای شروع بازیابی, ایجاد گره باغ وحش `/path_to_table/replica_name/flags/force_restore_data` با هر محتوا, و یا اجرای دستور برای بازگرداندن تمام جداول تکرار: `sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data` - -سپس سرور شروع (راه اندازی مجدد, اگر در حال حاضر در حال اجرا). داده خواهد شد از کپی دانلود. - -گزینه بازیابی جایگزین این است که حذف اطلاعات در مورد ماکت از دست رفته از باغ وحش (`/path_to_table/replica_name`), سپس ایجاد ماکت دوباره به عنوان شرح داده شده در “[ایجاد جداول تکرار شده](#creating-replicated-tables)”. - -در طول بازیابی هیچ محدودیتی در پهنای باند شبکه وجود ندارد. این را در ذهن اگر شما در حال بازگرداندن بسیاری از کپی در یک بار. - -## تبدیل از ادغام به تکرار غذای اصلی {#converting-from-mergetree-to-replicatedmergetree} - -ما از اصطلاح استفاده می کنیم `MergeTree` برای اشاره به تمام موتورهای جدول در `MergeTree family`, همان است که برای `ReplicatedMergeTree`. - -اگر شما تا به حال `MergeTree` جدول که به صورت دستی تکرار شد, شما می توانید به یک جدول تکرار تبدیل. شما ممکن است نیاز به انجام این کار اگر شما در حال حاضر مقدار زیادی از داده ها در یک `MergeTree` جدول و در حال حاضر شما می خواهید برای فعال کردن تکرار. - -اگر داده ها در کپی های مختلف متفاوت, برای اولین بار همگام سازی, و یا حذف این داده ها در تمام کپی به جز یکی. - -تغییر نام جدول ادغام موجود, سپس ایجاد یک `ReplicatedMergeTree` جدول با نام های قدیمی. -انتقال داده ها از جدول قدیمی به `detached` دایرکتوری فرعی در داخل دایرکتوری با داده های جدول جدید (`/var/lib/clickhouse/data/db_name/table_name/`). -سپس اجرا کنید `ALTER TABLE ATTACH PARTITION` در یکی از کپی برای اضافه کردن این قطعات داده به مجموعه کار. - -## تبدیل از تکراری به ادغام {#converting-from-replicatedmergetree-to-mergetree} - -ایجاد یک جدول ادغام با نام های مختلف. انتقال تمام داده ها از دایرکتوری با `ReplicatedMergeTree` داده های جدول به دایرکتوری داده جدول جدید. سپس حذف `ReplicatedMergeTree` جدول و راه اندازی مجدد سرور. - -اگر شما می خواهید برای خلاص شدن از شر `ReplicatedMergeTree` جدول بدون راه اندازی سرور: - -- حذف متناظر `.sql` پرونده در فهرست راهنمای فراداده (`/var/lib/clickhouse/metadata/`). -- حذف مسیر مربوطه در باغ وحش (`/path_to_table/replica_name`). - -بعد از این, شما می توانید سرور راه اندازی, ایجاد یک `MergeTree` جدول, انتقال داده ها به دایرکتوری خود, و سپس راه اندازی مجدد سرور. - -## بازیابی هنگامی که ابرداده در خوشه باغ وحش از دست داده و یا صدمه دیده است {#recovery-when-metadata-in-the-zookeeper-cluster-is-lost-or-damaged} - -اگر داده های موجود در باغ وحش از دست رفته یا صدمه دیده بود می توانید داده ها را با حرکت دادن به یک جدول بدون علامت همانطور که در بالا توضیح داده شد ذخیره کنید. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/replication/) diff --git a/docs/fa/engines/table-engines/mergetree-family/summingmergetree.md b/docs/fa/engines/table-engines/mergetree-family/summingmergetree.md deleted file mode 100644 index bb7d87ca3a9..00000000000 --- a/docs/fa/engines/table-engines/mergetree-family/summingmergetree.md +++ /dev/null @@ -1,141 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 34 -toc_title: "\u0633\u0627\u0645\u06CC\u0646\u06AF\u0645\u0631\u06AF\u062A\u0631\u06CC" ---- - -# سامینگمرگتری {#summingmergetree} - -موتور به ارث می برد از [ادغام](mergetree.md#table_engines-mergetree). تفاوت در این است که هنگامی که ادغام قطعات داده برای `SummingMergeTree` جداول تاتر جایگزین تمام ردیف با کلید اصلی همان (یا با دقت بیشتر ,با همان [کلید مرتب سازی](mergetree.md)) با یک ردیف که حاوی مقادیر خلاصه شده برای ستون ها با نوع داده عددی است. اگر کلید مرتب سازی در راه است که یک مقدار کلید تنها مربوط به تعداد زیادی از ردیف تشکیل شده, این به طور قابل توجهی کاهش می دهد حجم ذخیره سازی و سرعت بخشیدن به انتخاب داده ها. - -ما توصیه می کنیم به استفاده از موتور همراه با `MergeTree`. ذخیره اطلاعات کامل در `MergeTree` جدول و استفاده `SummingMergeTree` برای ذخیره سازی داده ها جمع, مثلا, هنگام تهیه گزارش. چنین رویکردی شما را از دست دادن اطلاعات با ارزش با توجه به کلید اولیه نادرست تشکیل شده جلوگیری می کند. - -## ایجاد یک جدول {#creating-a-table} - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... -) ENGINE = SummingMergeTree([columns]) -[PARTITION BY expr] -[ORDER BY expr] -[SAMPLE BY expr] -[SETTINGS name=value, ...] -``` - -برای شرح پارامترهای درخواست را ببینید [درخواست توضیحات](../../../sql-reference/statements/create.md). - -**پارامترهای سامینگمرگتری** - -- `columns` - یک تاپل با نام ستون که ارزش خلاصه خواهد شد. پارامتر اختیاری. - ستون باید از یک نوع عددی باشد و نباید در کلید اصلی باشد. - - اگر `columns` مشخص نشده, تاتر خلاصه مقادیر در تمام ستون ها با یک نوع داده عددی است که در کلید اصلی نیست. - -**بندهای پرسوجو** - -هنگام ایجاد یک `SummingMergeTree` جدول همان [بند](mergetree.md) در هنگام ایجاد یک مورد نیاز است `MergeTree` جدول - -
- -روش منسوخ برای ایجاد یک جدول - -!!! attention "توجه" - هنوز این روش در پروژه های جدید استفاده کنید و, در صورت امکان, تغییر پروژه های قدیمی به روش بالا توضیح. - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... -) ENGINE [=] SummingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, [columns]) -``` - -همه پارامترها به استثنای `columns` همان معنی را در `MergeTree`. - -- `columns` — tuple with names of columns values of which will be summarized. Optional parameter. For a description, see the text above. - -
- -## مثال طریقه استفاده {#usage-example} - -جدول زیر را در نظر بگیرید: - -``` sql -CREATE TABLE summtt -( - key UInt32, - value UInt32 -) -ENGINE = SummingMergeTree() -ORDER BY key -``` - -درج داده به این: - -``` sql -INSERT INTO summtt Values(1,1),(1,2),(2,1) -``` - -تاتر ممکن است تمام ردیف نه به طور کامل جمع ([پایین را ببینید](#data-processing)), بنابراین ما با استفاده از یک تابع کلی `sum` و `GROUP BY` بند در پرس و جو. - -``` sql -SELECT key, sum(value) FROM summtt GROUP BY key -``` - -``` text -┌─key─┬─sum(value)─┐ -│ 2 │ 1 │ -│ 1 │ 3 │ -└─────┴────────────┘ -``` - -## پردازش داده ها {#data-processing} - -هنگامی که داده ها را به یک جدول قرار داده, ذخیره می شوند به عنوان است. خانه رعیتی ادغام بخش قرار داده شده از داده ها به صورت دوره ای و این زمانی است که ردیف با کلید اصلی همان خلاصه و جایگزین با یکی برای هر بخش حاصل از داده ها. - -ClickHouse can merge the data parts so that different resulting parts of data cat consist rows with the same primary key, i.e. the summation will be incomplete. Therefore (`SELECT`) یک تابع جمع [جمع()](../../../sql-reference/aggregate-functions/reference.md#agg_function-sum) و `GROUP BY` بند باید در پرس و جو به عنوان مثال در بالا توضیح داده شده استفاده می شود. - -### قوانین مشترک برای جمع {#common-rules-for-summation} - -مقادیر در ستون با نوع داده عددی خلاصه شده است. مجموعه ای از ستون ها توسط پارامتر تعریف شده است `columns`. - -اگر ارزش شد 0 در تمام ستون ها برای جمع, ردیف حذف شده است. - -اگر ستون در کلید اصلی نیست و خلاصه نشده است, یک مقدار دلخواه از موجود انتخاب. - -مقادیر برای ستون در کلید اصلی خلاصه نشده است. - -### جمعبندی ستونها {#the-summation-in-the-aggregatefunction-columns} - -برای ستون [نوع تابع](../../../sql-reference/data-types/aggregatefunction.md) عمل کلیک به عنوان [ریزدانه](aggregatingmergetree.md) جمع موتور با توجه به عملکرد. - -### ساختارهای تو در تو {#nested-structures} - -جدول می تواند ساختارهای داده تو در تو که در یک راه خاص پردازش کرده اند. - -اگر نام یک جدول تو در تو با به پایان می رسد `Map` و این شامل حداقل دو ستون است که با معیارهای زیر مطابقت دارند: - -- ستون اول عددی است `(*Int*, Date, DateTime)` یا یک رشته `(String, FixedString)` بهش زنگ بزن `key`, -- ستون های دیگر حساب `(*Int*, Float32/64)` بهش زنگ بزن `(values...)`, - -سپس این جدول تو در تو به عنوان یک نقشه برداری از تفسیر `key => (values...)`, و هنگامی که ادغام ردیف خود, عناصر دو مجموعه داده ها با هم ادغام شدند `key` با جمع بندی مربوطه `(values...)`. - -مثالها: - -``` text -[(1, 100)] + [(2, 150)] -> [(1, 100), (2, 150)] -[(1, 100)] + [(1, 150)] -> [(1, 250)] -[(1, 100)] + [(1, 150), (2, 150)] -> [(1, 250), (2, 150)] -[(1, 100), (2, 150)] + [(1, -100)] -> [(2, 150)] -``` - -هنگام درخواست داده ها از [sumMap(key, value)](../../../sql-reference/aggregate-functions/reference.md) تابع برای تجمع `Map`. - -برای ساختار داده های تو در تو, شما لازم نیست که برای مشخص ستون خود را در تاپل ستون برای جمع. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/summingmergetree/) diff --git a/docs/fa/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/fa/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md deleted file mode 100644 index c6a12d0f136..00000000000 --- a/docs/fa/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ /dev/null @@ -1,239 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 37 -toc_title: "\u062F\u0631 \u062D\u0627\u0644 \u0628\u0627\u0631\u06AF\u0630\u0627\u0631\ - \u06CC" ---- - -# در حال بارگذاری {#versionedcollapsingmergetree} - -این موتور: - -- اجازه می دهد تا نوشتن سریع از کشورهای شی که به طور مستمر در حال تغییر. -- حذف کشورهای شی قدیمی در پس زمینه. این به طور قابل توجهی حجم ذخیره سازی را کاهش می دهد. - -بخش را ببینید [سقوط](#table_engines_versionedcollapsingmergetree) برای اطلاعات بیشتر. - -موتور به ارث می برد از [ادغام](mergetree.md#table_engines-mergetree) و می افزاید: منطق برای سقوط ردیف به الگوریتم برای ادغام قطعات داده. `VersionedCollapsingMergeTree` در خدمت همان هدف به عنوان [سقوط غذای اصلی](collapsingmergetree.md) اما با استفاده از یک الگوریتم سقوط های مختلف است که اجازه می دهد تا قرار دادن داده ها در هر جهت با موضوعات متعدد. به خصوص `Version` ستون کمک می کند تا به سقوط ردیف درستی حتی در صورتی که در جهت اشتباه قرار داده شده. در مقابل, `CollapsingMergeTree` اجازه می دهد تا درج تنها به شدت متوالی. - -## ایجاد یک جدول {#creating-a-table} - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... -) ENGINE = VersionedCollapsingMergeTree(sign, version) -[PARTITION BY expr] -[ORDER BY expr] -[SAMPLE BY expr] -[SETTINGS name=value, ...] -``` - -برای شرح پارامترهای پرس و جو, دیدن [توضیحات پرس و جو](../../../sql-reference/statements/create.md). - -**پارامترهای موتور** - -``` sql -VersionedCollapsingMergeTree(sign, version) -``` - -- `sign` — Name of the column with the type of row: `1` یک “state” سطر, `-1` یک “cancel” پارو زدن. - - نوع داده ستون باید باشد `Int8`. - -- `version` — Name of the column with the version of the object state. - - نوع داده ستون باید باشد `UInt*`. - -**بندهای پرسوجو** - -هنگام ایجاد یک `VersionedCollapsingMergeTree` جدول, همان [بند](mergetree.md) در هنگام ایجاد یک مورد نیاز است `MergeTree` جدول - -
- -روش منسوخ برای ایجاد یک جدول - -!!! attention "توجه" - از این روش در پروژه های جدید استفاده نکنید. در صورت امکان, تغییر پروژه های قدیمی به روش بالا توضیح. - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... -) ENGINE [=] VersionedCollapsingMergeTree(date-column [, samp#table_engines_versionedcollapsingmergetreeling_expression], (primary, key), index_granularity, sign, version) -``` - -همه پارامترها به جز `sign` و `version` همان معنی را در `MergeTree`. - -- `sign` — Name of the column with the type of row: `1` یک “state” سطر, `-1` یک “cancel” پارو زدن. - - Column Data Type — `Int8`. - -- `version` — Name of the column with the version of the object state. - - نوع داده ستون باید باشد `UInt*`. - -
- -## سقوط {#table_engines_versionedcollapsingmergetree} - -### داده {#data} - -در نظر بگیرید یک وضعیت که شما نیاز به ذخیره به طور مداوم در حال تغییر داده ها برای برخی از شی. این منطقی است که یک ردیف برای یک شی و به روز رسانی ردیف هر زمان که تغییرات وجود دارد. با این حال, عملیات به روز رسانی گران و کند برای یک سندرم تونل کارپ است چرا که نیاز به بازنویسی داده ها در ذخیره سازی. به روز رسانی قابل قبول نیست اگر شما نیاز به نوشتن داده ها به سرعت, اما شما می توانید تغییرات را به یک شی پی در پی به شرح زیر ارسال. - -استفاده از `Sign` ستون هنگام نوشتن ردیف. اگر `Sign = 1` این بدان معنی است که ردیف دولت از یک شی است (اجازه دهید این تماس “state” ردیف). اگر `Sign = -1` این نشان می دهد لغو دولت از یک شی با ویژگی های مشابه (اجازه دهید این پاسخ “cancel” ردیف). همچنین از `Version` ستون, که باید هر ایالت از یک شی با یک عدد جداگانه شناسایی. - -مثلا, ما می خواهیم برای محاسبه تعداد صفحات کاربران در برخی از سایت بازدید و چه مدت وجود دارد. در برخی از نقطه در زمان ما ارسال ردیف زیر را با دولت از فعالیت های کاربر: - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ -│ 4324182021466249494 │ 5 │ 146 │ 1 │ 1 | -└─────────────────────┴───────────┴──────────┴──────┴─────────┘ -``` - -در برخی موارد بعد ما تغییر فعالیت کاربر را ثبت می کنیم و با دو ردیف زیر می نویسیم. - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ -│ 4324182021466249494 │ 5 │ 146 │ -1 │ 1 | -│ 4324182021466249494 │ 6 │ 185 │ 1 │ 2 | -└─────────────────────┴───────────┴──────────┴──────┴─────────┘ -``` - -ردیف اول لغو حالت قبلی از جسم (کاربر). باید تمام زمینه های دولت لغو شده به جز کپی کنید `Sign`. - -ردیف دوم شامل وضعیت فعلی. - -چرا که ما نیاز به تنها دولت گذشته از فعالیت های کاربر ردیف - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ -│ 4324182021466249494 │ 5 │ 146 │ 1 │ 1 | -│ 4324182021466249494 │ 5 │ 146 │ -1 │ 1 | -└─────────────────────┴───────────┴──────────┴──────┴─────────┘ -``` - -می توان حذف, سقوط نامعتبر (قدیمی) دولت از جسم. `VersionedCollapsingMergeTree` این کار در حالی که ادغام قطعات داده. - -برای پیدا کردن که چرا ما نیاز به دو ردیف برای هر تغییر را ببینید [الگوریتم](#table_engines-versionedcollapsingmergetree-algorithm). - -**نکاتی در مورد استفاده** - -1. برنامه ای که می نویسد داده ها باید به یاد داشته باشید دولت از یک شی به منظور لغو. این “cancel” رشته باید یک کپی از “state” رشته با مخالف `Sign`. این باعث افزایش اندازه اولیه ذخیره سازی اما اجازه می دهد تا به نوشتن داده ها به سرعت. -2. در حال رشد طولانی در ستون کاهش بهره وری از موتور با توجه به بار برای نوشتن. ساده تر داده, بهتر بهره وری. -3. `SELECT` نتایج به شدت بستگی به قوام تاریخ تغییر شی. هنگام تهیه داده ها برای قرار دادن دقیق باشید. شما می توانید نتایج غیر قابل پیش بینی با اطلاعات متناقض از جمله مقادیر منفی برای معیارهای غیر منفی مانند عمق جلسه. - -### الگوریتم {#table_engines-versionedcollapsingmergetree-algorithm} - -هنگامی که مالکیت خانه ادغام قطعات داده, حذف هر جفت ردیف که کلید اولیه و نسخه های مختلف و همان `Sign`. منظور از ردیف مهم نیست. - -هنگامی که داده ها را درج خانه, دستور ردیف توسط کلید اصلی. اگر `Version` ستون در کلید اصلی نیست, خانه عروسکی اضافه می کند به کلید اصلی به طور ضمنی به عنوان زمینه گذشته و برای سفارش استفاده. - -## انتخاب داده ها {#selecting-data} - -تاتر تضمین نمی کند که همه از ردیف با کلید اصلی همان خواهد شد در همان بخش داده و در نتیجه و یا حتی بر روی سرور فیزیکی است. این درست است هر دو برای نوشتن داده ها و برای ادغام بعدی از قطعات داده است. علاوه بر این فرایندهای کلیک `SELECT` نمایش داده شد با موضوعات متعدد و منظور از ردیف در نتیجه نمی تواند پیش بینی کند. این به این معنی است که تجمع مورد نیاز است اگر نیاز به طور کامل وجود دارد “collapsed” داده ها از یک `VersionedCollapsingMergeTree` جدول - -برای نهایی سقوط, ارسال یک پرس و جو با یک `GROUP BY` بند و مجموع توابع است که برای ثبت نام حساب. برای مثال برای محاسبه مقدار استفاده کنید `sum(Sign)` به جای `count()`. برای محاسبه مجموع چیزی استفاده کنید `sum(Sign * x)` به جای `sum(x)` و اضافه کردن `HAVING sum(Sign) > 0`. - -مصالح `count`, `sum` و `avg` می توان محاسبه این راه. مجموع `uniq` می توان محاسبه اگر یک شی حداقل یک دولت غیر فروریخته. مصالح `min` و `max` نمی توان محاسبه کرد زیرا `VersionedCollapsingMergeTree` تاریخ ارزش های کشورهای فرو ریخت را نجات دهد. - -اگر شما نیاز به استخراج داده ها با “collapsing” اما بدون تجمع (مثلا, برای بررسی اینکه ردیف در حال حاضر که جدیدترین ارزش مطابقت شرایط خاصی هستند), شما می توانید با استفاده از `FINAL` تغییردهنده برای `FROM` بند بند. این روش بی فایده است و باید با جداول بزرگ استفاده نمی شود. - -## مثال استفاده {#example-of-use} - -اطلاعات نمونه: - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ -│ 4324182021466249494 │ 5 │ 146 │ 1 │ 1 | -│ 4324182021466249494 │ 5 │ 146 │ -1 │ 1 | -│ 4324182021466249494 │ 6 │ 185 │ 1 │ 2 | -└─────────────────────┴───────────┴──────────┴──────┴─────────┘ -``` - -ایجاد جدول: - -``` sql -CREATE TABLE UAct -( - UserID UInt64, - PageViews UInt8, - Duration UInt8, - Sign Int8, - Version UInt8 -) -ENGINE = VersionedCollapsingMergeTree(Sign, Version) -ORDER BY UserID -``` - -درج داده: - -``` sql -INSERT INTO UAct VALUES (4324182021466249494, 5, 146, 1, 1) -``` - -``` sql -INSERT INTO UAct VALUES (4324182021466249494, 5, 146, -1, 1),(4324182021466249494, 6, 185, 1, 2) -``` - -ما با استفاده از دو `INSERT` نمایش داده شد برای ایجاد دو بخش داده های مختلف. اگر ما داده ها را وارد کنید با یک پرس و جو تنها, تاتر ایجاد یک بخش داده و هرگز هیچ ادغام انجام خواهد داد. - -گرفتن داده ها: - -``` sql -SELECT * FROM UAct -``` - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ -│ 4324182021466249494 │ 5 │ 146 │ 1 │ 1 │ -└─────────────────────┴───────────┴──────────┴──────┴─────────┘ -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ -│ 4324182021466249494 │ 5 │ 146 │ -1 │ 1 │ -│ 4324182021466249494 │ 6 │ 185 │ 1 │ 2 │ -└─────────────────────┴───────────┴──────────┴──────┴─────────┘ -``` - -چه ما در اینجا مشاهده کنید و قطعات فروریخته کجا هستند? -ما دو بخش داده با استفاده از دو `INSERT` نمایش داده شد. این `SELECT` پرس و جو در دو موضوع انجام شد, و در نتیجه یک نظم تصادفی از ردیف است. -سقوط رخ نداد زیرا قطعات داده هنوز ادغام نشده اند. تاتر ادغام قطعات داده در یک نقطه ناشناخته در زمان است که ما نمی توانیم پیش بینی. - -به همین دلیل است که ما نیاز به تجمع: - -``` sql -SELECT - UserID, - sum(PageViews * Sign) AS PageViews, - sum(Duration * Sign) AS Duration, - Version -FROM UAct -GROUP BY UserID, Version -HAVING sum(Sign) > 0 -``` - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Version─┐ -│ 4324182021466249494 │ 6 │ 185 │ 2 │ -└─────────────────────┴───────────┴──────────┴─────────┘ -``` - -اگر ما تجمع نیاز ندارد و می خواهید به زور سقوط, ما می توانیم با استفاده از `FINAL` تغییردهنده برای `FROM` بند بند. - -``` sql -SELECT * FROM UAct FINAL -``` - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ -│ 4324182021466249494 │ 6 │ 185 │ 1 │ 2 │ -└─────────────────────┴───────────┴──────────┴──────┴─────────┘ -``` - -این یک راه بسیار کارامد برای انتخاب داده ها است. برای جداول بزرگ استفاده نکنید. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/versionedcollapsingmergetree/) diff --git a/docs/fa/engines/table-engines/special/buffer.md b/docs/fa/engines/table-engines/special/buffer.md deleted file mode 100644 index e96a963af43..00000000000 --- a/docs/fa/engines/table-engines/special/buffer.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 45 -toc_title: "\u0628\u0627\u0641\u0631" ---- - -# بافر {#buffer} - -بافر داده ها به نوشتن در رم, دوره گرگرفتگی به جدول دیگر. در طول عملیات به عنوان خوانده شده, داده ها از بافر و جدول دیگر به طور همزمان به عنوان خوانده شده. - -``` sql -Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes) -``` - -پارامترهای موتور: - -- `database` – Database name. Instead of the database name, you can use a constant expression that returns a string. -- `table` – Table to flush data to. -- `num_layers` – Parallelism layer. Physically, the table will be represented as `num_layers` از بافر مستقل. مقدار توصیه شده: 16. -- `min_time`, `max_time`, `min_rows`, `max_rows`, `min_bytes` و `max_bytes` – Conditions for flushing data from the buffer. - -داده ها از بافر سرخ و نوشته شده به جدول مقصد اگر همه `min*` شرایط و یا حداقل یک `max*` شرایط ملاقات کرد. - -- `min_time`, `max_time` – Condition for the time in seconds from the moment of the first write to the buffer. -- `min_rows`, `max_rows` – Condition for the number of rows in the buffer. -- `min_bytes`, `max_bytes` – Condition for the number of bytes in the buffer. - -در طول عملیات نوشتن داده ها به یک `num_layers` تعداد بافر تصادفی. یا, اگر بخش داده ها برای وارد کردن به اندازه کافی بزرگ است (بیشتر از `max_rows` یا `max_bytes`), این است که به طور مستقیم به جدول مقصد نوشته شده, حذف بافر. - -شرایط برای گرگرفتگی داده ها به طور جداگانه برای هر یک از محاسبه `num_layers` بافر. برای مثال اگر `num_layers = 16` و `max_bytes = 100000000`, حداکثر مصرف رم است 1.6 گیگابایت. - -مثال: - -``` sql -CREATE TABLE merge.hits_buffer AS merge.hits ENGINE = Buffer(merge, hits, 16, 10, 100, 10000, 1000000, 10000000, 100000000) -``` - -ایجاد یک ‘merge.hits_buffer’ جدول با ساختار مشابه ‘merge.hits’ و با استفاده از موتور بافر. هنگام نوشتن به این جدول, داده ها در رم بافر و بعد به نوشته ‘merge.hits’ جدول 16 بافر ایجاد می کند. اگر 100 ثانیه گذشت یا یک میلیون ردیف نوشته شده یا 100 مگابایت از داده ها نوشته شده است داده ها در هر یک از فوران است; یا اگر به طور همزمان 10 ثانیه گذشت و 10000 ردیف و 10 مگابایت داده ها نوشته شده است. مثلا, اگر فقط یک ردیف نوشته شده است, بعد از 100 ثانیه سرخ خواهد شد, مهم نیست که چه. اما اگر بسیاری از ردیف نوشته شده است, داده خواهد شد هر چه زودتر سرخ. - -هنگامی که سرور متوقف شده است, با جدول قطره و یا جدا جدول, داده های بافر نیز به جدول مقصد سرخ. - -شما می توانید رشته های خالی را در علامت نقل قول واحد برای پایگاه داده و نام جدول تنظیم کنید. این نشان می دهد عدم وجود یک جدول مقصد. در این مورد, زمانی که شرایط خیط و پیت کردن داده رسیده است, بافر است که به سادگی پاک. این ممکن است برای نگه داشتن یک پنجره داده ها در حافظه مفید باشد. - -هنگام خواندن از یک جدول بافر, داده ها هر دو از بافر و از جدول مقصد پردازش (اگر وجود دارد). -توجه داشته باشید که جداول بافر یک شاخص را پشتیبانی نمی کند. به عبارت دیگر, داده ها در بافر به طور کامل اسکن, که ممکن است کند برای بافر بزرگ. (برای داده ها در یک جدول تابع, شاخص است که پشتیبانی استفاده خواهد شد.) - -اگر مجموعه ای از ستون ها در جدول بافر می کند مجموعه ای از ستون ها در یک جدول تابع مطابقت ندارد, یک زیر مجموعه از ستون که در هر دو جدول وجود دارد قرار داده شده است. - -اگر انواع برای یکی از ستون ها در جدول بافر و یک جدول تابع مطابقت ندارد, یک پیام خطا در ورود به سیستم سرور وارد شده و بافر پاک شده است. -همین اتفاق می افتد اگر جدول تابع وجود ندارد زمانی که بافر سرخ است. - -اگر شما نیاز به اجرا را تغییر دهید برای یک جدول تابع و جدول بافر, توصیه می کنیم برای اولین بار حذف جدول بافر, در حال اجرا را تغییر دهید برای جدول تابع, سپس ایجاد جدول بافر دوباره. - -اگر سرور غیر طبیعی راه اندازی مجدد, داده ها در بافر از دست داده است. - -نهایی و نمونه به درستی برای جداول بافر کار نمی کند. این شرایط به جدول مقصد منتقل می شود, اما برای پردازش داده ها در بافر استفاده نمی شود. اگر این ویژگی های مورد نیاز توصیه می کنیم تنها با استفاده از جدول بافر برای نوشتن, در حالی که خواندن از جدول مقصد. - -هنگام اضافه کردن داده ها به یک بافر, یکی از بافر قفل شده است. این باعث تاخیر اگر یک عملیات به عنوان خوانده شده است به طور همزمان از جدول انجام. - -داده هایی که به یک جدول بافر قرار داده شده ممکن است در نهایت در جدول تابع در جهت های مختلف و در بلوک های مختلف. به خاطر همین, یک جدول بافر دشوار است به استفاده از برای نوشتن به یک سقوط به درستی. برای جلوگیری از مشکلات, شما می توانید مجموعه ‘num_layers’ به 1. - -اگر جدول مقصد تکرار شده است, برخی از ویژگی های مورد انتظار از جداول تکرار از دست داده در هنگام نوشتن به یک جدول بافر. تغییرات تصادفی به منظور از سطر و اندازه قطعات داده باعث تقسیم بندی داده ها به ترک کار, به این معنی که ممکن است به یک قابل اعتماد ‘exactly once’ ارسال به جداول تکرار. - -با توجه به این معایب, ما فقط می توانیم با استفاده از یک جدول بافر در موارد نادر توصیه. - -جدول بافر استفاده شده است که بیش از حد بسیاری از درج از تعداد زیادی از سرور بیش از یک واحد از زمان دریافت و داده ها را نمی توان قبل از درج بافر, که به معنی درج می توانید به اندازه کافی سریع اجرا کنید. - -توجه داشته باشید که این کار حس برای وارد کردن داده ها یک ردیف در یک زمان را ندارد, حتی برای جداول بافر. این تنها تولید خواهد شد سرعت چند هزار ردیف در هر ثانیه در حالی که قرار دادن بلوک های بزرگتر از داده ها می تواند تولید بیش از یک میلیون ردیف در هر ثانیه (نگاه کنید به بخش “Performance”). - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/buffer/) diff --git a/docs/fa/engines/table-engines/special/dictionary.md b/docs/fa/engines/table-engines/special/dictionary.md deleted file mode 100644 index f5f9cbd78fc..00000000000 --- a/docs/fa/engines/table-engines/special/dictionary.md +++ /dev/null @@ -1,97 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 35 -toc_title: "\u0648\u0627\u0698\u0647\u0646\u0627\u0645\u0647" ---- - -# واژهنامه {#dictionary} - -این `Dictionary` موتور نمایش [واژهنامه](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) داده ها به عنوان یک جدول کلیک. - -به عنوان مثال, در نظر گرفتن یک فرهنگ لغت از `products` با پیکربندی زیر: - -``` xml - - - products - - -
products
- DSN=some-db-server - - - - 300 - 360 - - - - - - - product_id - - - title - String - - - - - -``` - -پرس و جو داده فرهنگ لغت: - -``` sql -SELECT - name, - type, - key, - attribute.names, - attribute.types, - bytes_allocated, - element_count, - source -FROM system.dictionaries -WHERE name = 'products' -``` - -``` text -┌─name─────┬─type─┬─key────┬─attribute.names─┬─attribute.types─┬─bytes_allocated─┬─element_count─┬─source──────────┐ -│ products │ Flat │ UInt64 │ ['title'] │ ['String'] │ 23065376 │ 175032 │ ODBC: .products │ -└──────────┴──────┴────────┴─────────────────┴─────────────────┴─────────────────┴───────────────┴─────────────────┘ -``` - -شما می توانید از [دیکته کردن\*](../../../sql-reference/functions/ext-dict-functions.md#ext_dict_functions) تابع برای دریافت داده های فرهنگ لغت در این فرمت. - -این دیدگاه مفید نیست که شما نیاز به دریافت داده های خام, و یا در هنگام انجام یک `JOIN` عمل برای این موارد می توانید از `Dictionary` موتور, که نمایش داده فرهنگ لغت در یک جدول. - -نحو: - -``` sql -CREATE TABLE %table_name% (%fields%) engine = Dictionary(%dictionary_name%)` -``` - -مثال طریقه استفاده: - -``` sql -create table products (product_id UInt64, title String) Engine = Dictionary(products); -``` - - Ok - -نگاهی به در چه چیزی در جدول. - -``` sql -select * from products limit 1; -``` - -``` text -┌────product_id─┬─title───────────┐ -│ 152689 │ Some item │ -└───────────────┴─────────────────┘ -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/dictionary/) diff --git a/docs/fa/engines/table-engines/special/distributed.md b/docs/fa/engines/table-engines/special/distributed.md deleted file mode 100644 index 628f2fdd4f6..00000000000 --- a/docs/fa/engines/table-engines/special/distributed.md +++ /dev/null @@ -1,152 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 33 -toc_title: "\u062A\u0648\u0632\u06CC\u0639 \u0634\u062F\u0647" ---- - -# توزیع شده {#distributed} - -**جداول با موتور توزیع شده هیچ اطلاعاتی را توسط خود ذخیره نمی کنند**, اما اجازه می دهد پردازش پرس و جو توزیع شده بر روی سرورهای متعدد. -خواندن به طور خودکار موازی. در طول خواندن, شاخص جدول بر روی سرور از راه دور استفاده می شود, اگر وجود دارد. - -موتور توزیع پارامترها را می پذیرد: - -- نام خوشه در فایل پیکربندی سرور - -- نام یک پایگاه داده از راه دور - -- نام یک میز از راه دور - -- (اختیاری) sharding کلیدی - -- (اختیاری) نام سیاست, استفاده خواهد شد برای ذخیره فایل های موقت برای ارسال کالاهای کابل - - همچنین نگاه کنید به: - - - `insert_distributed_sync` تنظیم - - [ادغام](../mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) برای نمونه - -مثال: - -``` sql -Distributed(logs, default, hits[, sharding_key[, policy_name]]) -``` - -داده ها از تمام سرورها در ‘logs’ خوشه, از پیش فرض.جدول بازدیدها واقع در هر سرور در خوشه. -داده ها نه تنها به عنوان خوانده شده اما تا حدی بر روی سرور از راه دور پردازش (تا حدی که این امکان پذیر است). -مثلا, برای یک پرس و جو با گروه های, داده خواهد شد بر روی سرور از راه دور جمع, و کشورهای متوسط از توابع دانه خواهد شد به سرور درخواست ارسال. سپس داده ها بیشتر جمع خواهد شد. - -به جای نام پایگاه داده, شما می توانید یک عبارت ثابت است که یک رشته را برمی گرداند استفاده. در حال بارگذاری - -logs – The cluster name in the server's config file. - -خوشه ها مانند این تنظیم می شوند: - -``` xml - - - - - 1 - - false - - example01-01-1 - 9000 - - - example01-01-2 - 9000 - - - - 2 - false - - example01-02-1 - 9000 - - - example01-02-2 - 1 - 9440 - - - - -``` - -در اینجا یک خوشه با نام تعریف شده است ‘logs’ که متشکل از دو خرده ریز, که هر کدام شامل دو کپی. -خرده ریز به سرور که شامل بخش های مختلف از داده ها مراجعه (به منظور خواندن تمام داده ها, شما باید تمام خرده ریز دسترسی داشته باشید). -کپی در حال تکثیر سرور (به منظور خواندن تمام داده ها, شما می توانید داده ها بر روی هر یک از کپی دسترسی). - -نام خوشه باید حاوی نقطه نیست. - -پارامترها `host`, `port` و در صورت تمایل `user`, `password`, `secure`, `compression` برای هر سرور مشخص شده است: -- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server doesn't start. If you change the DNS record, restart the server. -- `port` – The TCP port for messenger activity (‘tcp_port’ در پیکربندی, معمولا به مجموعه 9000). نه اشتباه آن را با http_port. -- `user` – Name of the user for connecting to a remote server. Default value: default. This user must have access to connect to the specified server. Access is configured in the users.xml file. For more information, see the section [حقوق دسترسی](../../../operations/access-rights.md). -- `password` – The password for connecting to a remote server (not masked). Default value: empty string. -- `secure` - استفاده از اس اس ال برای اتصال, معمولا شما همچنین باید تعریف `port` = 9440. سرور باید گوش کند `9440` و گواهی صحیح. -- `compression` - استفاده از فشرده سازی داده ها. مقدار پیش فرض: درست. - -When specifying replicas, one of the available replicas will be selected for each of the shards when reading. You can configure the algorithm for load balancing (the preference for which replica to access) – see the [_تبالسازی](../../../operations/settings/settings.md#settings-load_balancing) تنظیمات. -اگر ارتباط با سرور ایجاد نشده است, وجود خواهد داشت تلاش برای ارتباط با یک ایست کوتاه. اگر اتصال شکست خورده, ماکت بعدی انتخاب خواهد شد, و به همین ترتیب برای همه کپی. اگر تلاش اتصال برای تمام کپی شکست خورده, تلاش تکرار خواهد شد به همان شیوه, چندین بار. -این کار به نفع حالت ارتجاعی, اما تحمل گسل کامل را فراهم نمی کند: یک سرور از راه دور ممکن است اتصال قبول, اما ممکن است کار نمی کند, و یا کار ضعیف. - -شما می توانید تنها یکی از خرده ریز مشخص (در این مورد, پردازش پرس و جو باید از راه دور به نام, به جای توزیع) و یا تا هر تعداد از خرده ریز. در هر سفال می توانید از یک به هر تعداد از کپی ها مشخص کنید. شما می توانید تعداد مختلف از کپی برای هر سفال مشخص. - -شما می توانید به عنوان بسیاری از خوشه های مشخص که شما در پیکربندی می خواهید. - -برای مشاهده خوشه های خود استفاده کنید ‘system.clusters’ جدول - -موتور توزیع اجازه می دهد تا کار با یک خوشه مانند یک سرور محلی. با این حال, خوشه غیر قابل اجتنابناپذیری است: شما باید پیکربندی خود را در فایل پیکربندی سرور ارسال (حتی بهتر, برای تمام سرورهای خوشه). - -The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you don't need to create a Distributed table – use the ‘remote’ تابع جدول به جای. بخش را ببینید [توابع جدول](../../../sql-reference/table-functions/index.md). - -دو روش برای نوشتن داده ها به یک خوشه وجود دارد: - -اولین, شما می توانید تعریف که سرور به ارسال که داده ها را به و انجام نوشتن به طور مستقیم در هر سفال. به عبارت دیگر, انجام درج در جداول که جدول توزیع “looks at”. این راه حل انعطاف پذیر ترین است که شما می توانید هر طرح شاردینگ استفاده, که می تواند غیر بدیهی با توجه به الزامات منطقه موضوع. این هم بهینه ترین راه حل از داده ها را می توان به خرده ریز های مختلف نوشته شده است به طور کامل به طور مستقل. - -دومین, شما می توانید درج در یک جدول توزیع انجام. در این مورد جدول توزیع داده های درج شده در سراسر سرور خود را. به منظور ارسال به یک جدول توزیع, باید یک مجموعه کلید شارژ دارند (پارامتر گذشته). علاوه بر این, اگر تنها یک سفال وجود دارد, عملیات نوشتن بدون مشخص کردن کلید شاردینگ کار می کند, چرا که هیچ چیز در این مورد معنی نیست. - -هر سفال می تواند وزن تعریف شده در فایل پیکربندی داشته باشد. به طور پیش فرض, وزن به یک برابر است. داده ها در سراسر خرده ریز در مقدار متناسب با وزن سفال توزیع. مثلا, اگر دو خرده ریز وجود دارد و برای اولین بار دارای وزن 9 در حالی که دوم دارای وزن 10, برای اولین بار ارسال خواهد شد 9 / 19 بخش هایی از ردیف, و دوم ارسال خواهد شد 10 / 19. - -هر سفال می تواند داشته باشد ‘internal_replication’ پارامتر تعریف شده در فایل پیکربندی. - -اگر این پارامتر قرار است به ‘true’ عملیات نوشتن اولین ماکت سالم را انتخاب می کند و داده ها را می نویسد. با استفاده از این جایگزین اگر جدول توزیع شده “looks at” جداول تکرار. به عبارت دیگر اگر جدول ای که داده ها نوشته می شود خود را تکرار می کند. - -اگر قرار است ‘false’ (به طور پیش فرض), داده ها به تمام کپی نوشته شده. در اصل این بدان معنی است که توزیع جدول تکرار داده های خود را. این بدتر از استفاده از جداول تکرار شده است زیرا سازگاری کپی ها بررسی نشده است و در طول زمان حاوی اطلاعات کمی متفاوت خواهد بود. - -برای انتخاب سفال که یک ردیف از داده های فرستاده شده به sharding بیان تجزيه و تحليل است و آن باقی مانده است از تقسیم آن با وزن کلی خرده ریز. ردیف به سفال که مربوط به نیمه فاصله از باقی مانده از ارسال ‘prev_weight’ به ‘prev_weights + weight’ کجا ‘prev_weights’ وزن کل خرده ریز با کمترین تعداد است, و ‘weight’ وزن این سفال است. مثلا, اگر دو خرده ریز وجود دارد, و برای اولین بار دارای یک وزن 9 در حالی که دوم دارای وزن 10, ردیف خواهد شد به سفال اول برای باقی مانده از محدوده ارسال \[0, 9), و دوم برای باقی مانده از محدوده \[9, 19). - -بیان شاردینگ می تواند هر عبارت از ثابت ها و ستون های جدول که یک عدد صحیح را برمی گرداند. برای مثال شما می توانید با استفاده از بیان ‘rand()’ برای توزیع تصادفی داده ها یا ‘UserID’ برای توزیع توسط باقی مانده از تقسیم شناسه کاربر (سپس داده ها از یک کاربر تنها بر روی یک سفال تنها اقامت, که ساده در حال اجرا در و پیوستن به کاربران). اگر یکی از ستون ها به طور مساوی توزیع نشده باشد می توانید در یک تابع هش قرار دهید: اینتاش64 (شناسه). - -یک یادآوری ساده از این بخش محدود است راه حل برای sharding و نیست همیشه مناسب است. این برای حجم متوسط و زیادی از داده ها کار می کند (ده ها تن از سرور), اما نه برای حجم بسیار زیادی از داده ها (صدها سرور یا بیشتر). در مورد دوم با استفاده از sharding طرح های مورد نیاز منطقه موضوع را به جای استفاده از مطالب موجود در توزیع جداول. - -SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don't have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. - -شما باید نگران sharding طرح در موارد زیر: - -- نمایش داده شد استفاده می شود که نیاز به پیوستن به داده ها (در یا پیوستن) توسط یک کلید خاص. اگر داده ها توسط این کلید پنهان, شما می توانید محلی در استفاده و یا پیوستن به جای جهانی در یا جهانی ملحق, که بسیار موثر تر است. -- تعداد زیادی از سرور استفاده شده است (صدها یا بیشتر) با تعداد زیادی از نمایش داده شد کوچک (نمایش داده شد فردی مشتریان - وب سایت, تبلیغ, و یا شرکای). به منظور نمایش داده شد کوچک به کل خوشه تاثیر نمی گذارد, این باعث می شود حس برای قرار دادن داده ها برای یک مشتری در یک سفال تنها. متناوبا, همانطور که ما در یاندکس انجام داده ام.متریکا, شما می توانید راه اندازی دو سطح شاردینگ: تقسیم کل خوشه را به “layers”, جایی که یک لایه ممکن است از تکه های متعدد تشکیل شده است. داده ها برای یک مشتری تنها بر روی یک لایه قرار دارد اما ذرات را می توان به یک لایه در صورت لزوم اضافه کرد و داده ها به طور تصادفی در داخل توزیع می شوند. جداول توزیع شده برای هر لایه ایجاد می شوند و یک جدول توزیع شده مشترک برای نمایش داده شد جهانی ایجاد می شود. - -داده ها ناهمگام نوشته شده است. هنگامی که در جدول قرار داده شده, بلوک داده ها فقط به سیستم فایل های محلی نوشته شده. داده ها به سرور از راه دور در پس زمینه در اسرع وقت ارسال می شود. دوره ارسال داده ها توسط مدیریت [در حال بارگذاری](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) و [در حال بارگذاری](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) تنظیمات. این `Distributed` موتور هر فایل می فرستد با داده های درج شده به طور جداگانه, اما شما می توانید دسته ای از ارسال فایل های با فعال [نمایش سایت](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) تنظیمات. این تنظیم را بهبود می بخشد عملکرد خوشه با استفاده بهتر از سرور محلی و منابع شبکه. شما باید بررسی کنید که داده ها با موفقیت با چک کردن لیست فایل ها (داده ها در حال انتظار برای ارسال) در دایرکتوری جدول ارسال می شود: `/var/lib/clickhouse/data/database/table/`. - -اگر سرور متوقف به وجود داشته باشد و یا راه اندازی مجدد خشن بود (مثلا, پس از یک شکست دستگاه) پس از قرار دادن به یک جدول توزیع, داده های درج شده ممکن است از دست داده. اگر بخشی از داده های خراب شده در دایرکتوری جدول شناسایی شود به ‘broken’ دایرکتوری فرعی و دیگر استفاده می شود. - -پردازش پرس و جو در سراسر تمام کپی در یک سفال واحد موازی است زمانی که گزینه حداکثر_پرورالهراپیلاس فعال است. برای کسب اطلاعات بیشتر به بخش مراجعه کنید [بیشینه_راپرال_راپیکال](../../../operations/settings/settings.md#settings-max_parallel_replicas). - -## ستونهای مجازی {#virtual-columns} - -- `_shard_num` — Contains the `shard_num` (از `system.clusters`). نوع: [UInt32](../../../sql-reference/data-types/int-uint.md). - -!!! note "یادداشت" - از [`remote`](../../../sql-reference/table-functions/remote.md)/`cluster` توابع جدول داخلی ایجاد نمونه موقت از همان توزیع موتور, `_shard_num` در دسترس وجود دارد بیش از حد. - -**همچنین نگاه کنید به** - -- [ستونهای مجازی](index.md#table_engines-virtual_columns) - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/distributed/) diff --git a/docs/fa/engines/table-engines/special/external-data.md b/docs/fa/engines/table-engines/special/external-data.md deleted file mode 100644 index 3c6a1bd69b5..00000000000 --- a/docs/fa/engines/table-engines/special/external-data.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 34 -toc_title: "\u062F\u0627\u062F\u0647\u0647\u0627\u06CC \u062E\u0627\u0631\u062C\u06CC" ---- - -# داده های خارجی برای پردازش پرس و جو {#external-data-for-query-processing} - -تاتر اجازه می دهد تا ارسال یک سرور داده ها که برای پردازش یک پرس و جو مورد نیاز است, همراه با پرس و جو را انتخاب کنید. این داده ها در یک جدول موقت قرار داده (نگاه کنید به بخش “Temporary tables”) و می تواند مورد استفاده قرار گیرد در پرس و جو (برای مثال در اپراتورها). - -مثلا, اگر شما یک فایل متنی با شناسه کاربر مهم, شما می توانید به سرور همراه پرس و جو است که با استفاده از فیلتراسیون توسط این لیست ارسال. - -اگر شما نیاز به اجرای بیش از یک پرس و جو با حجم زیادی از داده های خارجی از این ویژگی استفاده نکنید. بهتر است برای بارگذاری داده ها به دسی بل جلوتر از زمان. - -داده های خارجی را می توان با استفاده از مشتری خط فرمان (در حالت غیر تعاملی) و یا با استفاده از رابط قام ارسال می شود. - -در خط فرمان مشتری شما می توانید مشخص پارامترهای بخش در قالب - -``` bash ---external --file=... [--name=...] [--format=...] [--types=...|--structure=...] -``` - -شما ممکن است بخش های متعدد مثل این, برای تعدادی از جداول در حال انتقال. - -**–external** – Marks the beginning of a clause. -**–file** – Path to the file with the table dump, or -, which refers to stdin. -فقط یک جدول را می توان از استدین بازیابی. - -پارامترهای زیر اختیاری هستند: **–name**– Name of the table. If omitted, _data is used. -**–format** – Data format in the file. If omitted, TabSeparated is used. - -یکی از پارامترهای زیر مورد نیاز است:**–types** – A list of comma-separated column types. For example: `UInt64,String`. The columns will be named _1, _2, … -**–structure**– The table structure in the format`UserID UInt64`, `URL String`. تعریف نام ستون و انواع. - -فایل های مشخص شده در ‘file’ خواهد شد با فرمت مشخص شده در تجزیه ‘format’ با استفاده از انواع داده های مشخص شده در ‘types’ یا ‘structure’. جدول خواهد شد به سرور ارسال شده و در دسترس وجود دارد به عنوان یک جدول موقت با نام در ‘name’. - -مثالها: - -``` bash -$ echo -ne "1\n2\n3\n" | clickhouse-client --query="SELECT count() FROM test.visits WHERE TraficSourceID IN _data" --external --file=- --types=Int8 -849897 -$ cat /etc/passwd | sed 's/:/\t/g' | clickhouse-client --query="SELECT shell, count() AS c FROM passwd GROUP BY shell ORDER BY c DESC" --external --file=- --name=passwd --structure='login String, unused String, uid UInt16, gid UInt16, comment String, home String, shell String' -/bin/sh 20 -/bin/false 5 -/bin/bash 4 -/usr/sbin/nologin 1 -/bin/sync 1 -``` - -هنگام استفاده از رابط اچ تی پی, داده های خارجی در قالب چند/فرم داده به تصویب رسید. هر جدول به عنوان یک فایل جداگانه منتقل می شود. نام جدول از نام فایل گرفته شده است. این ‘query_string’ پارامترهای منتقل می شود ‘name_format’, ‘name_types’ و ‘name_structure’ کجا ‘name’ نام جدول که این پارامترها به مطابقت است. معنای پارامترهای همان است که در هنگام استفاده از مشتری خط فرمان است. - -مثال: - -``` bash -$ cat /etc/passwd | sed 's/:/\t/g' > passwd.tsv - -$ curl -F 'passwd=@passwd.tsv;' 'http://localhost:8123/?query=SELECT+shell,+count()+AS+c+FROM+passwd+GROUP+BY+shell+ORDER+BY+c+DESC&passwd_structure=login+String,+unused+String,+uid+UInt16,+gid+UInt16,+comment+String,+home+String,+shell+String' -/bin/sh 20 -/bin/false 5 -/bin/bash 4 -/usr/sbin/nologin 1 -/bin/sync 1 -``` - -برای پردازش پرس و جو توزیع, جداول موقت به تمام سرور از راه دور ارسال. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/external_data/) diff --git a/docs/fa/engines/table-engines/special/file.md b/docs/fa/engines/table-engines/special/file.md deleted file mode 100644 index 1d3fbc06b38..00000000000 --- a/docs/fa/engines/table-engines/special/file.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 37 -toc_title: "\u067E\u0631\u0648\u0646\u062F\u0647" ---- - -# پرونده {#table_engines-file} - -موتور جدول فایل داده ها را در یک فایل در یکی از پشتیبانی نگه می دارد [پرونده -فرشها](../../../interfaces/formats.md#formats) (تابسپار, بومی, و غیره.). - -نمونه های استفاده: - -- صادرات داده ها از خانه کلیک به فایل. -- تبدیل داده ها از یک فرمت به دیگری. -- به روز رسانی داده ها در تاتر از طریق ویرایش یک فایل بر روی یک دیسک. - -## استفاده در سرور کلیک {#usage-in-clickhouse-server} - -``` sql -File(Format) -``` - -این `Format` پارامتر یکی از فرمت های فایل های موجود را مشخص می کند. برای انجام -`SELECT` نمایش داده شد, فرمت باید برای ورودی پشتیبانی می شود, و به انجام -`INSERT` queries – for output. The available formats are listed in the -[فرشها](../../../interfaces/formats.md#formats) بخش. - -کلیک اجازه نمی دهد مسیر سیستم فایل را مشخص کنید`File`. این پوشه تعریف شده توسط استفاده کنید [مسیر](../../../operations/server-configuration-parameters/settings.md) تنظیم در پیکربندی سرور. - -هنگام ایجاد جدول با استفاده از `File(Format)` این دایرکتوری فرعی خالی در این پوشه ایجاد می کند. هنگامی که داده ها به جدول نوشته شده است, این را به قرار `data.Format` فایل در دایرکتوری فرعی. - -شما می توانید این زیر پوشه و فایل را در فایل سیستم سرور و سپس ایجاد کنید [ATTACH](../../../sql-reference/statements/misc.md) این جدول اطلاعات با نام تطبیق, بنابراین شما می توانید داده ها را از این فایل پرس و جو. - -!!! warning "اخطار" - مراقب باشید با این قابلیت, به دلیل تاتر می کند پیگیری تغییرات خارجی به چنین فایل را حفظ کند. نتیجه همزمان می نویسد: از طریق ClickHouse و خارج از ClickHouse تعریف نشده است. - -**مثال:** - -**1.** تنظیم `file_engine_table` جدول: - -``` sql -CREATE TABLE file_engine_table (name String, value UInt32) ENGINE=File(TabSeparated) -``` - -به طور پیش فرض کلیک خواهد پوشه ایجاد کنید `/var/lib/clickhouse/data/default/file_engine_table`. - -**2.** دستی ایجاد کنید `/var/lib/clickhouse/data/default/file_engine_table/data.TabSeparated` حاوی: - -``` bash -$ cat data.TabSeparated -one 1 -two 2 -``` - -**3.** پرسوجوی داده: - -``` sql -SELECT * FROM file_engine_table -``` - -``` text -┌─name─┬─value─┐ -│ one │ 1 │ -│ two │ 2 │ -└──────┴───────┘ -``` - -## استفاده در کلیک-محلی {#usage-in-clickhouse-local} - -داخل [کلیک-محلی](../../../operations/utilities/clickhouse-local.md) موتور فایل مسیر فایل علاوه بر می پذیرد `Format`. جریان های ورودی / خروجی پیش فرض را می توان با استفاده از نام های عددی یا قابل خواندن توسط انسان مشخص کرد `0` یا `stdin`, `1` یا `stdout`. -**مثال:** - -``` bash -$ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64) ENGINE = File(CSV, stdin); SELECT a, b FROM table; DROP TABLE table" -``` - -## اطلاعات پیاده سازی {#details-of-implementation} - -- چندگانه `SELECT` نمایش داده شد را می توان به صورت همزمان انجام, ولی `INSERT` نمایش داده شد هر یک از دیگر صبر کنید. -- پشتیبانی از ایجاد فایل جدید توسط `INSERT` پرس و جو. -- اگر پرونده وجود داشته باشد, `INSERT` ارزش های جدید را در این برنامه اضافه کنید. -- پشتیبانی نمیشود: - - `ALTER` - - `SELECT ... SAMPLE` - - شاخص ها - - تکرار - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/file/) diff --git a/docs/fa/engines/table-engines/special/generate.md b/docs/fa/engines/table-engines/special/generate.md deleted file mode 100644 index eb42b8b1b09..00000000000 --- a/docs/fa/engines/table-engines/special/generate.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 46 -toc_title: "\u0698\u0646\u0631\u0627\u0644" ---- - -# ژنرال {#table_engines-generate} - -موتور جدول عمومی تولید داده های تصادفی برای طرح جدول داده شده است. - -نمونه های استفاده: - -- استفاده در تست به جمعیت جدول بزرگ تجدید پذیر. -- تولید ورودی تصادفی برای تست ریش ریش شدن. - -## استفاده در سرور کلیک {#usage-in-clickhouse-server} - -``` sql -ENGINE = GenerateRandom(random_seed, max_string_length, max_array_length) -``` - -این `max_array_length` و `max_string_length` پارامترها حداکثر طول همه را مشخص می کنند -ستون ها و رشته های متناوب در داده های تولید شده مطابقت دارند. - -تولید موتور جدول پشتیبانی از تنها `SELECT` نمایش داده شد. - -این پشتیبانی از تمام [انواع داده](../../../sql-reference/data-types/index.md) این را می توان در یک جدول ذخیره کرد به جز `LowCardinality` و `AggregateFunction`. - -**مثال:** - -**1.** تنظیم `generate_engine_table` جدول: - -``` sql -CREATE TABLE generate_engine_table (name String, value UInt32) ENGINE = GenerateRandom(1, 5, 3) -``` - -**2.** پرسوجوی داده: - -``` sql -SELECT * FROM generate_engine_table LIMIT 3 -``` - -``` text -┌─name─┬──────value─┐ -│ c4xJ │ 1412771199 │ -│ r │ 1791099446 │ -│ 7#$ │ 124312908 │ -└──────┴────────────┘ -``` - -## اطلاعات پیاده سازی {#details-of-implementation} - -- پشتیبانی نمیشود: - - `ALTER` - - `SELECT ... SAMPLE` - - `INSERT` - - شاخص ها - - تکرار - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/generate/) diff --git a/docs/fa/engines/table-engines/special/index.md b/docs/fa/engines/table-engines/special/index.md deleted file mode 100644 index 425b84ce062..00000000000 --- a/docs/fa/engines/table-engines/special/index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0648\u06CC\u0698\u0647" -toc_priority: 31 ---- - - diff --git a/docs/fa/engines/table-engines/special/join.md b/docs/fa/engines/table-engines/special/join.md deleted file mode 100644 index b527767eb90..00000000000 --- a/docs/fa/engines/table-engines/special/join.md +++ /dev/null @@ -1,111 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 40 -toc_title: "\u067E\u06CC\u0648\u0633\u062A\u0646" ---- - -# پیوستن {#join} - -ساختار داده تهیه شده برای استفاده در [JOIN](../../../sql-reference/statements/select/join.md#select-join) عملیات. - -## ایجاد یک جدول {#creating-a-table} - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2], -) ENGINE = Join(join_strictness, join_type, k1[, k2, ...]) -``` - -شرح مفصلی از [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) پرس و جو. - -**پارامترهای موتور** - -- `join_strictness` – [پیوستن به سختی](../../../sql-reference/statements/select/join.md#select-join-types). -- `join_type` – [پیوستن به نوع](../../../sql-reference/statements/select/join.md#select-join-types). -- `k1[, k2, ...]` – Key columns from the `USING` بند که `JOIN` عملیات با ساخته شده. - -وارد کردن `join_strictness` و `join_type` پارامترهای بدون نقل قول, مثلا, `Join(ANY, LEFT, col1)`. اونا باید با `JOIN` عملیاتی که جدول خواهد شد برای استفاده. اگر پارامترها مطابقت ندارند, خانه عروسکی می کند یک استثنا پرتاب نمی کند و ممکن است داده های نادرست بازگشت. - -## استفاده از جدول {#table-usage} - -### مثال {#example} - -ایجاد جدول سمت چپ: - -``` sql -CREATE TABLE id_val(`id` UInt32, `val` UInt32) ENGINE = TinyLog -``` - -``` sql -INSERT INTO id_val VALUES (1,11)(2,12)(3,13) -``` - -ایجاد سمت راست `Join` جدول: - -``` sql -CREATE TABLE id_val_join(`id` UInt32, `val` UInt8) ENGINE = Join(ANY, LEFT, id) -``` - -``` sql -INSERT INTO id_val_join VALUES (1,21)(1,22)(3,23) -``` - -پیوستن به جداول: - -``` sql -SELECT * FROM id_val ANY LEFT JOIN id_val_join USING (id) SETTINGS join_use_nulls = 1 -``` - -``` text -┌─id─┬─val─┬─id_val_join.val─┐ -│ 1 │ 11 │ 21 │ -│ 2 │ 12 │ ᴺᵁᴸᴸ │ -│ 3 │ 13 │ 23 │ -└────┴─────┴─────────────────┘ -``` - -به عنوان یک جایگزین, شما می توانید داده ها را از بازیابی `Join` جدول مشخص کردن مقدار پیوستن کلید: - -``` sql -SELECT joinGet('id_val_join', 'val', toUInt32(1)) -``` - -``` text -┌─joinGet('id_val_join', 'val', toUInt32(1))─┐ -│ 21 │ -└────────────────────────────────────────────┘ -``` - -### انتخاب و قرار دادن داده ها {#selecting-and-inserting-data} - -شما می توانید استفاده کنید `INSERT` نمایش داده شد برای اضافه کردن داده ها به `Join`- جدول موتور . اگر جدول با ایجاد شد `ANY` سخت, داده ها برای کلید های تکراری نادیده گرفته می شوند. با `ALL` سخت, تمام ردیف اضافه می شوند. - -شما نمی توانید انجام دهید `SELECT` پرس و جو به طور مستقیم از جدول. بجای, استفاده از یکی از روش های زیر: - -- میز را به سمت راست قرار دهید `JOIN` بند بند. -- تماس با [جوینت](../../../sql-reference/functions/other-functions.md#joinget) تابع, که به شما امکان استخراج داده ها از جدول به همان شیوه به عنوان از یک فرهنگ لغت. - -### محدودیت ها و تنظیمات {#join-limitations-and-settings} - -هنگام ایجاد یک جدول تنظیمات زیر اعمال می شود: - -- [ارزشهای خبری عبارتند از:](../../../operations/settings/settings.md#join_use_nulls) -- [_پاک کردن _روشن گرافیک](../../../operations/settings/query-complexity.md#settings-max_rows_in_join) -- [_پویش همیشگی](../../../operations/settings/query-complexity.md#settings-max_bytes_in_join) -- [_شروع مجدد](../../../operations/settings/query-complexity.md#settings-join_overflow_mode) -- [نمایش سایت](../../../operations/settings/settings.md#settings-join_any_take_last_row) - -این `Join`- جداول موتور نمی تواند مورد استفاده قرار گیرد `GLOBAL JOIN` عملیات. - -این `Join`- موتور اجازه می دهد تا استفاده کنید [ارزشهای خبری عبارتند از:](../../../operations/settings/settings.md#join_use_nulls) تنظیم در `CREATE TABLE` بیانیه. و [SELECT](../../../sql-reference/statements/select/index.md) پرسوجو به کار میرود `join_use_nulls` منم همینطور اگر شما متفاوت است `join_use_nulls` تنظیمات, شما می توانید یک خطا پیوستن به جدول از. این بستگی به نوع پیوستن دارد. هنگام استفاده [جوینت](../../../sql-reference/functions/other-functions.md#joinget) تابع, شما مجبور به استفاده از همان `join_use_nulls` تنظیم در `CRATE TABLE` و `SELECT` اظهارات. - -## ذخیره سازی داده ها {#data-storage} - -`Join` داده های جدول است که همیشه در رم واقع. در هنگام قرار دادن ردیف به یک جدول, کلیکهاوس می نویسد بلوک های داده را به دایرکتوری بر روی دیسک به طوری که می توان ترمیم زمانی که سرور راه اندازی مجدد. - -اگر سرور نادرست راه اندازی مجدد بلوک داده ها بر روی دیسک از دست رفته یا صدمه دیده ممکن است. در این مورد ممکن است لازم باشد فایل را به صورت دستی با داده های خراب شده حذف کنید. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/join/) diff --git a/docs/fa/engines/table-engines/special/materializedview.md b/docs/fa/engines/table-engines/special/materializedview.md deleted file mode 100644 index 01b02766c9d..00000000000 --- a/docs/fa/engines/table-engines/special/materializedview.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 43 -toc_title: "\u0645\u0627\u062F\u0647 \u0628\u06CC\u0646\u06CC" ---- - -# ماده بینی {#materializedview} - -مورد استفاده برای اجرای نمایش محقق (برای اطلاعات بیشتر, دیدن [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query)). برای ذخیره سازی داده ها از یک موتور مختلف استفاده می کند که هنگام ایجاد دیدگاه مشخص شده است. هنگام خواندن از یک جدول, فقط با استفاده از این موتور. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/materializedview/) diff --git a/docs/fa/engines/table-engines/special/memory.md b/docs/fa/engines/table-engines/special/memory.md deleted file mode 100644 index 0fce28af5f7..00000000000 --- a/docs/fa/engines/table-engines/special/memory.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 44 -toc_title: "\u062D\u0627\u0641\u0638\u0647" ---- - -# حافظه {#memory} - -موتور حافظه ذخیره داده ها در رم, در شکل غیر فشرده. داده ها دقیقا به همان شکل ذخیره می شوند که هنگام خواندن دریافت می شود. به عبارت دیگر, خواندن از این جدول کاملا رایگان است. -همزمان دسترسی به داده ها هماهنگ شده است. قفل کوتاه هستند: خواندن و نوشتن عملیات یکدیگر را مسدود نمی کند. -شاخص پشتیبانی نمی شوند. خواندن موازی است. -بهره وری حداکثر (بر فراز 10 گیگابایت/ثانیه) در نمایش داده شد ساده رسیده, چرا که هیچ خواندن از دیسک وجود دارد, از حالت فشرده خارج, و یا کسب اطلاعات. (ما باید توجه داشته باشید که در بسیاری از موارد بهره وری موتور ادغام تقریبا به عنوان بالا است.) -هنگام راه اندازی مجدد یک سرور, داده ها از بین می رود از جدول و جدول خالی می شود. -به طور معمول, با استفاده از این موتور جدول توجیه نیست. اما, این می تواند مورد استفاده قرار گیرد برای تست, و برای کارهایی که حداکثر سرعت مورد نیاز است در تعداد نسبتا کمی از ردیف (تا حدود 100,000,000). - -موتور حافظه توسط سیستم برای جداول موقت با داده های پرس و جو خارجی استفاده می شود (بخش را ببینید “External data for processing a query”) , و برای اجرای جهانی در (نگاه کنید به بخش “IN operators”). - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/memory/) diff --git a/docs/fa/engines/table-engines/special/merge.md b/docs/fa/engines/table-engines/special/merge.md deleted file mode 100644 index e66fdbc2013..00000000000 --- a/docs/fa/engines/table-engines/special/merge.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 36 -toc_title: "\u0627\u062F\u063A\u0627\u0645" ---- - -# ادغام {#merge} - -این `Merge` موتور (با اشتباه گرفته شود `MergeTree`) اطلاعات خود را ذخیره نمی, اما اجازه می دهد تا خواندن از هر تعداد از جداول دیگر به طور همزمان. -خواندن به طور خودکار موازی. نوشتن به یک جدول پشتیبانی نمی شود. هنگام خواندن, شاخص جداول که در واقع در حال خواندن استفاده می شود, در صورتی که وجود داشته باشد. -این `Merge` موتور می پذیرد پارامترهای: نام پایگاه داده و یک عبارت منظم برای جداول. - -مثال: - -``` sql -Merge(hits, '^WatchLog') -``` - -داده خواهد شد از جداول در خواندن `hits` پایگاه داده است که نام هایی که مطابقت با عبارت منظم ‘`^WatchLog`’. - -به جای نام پایگاه داده, شما می توانید یک عبارت ثابت است که یک رشته را برمی گرداند استفاده. به عنوان مثال, `currentDatabase()`. - -Regular expressions — [شماره 2](https://github.com/google/re2) (پشتیبانی از یک زیر مجموعه از مدار چاپی), حساس به حروف. -یادداشت ها در مورد فرار نمادها در عبارات منظم در “match” بخش. - -هنگام انتخاب جداول برای خواندن `Merge` جدول خود را انتخاب نخواهد شد, حتی اگر منطبق عبارت منظم. این است که برای جلوگیری از حلقه. -ممکن است که به ایجاد دو `Merge` جداول که بی وقفه سعی خواهد کرد به خواندن داده های هر یک از دیگران, اما این یک ایده خوب نیست. - -راه معمولی برای استفاده از `Merge` موتور برای کار با تعداد زیادی از `TinyLog` جداول به عنوان اگر با یک جدول واحد. - -مثال 2: - -بیایید می گویند شما باید یک جدول (WatchLog_old) و تصمیم به تغییر پارتیشن بندی بدون حرکت داده ها به یک جدول جدید (WatchLog_new) و شما نیاز به مراجعه به داده ها از هر دو جدول. - -``` sql -CREATE TABLE WatchLog_old(date Date, UserId Int64, EventType String, Cnt UInt64) -ENGINE=MergeTree(date, (UserId, EventType), 8192); -INSERT INTO WatchLog_old VALUES ('2018-01-01', 1, 'hit', 3); - -CREATE TABLE WatchLog_new(date Date, UserId Int64, EventType String, Cnt UInt64) -ENGINE=MergeTree PARTITION BY date ORDER BY (UserId, EventType) SETTINGS index_granularity=8192; -INSERT INTO WatchLog_new VALUES ('2018-01-02', 2, 'hit', 3); - -CREATE TABLE WatchLog as WatchLog_old ENGINE=Merge(currentDatabase(), '^WatchLog'); - -SELECT * -FROM WatchLog -``` - -``` text -┌───────date─┬─UserId─┬─EventType─┬─Cnt─┐ -│ 2018-01-01 │ 1 │ hit │ 3 │ -└────────────┴────────┴───────────┴─────┘ -┌───────date─┬─UserId─┬─EventType─┬─Cnt─┐ -│ 2018-01-02 │ 2 │ hit │ 3 │ -└────────────┴────────┴───────────┴─────┘ -``` - -## ستونهای مجازی {#virtual-columns} - -- `_table` — Contains the name of the table from which data was read. Type: [رشته](../../../sql-reference/data-types/string.md). - - شما می توانید شرایط ثابت را تنظیم کنید `_table` در `WHERE/PREWHERE` بند (به عنوان مثال, `WHERE _table='xyz'`). در این مورد عملیات خواندن فقط برای جداول انجام می شود که شرط است `_table` راضی است, به طوری که `_table` ستون به عنوان یک شاخص عمل می کند. - -**همچنین نگاه کنید به** - -- [ستونهای مجازی](index.md#table_engines-virtual_columns) - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/merge/) diff --git a/docs/fa/engines/table-engines/special/null.md b/docs/fa/engines/table-engines/special/null.md deleted file mode 100644 index 4fcc91ab627..00000000000 --- a/docs/fa/engines/table-engines/special/null.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 38 -toc_title: "\u062E\u0627\u0644\u06CC" ---- - -# خالی {#null} - -هنگام نوشتن به یک جدول تهی, داده نادیده گرفته شده است. هنگام خواندن از یک جدول تهی, پاسخ خالی است. - -با این حال, شما می توانید یک نمایش تحقق در یک جدول تهی ایجاد. بنابراین داده های نوشته شده به جدول در نظر به پایان خواهد رسید. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/null/) diff --git a/docs/fa/engines/table-engines/special/set.md b/docs/fa/engines/table-engines/special/set.md deleted file mode 100644 index 118fe0627ab..00000000000 --- a/docs/fa/engines/table-engines/special/set.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 39 -toc_title: "\u062A\u0646\u0638\u06CC\u0645" ---- - -# تنظیم {#set} - -مجموعه داده است که همیشه در رم. این است که برای استفاده در سمت راست اپراتور در نظر گرفته شده (بخش را ببینید “IN operators”). - -شما می توانید برای وارد کردن داده ها در جدول استفاده کنید. عناصر جدید خواهد شد به مجموعه داده ها اضافه, در حالی که تکراری نادیده گرفته خواهد شد. -اما شما نمی توانید انجام را انتخاب کنید از جدول. تنها راه بازیابی اطلاعات با استفاده از در نیمه راست اپراتور است. - -داده ها همیشه در رم واقع. برای قرار دادن, بلوک از داده های درج شده نیز به دایرکتوری از جداول بر روی دیسک نوشته شده. هنگام شروع سرور, این داده ها به رم لود. به عبارت دیگر, پس از راه اندازی مجدد, داده ها در محل باقی مانده است. - -برای راه اندازی مجدد سرور خشن بلوک داده ها بر روی دیسک ممکن است از دست داده و یا صدمه دیده است. در مورد دوم ممکن است لازم باشد فایل را با داده های خراب شده به صورت دستی حذف کنید. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/set/) diff --git a/docs/fa/engines/table-engines/special/url.md b/docs/fa/engines/table-engines/special/url.md deleted file mode 100644 index e827bc3fbdc..00000000000 --- a/docs/fa/engines/table-engines/special/url.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 41 -toc_title: URL ---- - -# آدرس(URL, قالب) {#table_engines-url} - -مدیریت داده ها بر روی یک سرور کنترل از راه دور قام/قام. این موتور مشابه است -به [پرونده](file.md) موتور - -## با استفاده از موتور در سرور کلیک {#using-the-engine-in-the-clickhouse-server} - -این `format` باید یکی باشد که کلیک خانه می تواند در استفاده از -`SELECT` نمایش داده شد و, در صورت لزوم, به `INSERTs`. برای لیست کامل از فرمت های پشتیبانی شده, دیدن -[فرشها](../../../interfaces/formats.md#formats). - -این `URL` باید به ساختار یاب منابع یکنواخت مطابقت داشته باشد. نشانی وب مشخصشده باید به کارگزار اشاره کند -که با استفاده از قام یا قام. این هیچ نیاز ندارد -هدر اضافی برای گرفتن پاسخ از سرور. - -`INSERT` و `SELECT` نمایش داده شد به تبدیل `POST` و `GET` درخواست ها, -به ترتیب. برای پردازش `POST` درخواست, سرور از راه دور باید پشتیبانی -[کدگذاری انتقال داده شده](https://en.wikipedia.org/wiki/Chunked_transfer_encoding). - -شما می توانید حداکثر تعداد قام را محدود کنید تغییر مسیر هاپ به کواس با استفاده از [عناصر](../../../operations/settings/settings.md#setting-max_http_get_redirects) تنظیمات. - -**مثال:** - -**1.** ایجاد یک `url_engine_table` جدول روی کارگزار : - -``` sql -CREATE TABLE url_engine_table (word String, value UInt64) -ENGINE=URL('http://127.0.0.1:12345/', CSV) -``` - -**2.** ایجاد یک سرور اساسی قام با استفاده از پایتون استاندارد 3 ابزار و -شروع کن: - -``` python3 -from http.server import BaseHTTPRequestHandler, HTTPServer - -class CSVHTTPServer(BaseHTTPRequestHandler): - def do_GET(self): - self.send_response(200) - self.send_header('Content-type', 'text/csv') - self.end_headers() - - self.wfile.write(bytes('Hello,1\nWorld,2\n', "utf-8")) - -if __name__ == "__main__": - server_address = ('127.0.0.1', 12345) - HTTPServer(server_address, CSVHTTPServer).serve_forever() -``` - -``` bash -$ python3 server.py -``` - -**3.** درخواست اطلاعات: - -``` sql -SELECT * FROM url_engine_table -``` - -``` text -┌─word──┬─value─┐ -│ Hello │ 1 │ -│ World │ 2 │ -└───────┴───────┘ -``` - -## اطلاعات پیاده سازی {#details-of-implementation} - -- می خواند و می نویسد می تواند موازی -- پشتیبانی نمیشود: - - `ALTER` و `SELECT...SAMPLE` عملیات. - - شاخص. - - تکرار. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/url/) diff --git a/docs/fa/engines/table-engines/special/view.md b/docs/fa/engines/table-engines/special/view.md deleted file mode 100644 index f5be393db26..00000000000 --- a/docs/fa/engines/table-engines/special/view.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 42 -toc_title: "\u0646\u0645\u0627" ---- - -# نما {#table_engines-view} - -مورد استفاده برای اجرای نمایش (برای اطلاعات بیشتر, دیدن `CREATE VIEW query`). این کار داده ذخیره نمی, اما تنها فروشگاه مشخص `SELECT` پرس و جو. هنگام خواندن از یک جدول, اجرا می شود این پرس و جو (و حذف تمام ستون های غیر ضروری از پرس و جو). - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/table_engines/view/) diff --git a/docs/fa/faq/general.md b/docs/fa/faq/general.md deleted file mode 100644 index 91870614d3f..00000000000 --- a/docs/fa/faq/general.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 78 -toc_title: "\u0633\u0648\u0627\u0644\u0627\u062A \u0639\u0645\u0648\u0645\u06CC" ---- - -# سوالات عمومی {#general-questions} - -## چرا چیزی شبیه نگاشتکاهش استفاده نمی? {#why-not-use-something-like-mapreduce} - -ما می توانیم به سیستم هایی مانند نگاشتکاهش به عنوان سیستم های محاسبات توزیع شده اشاره کنیم که عملیات کاهش بر اساس مرتب سازی توزیع شده است. شایع ترین راه حل منبع باز در این کلاس است [Apache Hadoop](http://hadoop.apache.org). یاندکس از راه حل داخلی خود استفاده می کند. - -این سیستم ها به دلیل زمان تاخیر بالا برای نمایش داده شد اینترنتی مناسب نیست. به عبارت دیگر نمی توانند به عنوان یک رابط وب به پایان برسند. این نوع سیستم ها برای به روز رسانی داده های زمان واقعی مفید نیستند. مرتب سازی توزیع شده بهترین راه برای انجام عملیات کاهش نیست اگر نتیجه عملیات و تمام نتایج متوسط (اگر وجود داشته باشد) در رم یک سرور قرار دارد که معمولا مورد نمایش داده شد اینترنتی است. در چنین حالتی یک جدول هش یک راه بهینه برای کاهش عملیات است. یک رویکرد مشترک برای بهینه سازی نقشه کاهش وظایف قبل از تجمع (بخشی کاهش) با استفاده از یک جدول هش در رم است. کاربر این بهینه سازی را به صورت دستی انجام می دهد. مرتب سازی توزیع شده یکی از علل اصلی کاهش عملکرد در هنگام اجرای نقشه ساده است-کاهش وظایف. - -اکثر پیاده سازی نگاشتکاهش به شما اجازه اجرای کد دلخواه در یک خوشه. اما یک زبان پرس و جو اعلانی بهتر است به تاباندن لیزر به اجرا تجربه به سرعت مناسب است. مثلا, هادوپ است کندو و خوک. همچنین در نظر ابردرا ایمپالا یا کوسه (منسوخ شده) برای جرقه, و همچنین شمع جرقه, تند, و مته درد مقعد. عملکرد هنگامی که در حال اجرا از جمله وظایف بسیار زیر بهینه در مقایسه با سیستم های تخصصی, اما زمان تاخیر نسبتا بالا باعث می شود غیر واقعی برای استفاده از این سیستم به عنوان باطن برای یک رابط وب. - -## اگر من یک مشکل با کدگذاریها در هنگام استفاده از اوراکل از طریق ان بی سی دارند? {#oracle-odbc-encodings} - -اگر شما استفاده از اوراکل از طریق راننده او بی سی به عنوان یک منبع از لغت نامه های خارجی, شما نیاز به تنظیم مقدار صحیح برای `NLS_LANG` متغیر محیطی در `/etc/default/clickhouse`. برای کسب اطلاعات بیشتر, دیدن [اوراکل NLS_LANG پرسش و پاسخ](https://www.oracle.com/technetwork/products/globalization/nls-lang-099431.html). - -**مثال** - -``` sql -NLS_LANG=RUSSIAN_RUSSIA.UTF8 -``` - -## چگونه می توانم صادرات داده ها از خانه رعیتی به یک فایل? {#how-to-export-to-file} - -### با استفاده از به OUTFILE بند {#using-into-outfile-clause} - -افزودن یک [INTO OUTFILE](../sql-reference/statements/select/into-outfile.md#into-outfile-clause) بند به درخواست شما. - -به عنوان مثال: - -``` sql -SELECT * FROM table INTO OUTFILE 'file' -``` - -به طور پیش فرض, تاتر با استفاده از [جدول دار](../interfaces/formats.md#tabseparated) فرمت برای داده های خروجی. برای انتخاب [قالب داده](../interfaces/formats.md), استفاده از [بند فرمت](../sql-reference/statements/select/format.md#format-clause). - -به عنوان مثال: - -``` sql -SELECT * FROM table INTO OUTFILE 'file' FORMAT CSV -``` - -### با استفاده از جدول فایل موتور {#using-a-file-engine-table} - -ببینید [پرونده](../engines/table-engines/special/file.md). - -### با استفاده از تغییر مسیر خط فرمان {#using-command-line-redirection} - -``` sql -$ clickhouse-client --query "SELECT * from table" --format FormatName > result.txt -``` - -ببینید [کلیک مشتری](../interfaces/cli.md). - -{## [مقاله اصلی](https://clickhouse.tech/docs/en/faq/general/) ##} diff --git a/docs/fa/faq/index.md b/docs/fa/faq/index.md deleted file mode 100644 index a44dbb31e89..00000000000 --- a/docs/fa/faq/index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: F.A.Q. -toc_priority: 76 ---- - - diff --git a/docs/fa/getting-started/example-datasets/amplab-benchmark.md b/docs/fa/getting-started/example-datasets/amplab-benchmark.md deleted file mode 100644 index 593a3a2d669..00000000000 --- a/docs/fa/getting-started/example-datasets/amplab-benchmark.md +++ /dev/null @@ -1,131 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 17 -toc_title: "\u0645\u0639\u06CC\u0627\u0631 \u0628\u0632\u0631\u06AF \u062F\u0627\u062F\ - \u0647 \u0647\u0627\u06CC \u062A\u0642\u0648\u06CC\u062A \u06A9\u0646\u0646\u062F\ - \u0647" ---- - -# معیار بزرگ داده های تقویت کننده {#amplab-big-data-benchmark} - -ببینید https://amplab.cs.berkeley.edu/benchmark/ - -ثبت نام برای یک حساب کاربری رایگان در https://aws.amazon.com. این نیاز به یک کارت اعتباری, پست الکترونیک, و شماره تلفن. یک کلید دسترسی جدید در https://console.aws.amazon.com/iam/home?nc2=h_m_sc#security_credential - -زیر را در کنسول اجرا کنید: - -``` bash -$ sudo apt-get install s3cmd -$ mkdir tiny; cd tiny; -$ s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/tiny/ . -$ cd .. -$ mkdir 1node; cd 1node; -$ s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/1node/ . -$ cd .. -$ mkdir 5nodes; cd 5nodes; -$ s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/5nodes/ . -$ cd .. -``` - -اجرای نمایش داده شد زیر کلیک: - -``` sql -CREATE TABLE rankings_tiny -( - pageURL String, - pageRank UInt32, - avgDuration UInt32 -) ENGINE = Log; - -CREATE TABLE uservisits_tiny -( - sourceIP String, - destinationURL String, - visitDate Date, - adRevenue Float32, - UserAgent String, - cCode FixedString(3), - lCode FixedString(6), - searchWord String, - duration UInt32 -) ENGINE = MergeTree(visitDate, visitDate, 8192); - -CREATE TABLE rankings_1node -( - pageURL String, - pageRank UInt32, - avgDuration UInt32 -) ENGINE = Log; - -CREATE TABLE uservisits_1node -( - sourceIP String, - destinationURL String, - visitDate Date, - adRevenue Float32, - UserAgent String, - cCode FixedString(3), - lCode FixedString(6), - searchWord String, - duration UInt32 -) ENGINE = MergeTree(visitDate, visitDate, 8192); - -CREATE TABLE rankings_5nodes_on_single -( - pageURL String, - pageRank UInt32, - avgDuration UInt32 -) ENGINE = Log; - -CREATE TABLE uservisits_5nodes_on_single -( - sourceIP String, - destinationURL String, - visitDate Date, - adRevenue Float32, - UserAgent String, - cCode FixedString(3), - lCode FixedString(6), - searchWord String, - duration UInt32 -) ENGINE = MergeTree(visitDate, visitDate, 8192); -``` - -بازگشت به کنسول: - -``` bash -$ for i in tiny/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_tiny FORMAT CSV"; done -$ for i in tiny/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_tiny FORMAT CSV"; done -$ for i in 1node/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_1node FORMAT CSV"; done -$ for i in 1node/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_1node FORMAT CSV"; done -$ for i in 5nodes/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_5nodes_on_single FORMAT CSV"; done -$ for i in 5nodes/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_5nodes_on_single FORMAT CSV"; done -``` - -نمایش داده شد برای اخذ نمونه داده ها: - -``` sql -SELECT pageURL, pageRank FROM rankings_1node WHERE pageRank > 1000 - -SELECT substring(sourceIP, 1, 8), sum(adRevenue) FROM uservisits_1node GROUP BY substring(sourceIP, 1, 8) - -SELECT - sourceIP, - sum(adRevenue) AS totalRevenue, - avg(pageRank) AS pageRank -FROM rankings_1node ALL INNER JOIN -( - SELECT - sourceIP, - destinationURL AS pageURL, - adRevenue - FROM uservisits_1node - WHERE (visitDate > '1980-01-01') AND (visitDate < '1980-04-01') -) USING pageURL -GROUP BY sourceIP -ORDER BY totalRevenue DESC -LIMIT 1 -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/getting_started/example_datasets/amplab_benchmark/) diff --git a/docs/fa/getting-started/example-datasets/criteo.md b/docs/fa/getting-started/example-datasets/criteo.md deleted file mode 100644 index 74ca9fa3bd5..00000000000 --- a/docs/fa/getting-started/example-datasets/criteo.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 19 -toc_title: "\u062A\u0631\u0627\u0628\u0627\u06CC\u062A \u06A9\u0644\u06CC\u06A9 \u0633\ - \u06CC\u0627\u0647\u0647\u0647\u0627\u06CC \u0645\u0631\u0628\u0648\u0637 \u0627\ - \u0632 \u0645\u062E\u0644\u0648\u0642" ---- - -# ترابایت کلیک سیاهههای مربوط از مخلوق {#terabyte-of-click-logs-from-criteo} - -دانلود داده ها از http://labs.criteo.com/downloads/download-terabyte-click-logs/ - -ایجاد یک جدول برای وارد کردن ورود به سیستم: - -``` sql -CREATE TABLE criteo_log (date Date, clicked UInt8, int1 Int32, int2 Int32, int3 Int32, int4 Int32, int5 Int32, int6 Int32, int7 Int32, int8 Int32, int9 Int32, int10 Int32, int11 Int32, int12 Int32, int13 Int32, cat1 String, cat2 String, cat3 String, cat4 String, cat5 String, cat6 String, cat7 String, cat8 String, cat9 String, cat10 String, cat11 String, cat12 String, cat13 String, cat14 String, cat15 String, cat16 String, cat17 String, cat18 String, cat19 String, cat20 String, cat21 String, cat22 String, cat23 String, cat24 String, cat25 String, cat26 String) ENGINE = Log -``` - -داده ها را دانلود کنید: - -``` bash -$ for i in {00..23}; do echo $i; zcat datasets/criteo/day_${i#0}.gz | sed -r 's/^/2000-01-'${i/00/24}'\t/' | clickhouse-client --host=example-perftest01j --query="INSERT INTO criteo_log FORMAT TabSeparated"; done -``` - -ایجاد یک جدول برای داده های تبدیل شده: - -``` sql -CREATE TABLE criteo -( - date Date, - clicked UInt8, - int1 Int32, - int2 Int32, - int3 Int32, - int4 Int32, - int5 Int32, - int6 Int32, - int7 Int32, - int8 Int32, - int9 Int32, - int10 Int32, - int11 Int32, - int12 Int32, - int13 Int32, - icat1 UInt32, - icat2 UInt32, - icat3 UInt32, - icat4 UInt32, - icat5 UInt32, - icat6 UInt32, - icat7 UInt32, - icat8 UInt32, - icat9 UInt32, - icat10 UInt32, - icat11 UInt32, - icat12 UInt32, - icat13 UInt32, - icat14 UInt32, - icat15 UInt32, - icat16 UInt32, - icat17 UInt32, - icat18 UInt32, - icat19 UInt32, - icat20 UInt32, - icat21 UInt32, - icat22 UInt32, - icat23 UInt32, - icat24 UInt32, - icat25 UInt32, - icat26 UInt32 -) ENGINE = MergeTree(date, intHash32(icat1), (date, intHash32(icat1)), 8192) -``` - -داده ها را از ورود خام تغییر دهید و در جدول دوم قرار دهید: - -``` sql -INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int7, int8, int9, int10, int11, int12, int13, reinterpretAsUInt32(unhex(cat1)) AS icat1, reinterpretAsUInt32(unhex(cat2)) AS icat2, reinterpretAsUInt32(unhex(cat3)) AS icat3, reinterpretAsUInt32(unhex(cat4)) AS icat4, reinterpretAsUInt32(unhex(cat5)) AS icat5, reinterpretAsUInt32(unhex(cat6)) AS icat6, reinterpretAsUInt32(unhex(cat7)) AS icat7, reinterpretAsUInt32(unhex(cat8)) AS icat8, reinterpretAsUInt32(unhex(cat9)) AS icat9, reinterpretAsUInt32(unhex(cat10)) AS icat10, reinterpretAsUInt32(unhex(cat11)) AS icat11, reinterpretAsUInt32(unhex(cat12)) AS icat12, reinterpretAsUInt32(unhex(cat13)) AS icat13, reinterpretAsUInt32(unhex(cat14)) AS icat14, reinterpretAsUInt32(unhex(cat15)) AS icat15, reinterpretAsUInt32(unhex(cat16)) AS icat16, reinterpretAsUInt32(unhex(cat17)) AS icat17, reinterpretAsUInt32(unhex(cat18)) AS icat18, reinterpretAsUInt32(unhex(cat19)) AS icat19, reinterpretAsUInt32(unhex(cat20)) AS icat20, reinterpretAsUInt32(unhex(cat21)) AS icat21, reinterpretAsUInt32(unhex(cat22)) AS icat22, reinterpretAsUInt32(unhex(cat23)) AS icat23, reinterpretAsUInt32(unhex(cat24)) AS icat24, reinterpretAsUInt32(unhex(cat25)) AS icat25, reinterpretAsUInt32(unhex(cat26)) AS icat26 FROM criteo_log; - -DROP TABLE criteo_log; -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/getting_started/example_datasets/criteo/) diff --git a/docs/fa/getting-started/example-datasets/index.md b/docs/fa/getting-started/example-datasets/index.md deleted file mode 100644 index 8b72c0d04fc..00000000000 --- a/docs/fa/getting-started/example-datasets/index.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0628\u0647 \u0639\u0646\u0648\u0627\u0646 \u0645\u062B\u0627\u0644\ - \ \u0645\u062C\u0645\u0648\u0639\u0647 \u062F\u0627\u062F\u0647" -toc_priority: 12 -toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" ---- - -# به عنوان مثال مجموعه داده {#example-datasets} - -در این بخش چگونگی اخذ مجموعه داده ها به عنوان مثال و وارد کردن را به کلیک کنید. -برای برخی از نمونه های داده نمایش داده شد نمایش داده شد نیز در دسترس هستند. - -- [ناشناس یاندکس.مجموعه داده های متریکا](metrica.md) -- [معیار طرحواره ستاره](star-schema.md) -- [ویکیستات](wikistat.md) -- [ترابایت کلیک سیاهههای مربوط از مخلوق](criteo.md) -- [معیار بزرگ داده های تقویت کننده](amplab-benchmark.md) -- [داده های تاکسی نیویورک](nyc-taxi.md) -- [به موقع](ontime.md) - -[مقاله اصلی](https://clickhouse.tech/docs/en/getting_started/example_datasets) diff --git a/docs/fa/getting-started/example-datasets/metrica.md b/docs/fa/getting-started/example-datasets/metrica.md deleted file mode 100644 index ac6743309ef..00000000000 --- a/docs/fa/getting-started/example-datasets/metrica.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 14 -toc_title: "\u06CC\u0627\u0646\u062F\u06A9\u0633\u0627\u0637\u0644\u0627\u0639\u0627\ - \u062A \u0645\u062A\u0631\u06CC\u06A9\u0627" ---- - -# ناشناس یاندکس.اطلاعات متریکا {#anonymized-yandex-metrica-data} - -مجموعه داده شامل دو جدول حاوی داده های ناشناس در مورد بازدید (`hits_v1`) و بازدیدکننده داشته است (`visits_v1`) یاندکس . متریکا شما می توانید اطلاعات بیشتر در مورد یاندکس به عنوان خوانده شده.متریکا در [تاریخچه کلیک](../../introduction/history.md) بخش. - -مجموعه داده ها شامل دو جدول است که هر کدام می توانند به عنوان یک فشرده دانلود شوند `tsv.xz` فایل و یا به عنوان پارتیشن تهیه شده است. علاوه بر این, یک نسخه طولانی از `hits` جدول حاوی 100 میلیون ردیف به عنوان تسو در دسترس است https://datasets.clickhouse.tech/hits/tsv/hits_100m_obfuscated_v1.tsv.xz و به عنوان پارتیشن تهیه شده در https://datasets.clickhouse.tech/hits/partitions/hits_100m_obfuscated_v1.tar.xz. - -## اخذ جداول از پارتیشن های تهیه شده {#obtaining-tables-from-prepared-partitions} - -دانلود و وارد کردن جدول بازدید: - -``` bash -curl -O https://datasets.clickhouse.tech/hits/partitions/hits_v1.tar -tar xvf hits_v1.tar -C /var/lib/clickhouse # path to ClickHouse data directory -# check permissions on unpacked data, fix if required -sudo service clickhouse-server restart -clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" -``` - -دانلود و وارد کردن بازدیدکننده داشته است: - -``` bash -curl -O https://datasets.clickhouse.tech/visits/partitions/visits_v1.tar -tar xvf visits_v1.tar -C /var/lib/clickhouse # path to ClickHouse data directory -# check permissions on unpacked data, fix if required -sudo service clickhouse-server restart -clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" -``` - -## اخذ جداول از فایل تسو فشرده {#obtaining-tables-from-compressed-tsv-file} - -دانلود و وارد کردن بازدید از فایل تسو فشرده: - -``` bash -curl https://datasets.clickhouse.tech/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv -# now create table -clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" -clickhouse-client --query "CREATE TABLE datasets.hits_v1 ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" -# import data -cat hits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.hits_v1 FORMAT TSV" --max_insert_block_size=100000 -# optionally you can optimize table -clickhouse-client --query "OPTIMIZE TABLE datasets.hits_v1 FINAL" -clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" -``` - -دانلود و واردات بازدیدکننده داشته است از فشرده فایل: - -``` bash -curl https://datasets.clickhouse.tech/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv -# now create table -clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" -clickhouse-client --query "CREATE TABLE datasets.visits_v1 ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), Goals Nested(ID UInt32, Serial UInt32, EventTime DateTime, Price Int64, OrderID String, CurrencyID UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, TraficSource Nested(ID Int8, SearchEngineID UInt16, AdvEngineID UInt8, PlaceID UInt16, SocialSourceNetworkID UInt8, Domain String, SearchPhrase String, SocialSourcePage String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), Market Nested(Type UInt8, GoalID UInt32, OrderID String, OrderPrice Int64, PP UInt32, DirectPlaceID UInt32, DirectOrderID UInt32, DirectBannerID UInt32, GoodID String, GoodName String, GoodQuantity Int32, GoodPrice Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" -# import data -cat visits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.visits_v1 FORMAT TSV" --max_insert_block_size=100000 -# optionally you can optimize table -clickhouse-client --query "OPTIMIZE TABLE datasets.visits_v1 FINAL" -clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" -``` - -## به عنوان مثال نمایش داده شد {#example-queries} - -[اموزش کلیک](../../getting-started/tutorial.md) است در یاندکس بر اساس.مجموعه داده های متریکا و راه توصیه شده برای شروع این مجموعه داده ها فقط از طریق تدریس خصوصی است. - -نمونه های اضافی از نمایش داده شد به این جداول را می توان در میان یافت [تست های نفرت انگیز](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) از کلیک هاوس (به نام `test.hists` و `test.visits` وجود دارد). diff --git a/docs/fa/getting-started/example-datasets/nyc-taxi.md b/docs/fa/getting-started/example-datasets/nyc-taxi.md deleted file mode 100644 index 32fbd471b32..00000000000 --- a/docs/fa/getting-started/example-datasets/nyc-taxi.md +++ /dev/null @@ -1,391 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 16 -toc_title: "\u062F\u0627\u062F\u0647 \u0647\u0627\u06CC \u062A\u0627\u06A9\u0633\u06CC\ - \ \u0646\u06CC\u0648\u06CC\u0648\u0631\u06A9" ---- - -# داده های تاکسی نیویورک {#new-york-taxi-data} - -این مجموعه داده را می توان به دو روش دریافت کرد: - -- واردات از دادههای خام -- دانلود پارتیشن های تهیه شده - -## نحوه وارد کردن داده های خام {#how-to-import-the-raw-data} - -ببینید https://github.com/toddwschneider/nyc-taxi-data و http://tech.marksblogg.com/billion-nyc-taxi-rides-redshift.html برای شرح یک مجموعه داده ها و دستورالعمل ها برای دانلود. - -دانلود در مورد منجر خواهد شد 227 گیگابایت از داده های غیر فشرده در فایل های سی سی وی. دانلود حدود یک ساعت طول می کشد بیش از یک اتصال 1 گیگابیت (دانلود موازی از s3.amazonaws.com بازیابی حداقل نیمی از یک 1 گیگابیت کانال). -برخی از فایل ها ممکن است به طور کامل دانلود کنید. بررسی اندازه فایل و دوباره دانلود هر که به نظر می رسد تردید. - -برخی از فایل ها ممکن است حاوی ردیف نامعتبر است. شما می توانید به صورت زیر تعمیر کنید: - -``` bash -sed -E '/(.*,){18,}/d' data/yellow_tripdata_2010-02.csv > data/yellow_tripdata_2010-02.csv_ -sed -E '/(.*,){18,}/d' data/yellow_tripdata_2010-03.csv > data/yellow_tripdata_2010-03.csv_ -mv data/yellow_tripdata_2010-02.csv_ data/yellow_tripdata_2010-02.csv -mv data/yellow_tripdata_2010-03.csv_ data/yellow_tripdata_2010-03.csv -``` - -سپس داده ها باید قبل از پردازش در شرایط لازم. این انتخاب از نقاط در چند ضلعی ایجاد (برای مطابقت با نقاط بر روی نقشه با بخش نیویورک از شهر نیویورک) و ترکیب تمام داده ها را به یک جدول تخت جریمه تنها با استفاده از یک ملحق. برای انجام این کار, شما نیاز به نصب PostgreSQL با PostGIS پشتیبانی می کند. - -مراقب باشید در هنگام اجرا `initialize_database.sh` و به صورت دستی دوباره بررسی کنید که تمام جداول به درستی ایجاد شد. - -این در مورد طول می کشد 20-30 دقیقه برای پردازش ارزش هر ماه از داده ها در شرایط لازم, در مجموع در مورد 48 ساعت ها. - -شما می توانید تعداد ردیف های دانلود شده را به صورت زیر بررسی کنید: - -``` bash -$ time psql nyc-taxi-data -c "SELECT count(*) FROM trips;" -## Count - 1298979494 -(1 row) - -real 7m9.164s -``` - -(این کمی بیش از 1.1 میلیارد ردیف گزارش شده توسط علامت گذاری به عنوان لیتوینچیک در یک سری از پست های وبلاگ.) - -اطلاعات در مورد اتصالات از 370 گیگابایت فضا استفاده می کند. - -در حال بارگذاری: - -``` sql -COPY -( - SELECT trips.id, - trips.vendor_id, - trips.pickup_datetime, - trips.dropoff_datetime, - trips.store_and_fwd_flag, - trips.rate_code_id, - trips.pickup_longitude, - trips.pickup_latitude, - trips.dropoff_longitude, - trips.dropoff_latitude, - trips.passenger_count, - trips.trip_distance, - trips.fare_amount, - trips.extra, - trips.mta_tax, - trips.tip_amount, - trips.tolls_amount, - trips.ehail_fee, - trips.improvement_surcharge, - trips.total_amount, - trips.payment_type, - trips.trip_type, - trips.pickup, - trips.dropoff, - - cab_types.type cab_type, - - weather.precipitation_tenths_of_mm rain, - weather.snow_depth_mm, - weather.snowfall_mm, - weather.max_temperature_tenths_degrees_celsius max_temp, - weather.min_temperature_tenths_degrees_celsius min_temp, - weather.average_wind_speed_tenths_of_meters_per_second wind, - - pick_up.gid pickup_nyct2010_gid, - pick_up.ctlabel pickup_ctlabel, - pick_up.borocode pickup_borocode, - pick_up.boroname pickup_boroname, - pick_up.ct2010 pickup_ct2010, - pick_up.boroct2010 pickup_boroct2010, - pick_up.cdeligibil pickup_cdeligibil, - pick_up.ntacode pickup_ntacode, - pick_up.ntaname pickup_ntaname, - pick_up.puma pickup_puma, - - drop_off.gid dropoff_nyct2010_gid, - drop_off.ctlabel dropoff_ctlabel, - drop_off.borocode dropoff_borocode, - drop_off.boroname dropoff_boroname, - drop_off.ct2010 dropoff_ct2010, - drop_off.boroct2010 dropoff_boroct2010, - drop_off.cdeligibil dropoff_cdeligibil, - drop_off.ntacode dropoff_ntacode, - drop_off.ntaname dropoff_ntaname, - drop_off.puma dropoff_puma - FROM trips - LEFT JOIN cab_types - ON trips.cab_type_id = cab_types.id - LEFT JOIN central_park_weather_observations_raw weather - ON weather.date = trips.pickup_datetime::date - LEFT JOIN nyct2010 pick_up - ON pick_up.gid = trips.pickup_nyct2010_gid - LEFT JOIN nyct2010 drop_off - ON drop_off.gid = trips.dropoff_nyct2010_gid -) TO '/opt/milovidov/nyc-taxi-data/trips.tsv'; -``` - -عکس فوری داده ها با سرعت حدود 50 مگابایت در ثانیه ایجاد می شود. در حالی که ایجاد عکس فوری, شل می خواند از دیسک با سرعت حدود 28 مگابایت در ثانیه. -این طول می کشد حدود 5 ساعت ها. فایل حاصل تسو 590612904969 بایت است. - -ایجاد یک جدول موقت در کلیکهاوس: - -``` sql -CREATE TABLE trips -( -trip_id UInt32, -vendor_id String, -pickup_datetime DateTime, -dropoff_datetime Nullable(DateTime), -store_and_fwd_flag Nullable(FixedString(1)), -rate_code_id Nullable(UInt8), -pickup_longitude Nullable(Float64), -pickup_latitude Nullable(Float64), -dropoff_longitude Nullable(Float64), -dropoff_latitude Nullable(Float64), -passenger_count Nullable(UInt8), -trip_distance Nullable(Float64), -fare_amount Nullable(Float32), -extra Nullable(Float32), -mta_tax Nullable(Float32), -tip_amount Nullable(Float32), -tolls_amount Nullable(Float32), -ehail_fee Nullable(Float32), -improvement_surcharge Nullable(Float32), -total_amount Nullable(Float32), -payment_type Nullable(String), -trip_type Nullable(UInt8), -pickup Nullable(String), -dropoff Nullable(String), -cab_type Nullable(String), -precipitation Nullable(UInt8), -snow_depth Nullable(UInt8), -snowfall Nullable(UInt8), -max_temperature Nullable(UInt8), -min_temperature Nullable(UInt8), -average_wind_speed Nullable(UInt8), -pickup_nyct2010_gid Nullable(UInt8), -pickup_ctlabel Nullable(String), -pickup_borocode Nullable(UInt8), -pickup_boroname Nullable(String), -pickup_ct2010 Nullable(String), -pickup_boroct2010 Nullable(String), -pickup_cdeligibil Nullable(FixedString(1)), -pickup_ntacode Nullable(String), -pickup_ntaname Nullable(String), -pickup_puma Nullable(String), -dropoff_nyct2010_gid Nullable(UInt8), -dropoff_ctlabel Nullable(String), -dropoff_borocode Nullable(UInt8), -dropoff_boroname Nullable(String), -dropoff_ct2010 Nullable(String), -dropoff_boroct2010 Nullable(String), -dropoff_cdeligibil Nullable(String), -dropoff_ntacode Nullable(String), -dropoff_ntaname Nullable(String), -dropoff_puma Nullable(String) -) ENGINE = Log; -``` - -برای تبدیل زمینه ها به انواع داده های صحیح تر و در صورت امکان برای از بین بردن نقاط صفر مورد نیاز است. - -``` bash -$ time clickhouse-client --query="INSERT INTO trips FORMAT TabSeparated" < trips.tsv - -real 75m56.214s -``` - -داده ها با سرعت 112-140 مگابایت در ثانیه خوانده می شوند. -بارگذاری داده ها را به یک جدول نوع ورود به سیستم در یک جریان و جو در زمان 76 دقیقه. -داده ها در این جدول با استفاده از 142 گیگابایت. - -(وارد کردن داده ها به طور مستقیم از پستگرس نیز ممکن است با استفاده از `COPY ... TO PROGRAM`.) - -Unfortunately, all the fields associated with the weather (precipitation…average_wind_speed) were filled with NULL. Because of this, we will remove them from the final data set. - -برای شروع, ما یک جدول بر روی یک سرور ایجاد. بعد ما را به جدول توزیع. - -ایجاد و پر کردن یک جدول خلاصه: - -``` sql -CREATE TABLE trips_mergetree -ENGINE = MergeTree(pickup_date, pickup_datetime, 8192) -AS SELECT - -trip_id, -CAST(vendor_id AS Enum8('1' = 1, '2' = 2, 'CMT' = 3, 'VTS' = 4, 'DDS' = 5, 'B02512' = 10, 'B02598' = 11, 'B02617' = 12, 'B02682' = 13, 'B02764' = 14)) AS vendor_id, -toDate(pickup_datetime) AS pickup_date, -ifNull(pickup_datetime, toDateTime(0)) AS pickup_datetime, -toDate(dropoff_datetime) AS dropoff_date, -ifNull(dropoff_datetime, toDateTime(0)) AS dropoff_datetime, -assumeNotNull(store_and_fwd_flag) IN ('Y', '1', '2') AS store_and_fwd_flag, -assumeNotNull(rate_code_id) AS rate_code_id, -assumeNotNull(pickup_longitude) AS pickup_longitude, -assumeNotNull(pickup_latitude) AS pickup_latitude, -assumeNotNull(dropoff_longitude) AS dropoff_longitude, -assumeNotNull(dropoff_latitude) AS dropoff_latitude, -assumeNotNull(passenger_count) AS passenger_count, -assumeNotNull(trip_distance) AS trip_distance, -assumeNotNull(fare_amount) AS fare_amount, -assumeNotNull(extra) AS extra, -assumeNotNull(mta_tax) AS mta_tax, -assumeNotNull(tip_amount) AS tip_amount, -assumeNotNull(tolls_amount) AS tolls_amount, -assumeNotNull(ehail_fee) AS ehail_fee, -assumeNotNull(improvement_surcharge) AS improvement_surcharge, -assumeNotNull(total_amount) AS total_amount, -CAST((assumeNotNull(payment_type) AS pt) IN ('CSH', 'CASH', 'Cash', 'CAS', 'Cas', '1') ? 'CSH' : (pt IN ('CRD', 'Credit', 'Cre', 'CRE', 'CREDIT', '2') ? 'CRE' : (pt IN ('NOC', 'No Charge', 'No', '3') ? 'NOC' : (pt IN ('DIS', 'Dispute', 'Dis', '4') ? 'DIS' : 'UNK'))) AS Enum8('CSH' = 1, 'CRE' = 2, 'UNK' = 0, 'NOC' = 3, 'DIS' = 4)) AS payment_type_, -assumeNotNull(trip_type) AS trip_type, -ifNull(toFixedString(unhex(pickup), 25), toFixedString('', 25)) AS pickup, -ifNull(toFixedString(unhex(dropoff), 25), toFixedString('', 25)) AS dropoff, -CAST(assumeNotNull(cab_type) AS Enum8('yellow' = 1, 'green' = 2, 'uber' = 3)) AS cab_type, - -assumeNotNull(pickup_nyct2010_gid) AS pickup_nyct2010_gid, -toFloat32(ifNull(pickup_ctlabel, '0')) AS pickup_ctlabel, -assumeNotNull(pickup_borocode) AS pickup_borocode, -CAST(assumeNotNull(pickup_boroname) AS Enum8('Manhattan' = 1, 'Queens' = 4, 'Brooklyn' = 3, '' = 0, 'Bronx' = 2, 'Staten Island' = 5)) AS pickup_boroname, -toFixedString(ifNull(pickup_ct2010, '000000'), 6) AS pickup_ct2010, -toFixedString(ifNull(pickup_boroct2010, '0000000'), 7) AS pickup_boroct2010, -CAST(assumeNotNull(ifNull(pickup_cdeligibil, ' ')) AS Enum8(' ' = 0, 'E' = 1, 'I' = 2)) AS pickup_cdeligibil, -toFixedString(ifNull(pickup_ntacode, '0000'), 4) AS pickup_ntacode, - -CAST(assumeNotNull(pickup_ntaname) AS Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195)) AS pickup_ntaname, - -toUInt16(ifNull(pickup_puma, '0')) AS pickup_puma, - -assumeNotNull(dropoff_nyct2010_gid) AS dropoff_nyct2010_gid, -toFloat32(ifNull(dropoff_ctlabel, '0')) AS dropoff_ctlabel, -assumeNotNull(dropoff_borocode) AS dropoff_borocode, -CAST(assumeNotNull(dropoff_boroname) AS Enum8('Manhattan' = 1, 'Queens' = 4, 'Brooklyn' = 3, '' = 0, 'Bronx' = 2, 'Staten Island' = 5)) AS dropoff_boroname, -toFixedString(ifNull(dropoff_ct2010, '000000'), 6) AS dropoff_ct2010, -toFixedString(ifNull(dropoff_boroct2010, '0000000'), 7) AS dropoff_boroct2010, -CAST(assumeNotNull(ifNull(dropoff_cdeligibil, ' ')) AS Enum8(' ' = 0, 'E' = 1, 'I' = 2)) AS dropoff_cdeligibil, -toFixedString(ifNull(dropoff_ntacode, '0000'), 4) AS dropoff_ntacode, - -CAST(assumeNotNull(dropoff_ntaname) AS Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195)) AS dropoff_ntaname, - -toUInt16(ifNull(dropoff_puma, '0')) AS dropoff_puma - -FROM trips -``` - -این طول می کشد 3030 ثانیه در سرعت حدود 428,000 ردیف در هر ثانیه. -برای بارگذاری سریع تر می توانید جدول را با `Log` موتور به جای `MergeTree`. در این مورد, دانلود کار می کند سریع تر از 200 ثانیه. - -جدول با استفاده از 126 گیگابایت فضای دیسک. - -``` sql -SELECT formatReadableSize(sum(bytes)) FROM system.parts WHERE table = 'trips_mergetree' AND active -``` - -``` text -┌─formatReadableSize(sum(bytes))─┐ -│ 126.18 GiB │ -└────────────────────────────────┘ -``` - -در میان چیزهای دیگر, شما می توانید پرس و جو بهینه سازی در ادغام اجرا. اما لازم نیست که همه چیز بدون این خوب باشد. - -## دانلود پارتیشن های تهیه شده {#download-of-prepared-partitions} - -``` bash -$ curl -O https://datasets.clickhouse.tech/trips_mergetree/partitions/trips_mergetree.tar -$ tar xvf trips_mergetree.tar -C /var/lib/clickhouse # path to ClickHouse data directory -$ # check permissions of unpacked data, fix if required -$ sudo service clickhouse-server restart -$ clickhouse-client --query "select count(*) from datasets.trips_mergetree" -``` - -!!! info "اطلاعات" - اگر شما نمایش داده شد شرح داده شده در زیر اجرا خواهد شد, شما مجبور به استفاده از نام جدول کامل, `datasets.trips_mergetree`. - -## نتایج بر روی سرور تک {#results-on-single-server} - -Q1: - -``` sql -SELECT cab_type, count(*) FROM trips_mergetree GROUP BY cab_type -``` - -0.490 ثانیه است. - -Q2: - -``` sql -SELECT passenger_count, avg(total_amount) FROM trips_mergetree GROUP BY passenger_count -``` - -1.224 ثانیه - -Q3: - -``` sql -SELECT passenger_count, toYear(pickup_date) AS year, count(*) FROM trips_mergetree GROUP BY passenger_count, year -``` - -2.104 ثانیه است. - -Q4: - -``` sql -SELECT passenger_count, toYear(pickup_date) AS year, round(trip_distance) AS distance, count(*) -FROM trips_mergetree -GROUP BY passenger_count, year, distance -ORDER BY year, count(*) DESC -``` - -3.593 ثانیه است. - -سرور زیر مورد استفاده قرار گرفت: - -دو اینتل(ر) یون (ر) پردازنده ای5-2650 ولت2 @ 2.60 گیگاهرتز, 16 هسته فیزیکی کل, 128 دستگاه گوارش رم, 8ایکس6 سل اچ دی در حمله سخت افزار-5 - -زمان اجرای بهترین از سه اجرا می شود. اما با شروع از اجرا دوم نمایش داده شد خواندن داده ها از حافظه پنهان سیستم فایل. بدون ذخیره بیشتر رخ می دهد: داده ها به عنوان خوانده شده و پردازش در هر اجرا. - -ایجاد یک جدول در سه سرور: - -در هر سرور: - -``` sql -CREATE TABLE default.trips_mergetree_third ( trip_id UInt32, vendor_id Enum8('1' = 1, '2' = 2, 'CMT' = 3, 'VTS' = 4, 'DDS' = 5, 'B02512' = 10, 'B02598' = 11, 'B02617' = 12, 'B02682' = 13, 'B02764' = 14), pickup_date Date, pickup_datetime DateTime, dropoff_date Date, dropoff_datetime DateTime, store_and_fwd_flag UInt8, rate_code_id UInt8, pickup_longitude Float64, pickup_latitude Float64, dropoff_longitude Float64, dropoff_latitude Float64, passenger_count UInt8, trip_distance Float64, fare_amount Float32, extra Float32, mta_tax Float32, tip_amount Float32, tolls_amount Float32, ehail_fee Float32, improvement_surcharge Float32, total_amount Float32, payment_type_ Enum8('UNK' = 0, 'CSH' = 1, 'CRE' = 2, 'NOC' = 3, 'DIS' = 4), trip_type UInt8, pickup FixedString(25), dropoff FixedString(25), cab_type Enum8('yellow' = 1, 'green' = 2, 'uber' = 3), pickup_nyct2010_gid UInt8, pickup_ctlabel Float32, pickup_borocode UInt8, pickup_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), pickup_ct2010 FixedString(6), pickup_boroct2010 FixedString(7), pickup_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), pickup_ntacode FixedString(4), pickup_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), pickup_puma UInt16, dropoff_nyct2010_gid UInt8, dropoff_ctlabel Float32, dropoff_borocode UInt8, dropoff_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), dropoff_ct2010 FixedString(6), dropoff_boroct2010 FixedString(7), dropoff_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), dropoff_ntacode FixedString(4), dropoff_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), dropoff_puma UInt16) ENGINE = MergeTree(pickup_date, pickup_datetime, 8192) -``` - -در سرور منبع: - -``` sql -CREATE TABLE trips_mergetree_x3 AS trips_mergetree_third ENGINE = Distributed(perftest, default, trips_mergetree_third, rand()) -``` - -پرس و جو زیر توزیع داده ها: - -``` sql -INSERT INTO trips_mergetree_x3 SELECT * FROM trips_mergetree -``` - -این طول می کشد 2454 ثانیه. - -در سه سرور: - -Q1: 0.212 ثانیه است. -Q2: 0.438 ثانیه است. -Q3: 0.733 ثانیه است. -Q4: 1.241 ثانیه است. - -هیچ شگفتی در اینجا, از نمایش داده شد خطی کوچک. - -ما همچنین نتایج حاصل از یک خوشه از 140 سرور: - -Q1: 0.028 sec. -Q2: 0.043 sec. -Q3: 0.051 sec. -Q4: 0.072 sec. - -در این مورد زمان پردازش پرس و جو بالاتر از همه تاخیر شبکه تعیین می شود. -ما نمایش داده شد با استفاده از یک مشتری واقع در یک مرکز داده یاندکس در فنلاند در یک خوشه در روسیه فرار, که اضافه شده در مورد 20 خانم از تاخیر. - -## خلاصه {#summary} - -| کارگزارها | Q1 | Q2 | Q3 | Q4 | -|-----------|-------|-------|-------|-------| -| 1 | 0.490 | 1.224 | 2.104 | 3.593 | -| 3 | 0.212 | 0.438 | 0.733 | 1.241 | -| 140 | 0.028 | 0.043 | 0.051 | 0.072 | - -[مقاله اصلی](https://clickhouse.tech/docs/en/getting_started/example_datasets/nyc_taxi/) diff --git a/docs/fa/getting-started/example-datasets/ontime.md b/docs/fa/getting-started/example-datasets/ontime.md deleted file mode 100644 index 7282c3c29bb..00000000000 --- a/docs/fa/getting-started/example-datasets/ontime.md +++ /dev/null @@ -1,412 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 15 -toc_title: "\u0628\u0647 \u0645\u0648\u0642\u0639" ---- - -# به موقع {#ontime} - -این مجموعه داده را می توان به دو روش دریافت کرد: - -- واردات از دادههای خام -- دانلود پارتیشن های تهیه شده - -## واردات از دادههای خام {#import-from-raw-data} - -بارگیری داده ها: - -``` bash -for s in `seq 1987 2018` -do -for m in `seq 1 12` -do -wget https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_${s}_${m}.zip -done -done -``` - -(از https://github.com/Percona-Lab/ontime-airline-performance/blob/master/download.sh ) - -ایجاد یک جدول: - -``` sql -CREATE TABLE `ontime` ( - `Year` UInt16, - `Quarter` UInt8, - `Month` UInt8, - `DayofMonth` UInt8, - `DayOfWeek` UInt8, - `FlightDate` Date, - `UniqueCarrier` FixedString(7), - `AirlineID` Int32, - `Carrier` FixedString(2), - `TailNum` String, - `FlightNum` String, - `OriginAirportID` Int32, - `OriginAirportSeqID` Int32, - `OriginCityMarketID` Int32, - `Origin` FixedString(5), - `OriginCityName` String, - `OriginState` FixedString(2), - `OriginStateFips` String, - `OriginStateName` String, - `OriginWac` Int32, - `DestAirportID` Int32, - `DestAirportSeqID` Int32, - `DestCityMarketID` Int32, - `Dest` FixedString(5), - `DestCityName` String, - `DestState` FixedString(2), - `DestStateFips` String, - `DestStateName` String, - `DestWac` Int32, - `CRSDepTime` Int32, - `DepTime` Int32, - `DepDelay` Int32, - `DepDelayMinutes` Int32, - `DepDel15` Int32, - `DepartureDelayGroups` String, - `DepTimeBlk` String, - `TaxiOut` Int32, - `WheelsOff` Int32, - `WheelsOn` Int32, - `TaxiIn` Int32, - `CRSArrTime` Int32, - `ArrTime` Int32, - `ArrDelay` Int32, - `ArrDelayMinutes` Int32, - `ArrDel15` Int32, - `ArrivalDelayGroups` Int32, - `ArrTimeBlk` String, - `Cancelled` UInt8, - `CancellationCode` FixedString(1), - `Diverted` UInt8, - `CRSElapsedTime` Int32, - `ActualElapsedTime` Int32, - `AirTime` Int32, - `Flights` Int32, - `Distance` Int32, - `DistanceGroup` UInt8, - `CarrierDelay` Int32, - `WeatherDelay` Int32, - `NASDelay` Int32, - `SecurityDelay` Int32, - `LateAircraftDelay` Int32, - `FirstDepTime` String, - `TotalAddGTime` String, - `LongestAddGTime` String, - `DivAirportLandings` String, - `DivReachedDest` String, - `DivActualElapsedTime` String, - `DivArrDelay` String, - `DivDistance` String, - `Div1Airport` String, - `Div1AirportID` Int32, - `Div1AirportSeqID` Int32, - `Div1WheelsOn` String, - `Div1TotalGTime` String, - `Div1LongestGTime` String, - `Div1WheelsOff` String, - `Div1TailNum` String, - `Div2Airport` String, - `Div2AirportID` Int32, - `Div2AirportSeqID` Int32, - `Div2WheelsOn` String, - `Div2TotalGTime` String, - `Div2LongestGTime` String, - `Div2WheelsOff` String, - `Div2TailNum` String, - `Div3Airport` String, - `Div3AirportID` Int32, - `Div3AirportSeqID` Int32, - `Div3WheelsOn` String, - `Div3TotalGTime` String, - `Div3LongestGTime` String, - `Div3WheelsOff` String, - `Div3TailNum` String, - `Div4Airport` String, - `Div4AirportID` Int32, - `Div4AirportSeqID` Int32, - `Div4WheelsOn` String, - `Div4TotalGTime` String, - `Div4LongestGTime` String, - `Div4WheelsOff` String, - `Div4TailNum` String, - `Div5Airport` String, - `Div5AirportID` Int32, - `Div5AirportSeqID` Int32, - `Div5WheelsOn` String, - `Div5TotalGTime` String, - `Div5LongestGTime` String, - `Div5WheelsOff` String, - `Div5TailNum` String -) ENGINE = MergeTree -PARTITION BY Year -ORDER BY (Carrier, FlightDate) -SETTINGS index_granularity = 8192; -``` - -بارگیری داده: - -``` bash -$ for i in *.zip; do echo $i; unzip -cq $i '*.csv' | sed 's/\.00//g' | clickhouse-client --host=example-perftest01j --query="INSERT INTO ontime FORMAT CSVWithNames"; done -``` - -## دانلود پارتیشن های تهیه شده {#download-of-prepared-partitions} - -``` bash -$ curl -O https://datasets.clickhouse.tech/ontime/partitions/ontime.tar -$ tar xvf ontime.tar -C /var/lib/clickhouse # path to ClickHouse data directory -$ # check permissions of unpacked data, fix if required -$ sudo service clickhouse-server restart -$ clickhouse-client --query "select count(*) from datasets.ontime" -``` - -!!! info "اطلاعات" - اگر شما نمایش داده شد شرح داده شده در زیر اجرا خواهد شد, شما مجبور به استفاده از نام جدول کامل, `datasets.ontime`. - -## نمایش داده شد {#queries} - -Q0. - -``` sql -SELECT avg(c1) -FROM -( - SELECT Year, Month, count(*) AS c1 - FROM ontime - GROUP BY Year, Month -); -``` - -Q1. تعداد پرواز در روز از سال 2000 تا 2008 - -``` sql -SELECT DayOfWeek, count(*) AS c -FROM ontime -WHERE Year>=2000 AND Year<=2008 -GROUP BY DayOfWeek -ORDER BY c DESC; -``` - -Q2. تعداد پروازهای تاخیر بیش از 10 دقیقه, گروه بندی شده توسط روز هفته, برای 2000-2008 - -``` sql -SELECT DayOfWeek, count(*) AS c -FROM ontime -WHERE DepDelay>10 AND Year>=2000 AND Year<=2008 -GROUP BY DayOfWeek -ORDER BY c DESC; -``` - -پرسش 3 تعداد تاخیر در فرودگاه برای 2000-2008 - -``` sql -SELECT Origin, count(*) AS c -FROM ontime -WHERE DepDelay>10 AND Year>=2000 AND Year<=2008 -GROUP BY Origin -ORDER BY c DESC -LIMIT 10; -``` - -پرسش4. تعداد تاخیر توسط حامل برای 2007 - -``` sql -SELECT Carrier, count(*) -FROM ontime -WHERE DepDelay>10 AND Year=2007 -GROUP BY Carrier -ORDER BY count(*) DESC; -``` - -پرسش 5 درصد تاخیر توسط حامل برای 2007 - -``` sql -SELECT Carrier, c, c2, c*100/c2 as c3 -FROM -( - SELECT - Carrier, - count(*) AS c - FROM ontime - WHERE DepDelay>10 - AND Year=2007 - GROUP BY Carrier -) -JOIN -( - SELECT - Carrier, - count(*) AS c2 - FROM ontime - WHERE Year=2007 - GROUP BY Carrier -) USING Carrier -ORDER BY c3 DESC; -``` - -نسخه بهتر از پرس و جو همان: - -``` sql -SELECT Carrier, avg(DepDelay>10)*100 AS c3 -FROM ontime -WHERE Year=2007 -GROUP BY Carrier -ORDER BY c3 DESC -``` - -س6 درخواست قبلی برای طیف وسیع تری از سال 2000-2008 - -``` sql -SELECT Carrier, c, c2, c*100/c2 as c3 -FROM -( - SELECT - Carrier, - count(*) AS c - FROM ontime - WHERE DepDelay>10 - AND Year>=2000 AND Year<=2008 - GROUP BY Carrier -) -JOIN -( - SELECT - Carrier, - count(*) AS c2 - FROM ontime - WHERE Year>=2000 AND Year<=2008 - GROUP BY Carrier -) USING Carrier -ORDER BY c3 DESC; -``` - -نسخه بهتر از پرس و جو همان: - -``` sql -SELECT Carrier, avg(DepDelay>10)*100 AS c3 -FROM ontime -WHERE Year>=2000 AND Year<=2008 -GROUP BY Carrier -ORDER BY c3 DESC; -``` - -پرسش 7 درصد پرواز به تاخیر افتاد برای بیش از 10 دقیقه, به سال - -``` sql -SELECT Year, c1/c2 -FROM -( - select - Year, - count(*)*100 as c1 - from ontime - WHERE DepDelay>10 - GROUP BY Year -) -JOIN -( - select - Year, - count(*) as c2 - from ontime - GROUP BY Year -) USING (Year) -ORDER BY Year; -``` - -نسخه بهتر از پرس و جو همان: - -``` sql -SELECT Year, avg(DepDelay>10)*100 -FROM ontime -GROUP BY Year -ORDER BY Year; -``` - -س8 محبوب ترین مقصد توسط تعدادی از شهرستانها به طور مستقیم متصل برای محدوده های مختلف سال - -``` sql -SELECT DestCityName, uniqExact(OriginCityName) AS u -FROM ontime -WHERE Year >= 2000 and Year <= 2010 -GROUP BY DestCityName -ORDER BY u DESC LIMIT 10; -``` - -Q9. - -``` sql -SELECT Year, count(*) AS c1 -FROM ontime -GROUP BY Year; -``` - -Q10. - -``` sql -SELECT - min(Year), max(Year), Carrier, count(*) AS cnt, - sum(ArrDelayMinutes>30) AS flights_delayed, - round(sum(ArrDelayMinutes>30)/count(*),2) AS rate -FROM ontime -WHERE - DayOfWeek NOT IN (6,7) AND OriginState NOT IN ('AK', 'HI', 'PR', 'VI') - AND DestState NOT IN ('AK', 'HI', 'PR', 'VI') - AND FlightDate < '2010-01-01' -GROUP by Carrier -HAVING cnt>100000 and max(Year)>1990 -ORDER by rate DESC -LIMIT 1000; -``` - -پاداش: - -``` sql -SELECT avg(cnt) -FROM -( - SELECT Year,Month,count(*) AS cnt - FROM ontime - WHERE DepDel15=1 - GROUP BY Year,Month -); - -SELECT avg(c1) FROM -( - SELECT Year,Month,count(*) AS c1 - FROM ontime - GROUP BY Year,Month -); - -SELECT DestCityName, uniqExact(OriginCityName) AS u -FROM ontime -GROUP BY DestCityName -ORDER BY u DESC -LIMIT 10; - -SELECT OriginCityName, DestCityName, count() AS c -FROM ontime -GROUP BY OriginCityName, DestCityName -ORDER BY c DESC -LIMIT 10; - -SELECT OriginCityName, count() AS c -FROM ontime -GROUP BY OriginCityName -ORDER BY c DESC -LIMIT 10; -``` - -این تست عملکرد توسط وادیم تکچنکو ایجاد شد. ببینید: - -- https://www.percona.com/blog/2009/10/02/analyzing-air-traffic-performance-with-infobright-and-monetdb/ -- https://www.percona.com/blog/2009/10/26/air-traffic-queries-in-luciddb/ -- https://www.percona.com/blog/2009/11/02/air-traffic-queries-in-infinidb-early-alpha/ -- https://www.percona.com/blog/2014/04/21/using-apache-hadoop-and-impala-together-with-mysql-for-data-analysis/ -- https://www.percona.com/blog/2016/01/07/apache-spark-with-air-ontime-performance-data/ -- http://nickmakos.blogspot.ru/2012/08/analyzing-air-traffic-performance-with.html - -[مقاله اصلی](https://clickhouse.tech/docs/en/getting_started/example_datasets/ontime/) diff --git a/docs/fa/getting-started/example-datasets/star-schema.md b/docs/fa/getting-started/example-datasets/star-schema.md deleted file mode 100644 index 3a8ec02bae0..00000000000 --- a/docs/fa/getting-started/example-datasets/star-schema.md +++ /dev/null @@ -1,371 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 20 -toc_title: "\u0645\u0639\u06CC\u0627\u0631 \u0637\u0631\u062D\u0648\u0627\u0631\u0647\ - \ \u0633\u062A\u0627\u0631\u0647" ---- - -# معیار طرحواره ستاره {#star-schema-benchmark} - -تدوین نرم افزار: - -``` bash -$ git clone git@github.com:vadimtk/ssb-dbgen.git -$ cd ssb-dbgen -$ make -``` - -تولید داده: - -!!! warning "توجه" - با `-s 100` تولید نرم افزار 600 میلیون ردیف (67 گیگابایت), در حالی که `-s 1000` این تولید 6 میلیارد ردیف (که طول می کشد زمان زیادی) - -``` bash -$ ./dbgen -s 1000 -T c -$ ./dbgen -s 1000 -T l -$ ./dbgen -s 1000 -T p -$ ./dbgen -s 1000 -T s -$ ./dbgen -s 1000 -T d -``` - -ایجاد جداول در محل کلیک: - -``` sql -CREATE TABLE customer -( - C_CUSTKEY UInt32, - C_NAME String, - C_ADDRESS String, - C_CITY LowCardinality(String), - C_NATION LowCardinality(String), - C_REGION LowCardinality(String), - C_PHONE String, - C_MKTSEGMENT LowCardinality(String) -) -ENGINE = MergeTree ORDER BY (C_CUSTKEY); - -CREATE TABLE lineorder -( - LO_ORDERKEY UInt32, - LO_LINENUMBER UInt8, - LO_CUSTKEY UInt32, - LO_PARTKEY UInt32, - LO_SUPPKEY UInt32, - LO_ORDERDATE Date, - LO_ORDERPRIORITY LowCardinality(String), - LO_SHIPPRIORITY UInt8, - LO_QUANTITY UInt8, - LO_EXTENDEDPRICE UInt32, - LO_ORDTOTALPRICE UInt32, - LO_DISCOUNT UInt8, - LO_REVENUE UInt32, - LO_SUPPLYCOST UInt32, - LO_TAX UInt8, - LO_COMMITDATE Date, - LO_SHIPMODE LowCardinality(String) -) -ENGINE = MergeTree PARTITION BY toYear(LO_ORDERDATE) ORDER BY (LO_ORDERDATE, LO_ORDERKEY); - -CREATE TABLE part -( - P_PARTKEY UInt32, - P_NAME String, - P_MFGR LowCardinality(String), - P_CATEGORY LowCardinality(String), - P_BRAND LowCardinality(String), - P_COLOR LowCardinality(String), - P_TYPE LowCardinality(String), - P_SIZE UInt8, - P_CONTAINER LowCardinality(String) -) -ENGINE = MergeTree ORDER BY P_PARTKEY; - -CREATE TABLE supplier -( - S_SUPPKEY UInt32, - S_NAME String, - S_ADDRESS String, - S_CITY LowCardinality(String), - S_NATION LowCardinality(String), - S_REGION LowCardinality(String), - S_PHONE String -) -ENGINE = MergeTree ORDER BY S_SUPPKEY; -``` - -درج داده: - -``` bash -$ clickhouse-client --query "INSERT INTO customer FORMAT CSV" < customer.tbl -$ clickhouse-client --query "INSERT INTO part FORMAT CSV" < part.tbl -$ clickhouse-client --query "INSERT INTO supplier FORMAT CSV" < supplier.tbl -$ clickhouse-client --query "INSERT INTO lineorder FORMAT CSV" < lineorder.tbl -``` - -تبدیل “star schema” به جریمه “flat schema”: - -``` sql -SET max_memory_usage = 20000000000; - -CREATE TABLE lineorder_flat -ENGINE = MergeTree -PARTITION BY toYear(LO_ORDERDATE) -ORDER BY (LO_ORDERDATE, LO_ORDERKEY) AS -SELECT - l.LO_ORDERKEY AS LO_ORDERKEY, - l.LO_LINENUMBER AS LO_LINENUMBER, - l.LO_CUSTKEY AS LO_CUSTKEY, - l.LO_PARTKEY AS LO_PARTKEY, - l.LO_SUPPKEY AS LO_SUPPKEY, - l.LO_ORDERDATE AS LO_ORDERDATE, - l.LO_ORDERPRIORITY AS LO_ORDERPRIORITY, - l.LO_SHIPPRIORITY AS LO_SHIPPRIORITY, - l.LO_QUANTITY AS LO_QUANTITY, - l.LO_EXTENDEDPRICE AS LO_EXTENDEDPRICE, - l.LO_ORDTOTALPRICE AS LO_ORDTOTALPRICE, - l.LO_DISCOUNT AS LO_DISCOUNT, - l.LO_REVENUE AS LO_REVENUE, - l.LO_SUPPLYCOST AS LO_SUPPLYCOST, - l.LO_TAX AS LO_TAX, - l.LO_COMMITDATE AS LO_COMMITDATE, - l.LO_SHIPMODE AS LO_SHIPMODE, - c.C_NAME AS C_NAME, - c.C_ADDRESS AS C_ADDRESS, - c.C_CITY AS C_CITY, - c.C_NATION AS C_NATION, - c.C_REGION AS C_REGION, - c.C_PHONE AS C_PHONE, - c.C_MKTSEGMENT AS C_MKTSEGMENT, - s.S_NAME AS S_NAME, - s.S_ADDRESS AS S_ADDRESS, - s.S_CITY AS S_CITY, - s.S_NATION AS S_NATION, - s.S_REGION AS S_REGION, - s.S_PHONE AS S_PHONE, - p.P_NAME AS P_NAME, - p.P_MFGR AS P_MFGR, - p.P_CATEGORY AS P_CATEGORY, - p.P_BRAND AS P_BRAND, - p.P_COLOR AS P_COLOR, - p.P_TYPE AS P_TYPE, - p.P_SIZE AS P_SIZE, - p.P_CONTAINER AS P_CONTAINER -FROM lineorder AS l -INNER JOIN customer AS c ON c.C_CUSTKEY = l.LO_CUSTKEY -INNER JOIN supplier AS s ON s.S_SUPPKEY = l.LO_SUPPKEY -INNER JOIN part AS p ON p.P_PARTKEY = l.LO_PARTKEY; -``` - -در حال اجرا نمایش داده شد: - -Q1.1 - -``` sql -SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue -FROM lineorder_flat -WHERE toYear(LO_ORDERDATE) = 1993 AND LO_DISCOUNT BETWEEN 1 AND 3 AND LO_QUANTITY < 25; -``` - -Q1.2 - -``` sql -SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue -FROM lineorder_flat -WHERE toYYYYMM(LO_ORDERDATE) = 199401 AND LO_DISCOUNT BETWEEN 4 AND 6 AND LO_QUANTITY BETWEEN 26 AND 35; -``` - -Q1.3 - -``` sql -SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue -FROM lineorder_flat -WHERE toISOWeek(LO_ORDERDATE) = 6 AND toYear(LO_ORDERDATE) = 1994 - AND LO_DISCOUNT BETWEEN 5 AND 7 AND LO_QUANTITY BETWEEN 26 AND 35; -``` - -Q2.1 - -``` sql -SELECT - sum(LO_REVENUE), - toYear(LO_ORDERDATE) AS year, - P_BRAND -FROM lineorder_flat -WHERE P_CATEGORY = 'MFGR#12' AND S_REGION = 'AMERICA' -GROUP BY - year, - P_BRAND -ORDER BY - year, - P_BRAND; -``` - -Q2.2 - -``` sql -SELECT - sum(LO_REVENUE), - toYear(LO_ORDERDATE) AS year, - P_BRAND -FROM lineorder_flat -WHERE P_BRAND >= 'MFGR#2221' AND P_BRAND <= 'MFGR#2228' AND S_REGION = 'ASIA' -GROUP BY - year, - P_BRAND -ORDER BY - year, - P_BRAND; -``` - -Q2.3 - -``` sql -SELECT - sum(LO_REVENUE), - toYear(LO_ORDERDATE) AS year, - P_BRAND -FROM lineorder_flat -WHERE P_BRAND = 'MFGR#2239' AND S_REGION = 'EUROPE' -GROUP BY - year, - P_BRAND -ORDER BY - year, - P_BRAND; -``` - -Q3.1 - -``` sql -SELECT - C_NATION, - S_NATION, - toYear(LO_ORDERDATE) AS year, - sum(LO_REVENUE) AS revenue -FROM lineorder_flat -WHERE C_REGION = 'ASIA' AND S_REGION = 'ASIA' AND year >= 1992 AND year <= 1997 -GROUP BY - C_NATION, - S_NATION, - year -ORDER BY - year ASC, - revenue DESC; -``` - -Q3.2 - -``` sql -SELECT - C_CITY, - S_CITY, - toYear(LO_ORDERDATE) AS year, - sum(LO_REVENUE) AS revenue -FROM lineorder_flat -WHERE C_NATION = 'UNITED STATES' AND S_NATION = 'UNITED STATES' AND year >= 1992 AND year <= 1997 -GROUP BY - C_CITY, - S_CITY, - year -ORDER BY - year ASC, - revenue DESC; -``` - -Q3.3 - -``` sql -SELECT - C_CITY, - S_CITY, - toYear(LO_ORDERDATE) AS year, - sum(LO_REVENUE) AS revenue -FROM lineorder_flat -WHERE (C_CITY = 'UNITED KI1' OR C_CITY = 'UNITED KI5') AND (S_CITY = 'UNITED KI1' OR S_CITY = 'UNITED KI5') AND year >= 1992 AND year <= 1997 -GROUP BY - C_CITY, - S_CITY, - year -ORDER BY - year ASC, - revenue DESC; -``` - -Q3.4 - -``` sql -SELECT - C_CITY, - S_CITY, - toYear(LO_ORDERDATE) AS year, - sum(LO_REVENUE) AS revenue -FROM lineorder_flat -WHERE (C_CITY = 'UNITED KI1' OR C_CITY = 'UNITED KI5') AND (S_CITY = 'UNITED KI1' OR S_CITY = 'UNITED KI5') AND toYYYYMM(LO_ORDERDATE) = 199712 -GROUP BY - C_CITY, - S_CITY, - year -ORDER BY - year ASC, - revenue DESC; -``` - -Q4.1 - -``` sql -SELECT - toYear(LO_ORDERDATE) AS year, - C_NATION, - sum(LO_REVENUE - LO_SUPPLYCOST) AS profit -FROM lineorder_flat -WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') -GROUP BY - year, - C_NATION -ORDER BY - year ASC, - C_NATION ASC; -``` - -Q4.2 - -``` sql -SELECT - toYear(LO_ORDERDATE) AS year, - S_NATION, - P_CATEGORY, - sum(LO_REVENUE - LO_SUPPLYCOST) AS profit -FROM lineorder_flat -WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (year = 1997 OR year = 1998) AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') -GROUP BY - year, - S_NATION, - P_CATEGORY -ORDER BY - year ASC, - S_NATION ASC, - P_CATEGORY ASC; -``` - -Q4.3 - -``` sql -SELECT - toYear(LO_ORDERDATE) AS year, - S_CITY, - P_BRAND, - sum(LO_REVENUE - LO_SUPPLYCOST) AS profit -FROM lineorder_flat -WHERE S_NATION = 'UNITED STATES' AND (year = 1997 OR year = 1998) AND P_CATEGORY = 'MFGR#14' -GROUP BY - year, - S_CITY, - P_BRAND -ORDER BY - year ASC, - S_CITY ASC, - P_BRAND ASC; -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/getting_started/example_datasets/star_schema/) diff --git a/docs/fa/getting-started/example-datasets/wikistat.md b/docs/fa/getting-started/example-datasets/wikistat.md deleted file mode 100644 index b85121b61a8..00000000000 --- a/docs/fa/getting-started/example-datasets/wikistat.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 18 -toc_title: "\u0648\u06CC\u06A9\u06CC\u0633\u062A\u0627\u062A" ---- - -# ویکیستات {#wikistat} - -ببینید: http://dumps.wikimedia.org/other/pagecounts-raw/ - -ایجاد یک جدول: - -``` sql -CREATE TABLE wikistat -( - date Date, - time DateTime, - project String, - subproject String, - path String, - hits UInt64, - size UInt64 -) ENGINE = MergeTree(date, (path, time), 8192); -``` - -بارگیری داده: - -``` bash -$ for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sSL "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt -$ cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done -$ ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/getting_started/example_datasets/wikistat/) diff --git a/docs/fa/getting-started/index.md b/docs/fa/getting-started/index.md deleted file mode 100644 index d62f12158ed..00000000000 --- a/docs/fa/getting-started/index.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0634\u0631\u0648\u0639 \u06A9\u0627\u0631" -toc_hidden: true -toc_priority: 8 -toc_title: "\u0645\u062E\u0641\u06CC" ---- - -# شروع کار {#getting-started} - -اگر شما تازه به تاتر هستند و می خواهید برای دریافت یک دست در احساس عملکرد خود را, اول از همه, شما نیاز به از طریق رفتن [مراحل نصب](install.md). بعد از که شما می توانید: - -- [برو از طریق مفصل](tutorial.md) -- [تجربه با مجموعه داده های نمونه](example-datasets/ontime.md) - -[مقاله اصلی](https://clickhouse.tech/docs/en/getting_started/) diff --git a/docs/fa/getting-started/install.md b/docs/fa/getting-started/install.md deleted file mode 100644 index 50a9d6230b4..00000000000 --- a/docs/fa/getting-started/install.md +++ /dev/null @@ -1,183 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 11 -toc_title: "\u0646\u0635\u0628 \u0648 \u0631\u0627\u0647 \u0627\u0646\u062F\u0627\u0632\ - \u06CC" ---- - -# نصب و راه اندازی {#installation} - -## سیستم مورد نیاز {#system-requirements} - -ClickHouse می تواند اجرا بر روی هر Linux, FreeBSD یا سیستم عامل Mac OS X با x86_64, AArch64 یا PowerPC64LE معماری CPU. - -رسمی از پیش ساخته شده باینری به طور معمول وارد شده برای ایکس86_64 و اهرم بورس تحصیلی 4.2 مجموعه دستورالعمل, بنابراین مگر اینکه در غیر این صورت اعلام کرد استفاده از پردازنده است که پشتیبانی می شود یک سیستم اضافی مورد نیاز. در اینجا دستور برای بررسی اگر پردازنده فعلی دارای پشتیبانی برای اس اس 4.2: - -``` bash -$ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" -``` - -برای اجرای ClickHouse در پردازنده هایی که پشتیبانی نمی SSE 4.2 یا AArch64 یا PowerPC64LE معماری شما باید [ساخت کلیک از منابع](#from-sources) با تنظیمات پیکربندی مناسب. - -## گزینه های نصب موجود {#available-installation-options} - -### از بسته های دب {#install-from-deb-packages} - -توصیه می شود به استفاده از رسمی از پیش وارد شده `deb` بسته برای دبیان یا اوبونتو. اجرای این دستورات برای نصب بسته: - -``` bash -{% include 'install/deb.sh' %} -``` - -اگر شما می خواهید به استفاده از نسخه های اخیر, جایگزین کردن `stable` با `testing` (این است که برای محیط های تست خود را توصیه می شود). - -شما همچنین می توانید بسته ها را به صورت دستی دانلود و نصب کنید [اینجا](https://repo.clickhouse.tech/deb/stable/main/). - -#### بستهها {#packages} - -- `clickhouse-common-static` — Installs ClickHouse compiled binary files. -- `clickhouse-server` — Creates a symbolic link for `clickhouse-server` و نصب پیکربندی سرور به طور پیش فرض. -- `clickhouse-client` — Creates a symbolic link for `clickhouse-client` و دیگر ابزار مربوط به مشتری. و نصب فایل های پیکربندی مشتری. -- `clickhouse-common-static-dbg` — Installs ClickHouse compiled binary files with debug info. - -### از بسته های دور در دقیقه {#from-rpm-packages} - -توصیه می شود به استفاده از رسمی از پیش وارد شده `rpm` بسته برای لینوکس لینوکس, کلاه قرمز, و همه توزیع های لینوکس مبتنی بر دور در دقیقه دیگر. - -اولین, شما نیاز به اضافه کردن مخزن رسمی: - -``` bash -sudo yum install yum-utils -sudo rpm --import https://repo.clickhouse.tech/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.clickhouse.tech/rpm/stable/x86_64 -``` - -اگر شما می خواهید به استفاده از نسخه های اخیر, جایگزین کردن `stable` با `testing` (این است که برای محیط های تست خود را توصیه می شود). این `prestable` برچسب است که گاهی اوقات در دسترس بیش از حد. - -سپس این دستورات را برای نصب بسته ها اجرا کنید: - -``` bash -sudo yum install clickhouse-server clickhouse-client -``` - -شما همچنین می توانید بسته ها را به صورت دستی دانلود و نصب کنید [اینجا](https://repo.clickhouse.tech/rpm/stable/x86_64). - -### از بایگانی {#from-tgz-archives} - -توصیه می شود به استفاده از رسمی از پیش وارد شده `tgz` بایگانی برای همه توزیع های لینوکس, که نصب و راه اندازی `deb` یا `rpm` بسته امکان پذیر نیست. - -نسخه مورد نیاز را می توان با دانلود `curl` یا `wget` از مخزن https://repo.clickhouse.tech/tgz/. -پس از که دانلود بایگانی باید غیر بستهای و نصب شده با اسکریپت نصب و راه اندازی. به عنوان مثال برای جدیدترین نسخه: - -``` bash -export LATEST_VERSION=`curl https://api.github.com/repos/ClickHouse/ClickHouse/tags 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n 1` -curl -O https://repo.clickhouse.tech/tgz/clickhouse-common-static-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.tech/tgz/clickhouse-common-static-dbg-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.tech/tgz/clickhouse-server-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.tech/tgz/clickhouse-client-$LATEST_VERSION.tgz - -tar -xzvf clickhouse-common-static-$LATEST_VERSION.tgz -sudo clickhouse-common-static-$LATEST_VERSION/install/doinst.sh - -tar -xzvf clickhouse-common-static-dbg-$LATEST_VERSION.tgz -sudo clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh - -tar -xzvf clickhouse-server-$LATEST_VERSION.tgz -sudo clickhouse-server-$LATEST_VERSION/install/doinst.sh -sudo /etc/init.d/clickhouse-server start - -tar -xzvf clickhouse-client-$LATEST_VERSION.tgz -sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh -``` - -برای محیط های تولید توصیه می شود از جدیدترین استفاده کنید `stable`- نسخه شما می توانید شماره خود را در صفحه گیتهاب پیدا https://github.com/ClickHouse/ClickHouse/tags با پسوند `-stable`. - -### از تصویر کارگر بارانداز {#from-docker-image} - -برای اجرای کلیک در داخل کارگر بارانداز راهنمای دنبال کنید [داکر توپی](https://hub.docker.com/r/yandex/clickhouse-server/). این تصاویر استفاده رسمی `deb` بسته در داخل. - -### از منابع {#from-sources} - -به صورت دستی کامپایل فاحشه خانه, دستورالعمل برای دنبال [لینوکس](../development/build.md) یا [سیستم عامل مک ایکس](../development/build-osx.md). - -شما می توانید بسته های کامپایل و نصب و یا استفاده از برنامه های بدون نصب بسته. همچنین با ساخت دستی شما می توانید ثانیه 4.2 مورد نیاز غیر فعال کردن و یا ساخت برای ایالت64 پردازنده. - - Client: programs/clickhouse-client - Server: programs/clickhouse-server - -شما نیاز به ایجاد یک داده ها و پوشه ابرداده و `chown` برای کاربر مورد نظر. مسیر خود را می توان در پیکربندی سرور تغییر (سری سی/برنامه/سرور/پیکربندی.به طور پیش فرض: - - /opt/clickhouse/data/default/ - /opt/clickhouse/metadata/default/ - -در جنتو, شما فقط می توانید استفاده کنید `emerge clickhouse` برای نصب کلیک از منابع. - -## راهاندازی {#launch} - -برای شروع سرور به عنوان یک شبح, اجرا: - -``` bash -$ sudo service clickhouse-server start -``` - -اگر شما لازم نیست `service` فرمان, اجرا به عنوان - -``` bash -$ sudo /etc/init.d/clickhouse-server start -``` - -سیاهههای مربوط در `/var/log/clickhouse-server/` فهرست راهنما. - -اگر سرور شروع نمی کند, بررسی تنظیمات در فایل `/etc/clickhouse-server/config.xml`. - -شما همچنین می توانید سرور را از کنسول به صورت دستی راه اندازی کنید: - -``` bash -$ clickhouse-server --config-file=/etc/clickhouse-server/config.xml -``` - -در این مورد, ورود به سیستم خواهد شد به کنسول چاپ, که مناسب است در طول توسعه. -اگر فایل پیکربندی در دایرکتوری فعلی است, شما لازم نیست برای مشخص کردن `--config-file` پارامتر. به طور پیش فرض استفاده می کند `./config.xml`. - -تاتر پشتیبانی از تنظیمات محدودیت دسترسی. این در واقع `users.xml` پرونده) در کنار ( `config.xml`). -به طور پیش فرض, دسترسی از هر نقطه برای اجازه `default` کاربر, بدون رمز عبور. ببینید `user/default/networks`. -برای کسب اطلاعات بیشتر به بخش مراجعه کنید [“Configuration Files”](../operations/configuration-files.md). - -پس از راه اندازی سرور, شما می توانید مشتری خط فرمان برای اتصال به استفاده: - -``` bash -$ clickhouse-client -``` - -به طور پیش فرض به `localhost:9000` از طرف کاربر `default` بدون رمز عبور. همچنین می تواند مورد استفاده قرار گیرد برای اتصال به یک سرور از راه دور با استفاده از `--host` استدلال کردن. - -ترمینال باید از کدگذاری جی تی اف 8 استفاده کند. -برای کسب اطلاعات بیشتر به بخش مراجعه کنید [“Command-line client”](../interfaces/cli.md). - -مثال: - -``` bash -$ ./clickhouse-client -ClickHouse client version 0.0.18749. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.18749. - -:) SELECT 1 - -SELECT 1 - -┌─1─┐ -│ 1 │ -└───┘ - -1 rows in set. Elapsed: 0.003 sec. - -:) -``` - -**تبریک, سیستم کار می کند!** - -برای ادامه تجربه, شما می توانید یکی از مجموعه داده های تست دانلود و یا رفتن را از طریق [اموزش](https://clickhouse.tech/tutorial.html). - -[مقاله اصلی](https://clickhouse.tech/docs/en/getting_started/install/) diff --git a/docs/fa/getting-started/playground.md b/docs/fa/getting-started/playground.md deleted file mode 100644 index ccafc2723cb..00000000000 --- a/docs/fa/getting-started/playground.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 14 -toc_title: "\u0632\u0645\u06CC\u0646 \u0628\u0627\u0632\u06CC" ---- - -# تاتر زمین بازی {#clickhouse-playground} - -[تاتر زمین بازی](https://play.clickhouse.tech?file=welcome) اجازه می دهد تا مردم را به تجربه با تاتر در حال اجرا نمایش داده شد فورا, بدون راه اندازی سرور و یا خوشه خود را. -چند مجموعه داده به عنوان مثال در زمین بازی و همچنین نمونه نمایش داده شد که نشان می دهد ویژگی های تاتر در دسترس هستند. - -نمایش داده شد به عنوان یک کاربر فقط خواندنی اجرا شده است. این نشان میدهد برخی از محدودیت: - -- پرسشهای دادل مجاز نیستند -- درج نمایش داده شد امکان پذیر نیست - -تنظیمات زیر نیز اجرا می شوند: -- [`max_result_bytes=10485760`](../operations/settings/query_complexity/#max-result-bytes) -- [`max_result_rows=2000`](../operations/settings/query_complexity/#setting-max_result_rows) -- [`result_overflow_mode=break`](../operations/settings/query_complexity/#result-overflow-mode) -- [`max_execution_time=60000`](../operations/settings/query_complexity/#max-execution-time) - -زمین بازی کلیک می دهد تجربه متر2.کوچک -[خدمات مدیریت شده برای کلیک](https://cloud.yandex.com/services/managed-clickhouse) -به عنوان مثال میزبانی شده در [یاندکسابر](https://cloud.yandex.com/). -اطلاعات بیشتر در مورد [ابر دهندگان](../commercial/cloud.md). - -ClickHouse زمین بازی و رابط کاربری وب سایت باعث می شود درخواست از طریق ClickHouse [HTTP API](../interfaces/http.md). -باطن زمین بازی فقط یک خوشه محل کلیک بدون هیچ گونه نرم افزار سمت سرور اضافی است. -نقطه پایانی کلیک اچتیتیپس نیز به عنوان بخشی از زمین بازی در دسترس است. - -شما می توانید نمایش داده شد به زمین بازی با استفاده از هر مشتری قام را, مثلا [حلقه](https://curl.haxx.se) یا [عناصر](https://www.gnu.org/software/wget/), و یا راه اندازی یک اتصال با استفاده از [JDBC](../interfaces/jdbc.md) یا [ODBC](../interfaces/odbc.md) رانندگان. -اطلاعات بیشتر در مورد محصولات نرم افزاری است که پشتیبانی از تاتر در دسترس است [اینجا](../interfaces/index.md). - -| پارامتر | مقدار | -|:------------|:-----------------------------------------| -| نقطه پایانی | https://play-api.فاحشه خانه.فناوری: 8443 | -| کاربر | `playground` | -| اسم رمز | `clickhouse` | - -توجه داشته باشید که این نقطه پایانی نیاز به یک اتصال امن. - -مثال: - -``` bash -curl "https://play-api.clickhouse.tech:8443/?query=SELECT+'Play+ClickHouse!';&user=playground&password=clickhouse&database=datasets" -``` diff --git a/docs/fa/getting-started/tutorial.md b/docs/fa/getting-started/tutorial.md deleted file mode 100644 index 595d17667ae..00000000000 --- a/docs/fa/getting-started/tutorial.md +++ /dev/null @@ -1,666 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 12 -toc_title: "\u0627\u0645\u0648\u0632\u0634" ---- - -# اموزش کلیک {#clickhouse-tutorial} - -## چه انتظار از این مقاله? {#what-to-expect-from-this-tutorial} - -با رفتن را از طریق این مقاله, شما یاد بگیرند که چگونه به راه اندازی یک خوشه ساده تاتر. این کوچک خواهد بود, اما مقاوم در برابر خطا و مقیاس پذیر. سپس ما از یکی از مجموعه داده های نمونه برای پر کردن داده ها و اجرای برخی از نمایش های نسخه ی نمایشی استفاده خواهیم کرد. - -## راه اندازی تک گره {#single-node-setup} - -برای به تعویق انداختن پیچیدگی های یک محیط توزیع, ما با استقرار کلیک بر روی یک سرور و یا ماشین مجازی شروع. خانه کلیک است که معمولا از نصب [دب](install.md#install-from-deb-packages) یا [دور در دقیقه](install.md#from-rpm-packages) بسته, اما وجود دارد [جایگزین ها](install.md#from-docker-image) برای سیستم عامل هایی که هیچ پشتیبانی نمی کنند. - -مثلا, شما را انتخاب کرده اند `deb` بسته ها و اعدام: - -``` bash -{% include 'install/deb.sh' %} -``` - -در بسته هایی که نصب شده اند چه چیزی داریم: - -- `clickhouse-client` بسته شامل [کلیک مشتری](../interfaces/cli.md) کاربرد, تعاملی مشتری کنسول تاتر. -- `clickhouse-common` بسته شامل یک فایل اجرایی کلیک. -- `clickhouse-server` بسته شامل فایل های پیکربندی برای اجرای تاتر به عنوان یک سرور. - -فایل های پیکربندی سرور در واقع `/etc/clickhouse-server/`. قبل از رفتن بیشتر, لطفا توجه کنید `` عنصر در `config.xml`. مسیر تعیین محل ذخیره سازی داده ها, بنابراین باید در حجم با ظرفیت دیسک بزرگ واقع; مقدار پیش فرض است `/var/lib/clickhouse/`. اگر شما می خواهید برای تنظیم پیکربندی, این دستی به طور مستقیم ویرایش کنید `config.xml` فایل, با توجه به اینکه ممکن است در به روز رسانی بسته های بعدی بازنویسی. راه توصیه می شود به نادیده گرفتن عناصر پیکربندی است که برای ایجاد [فایل ها در پیکربندی.فهرست راهنما](../operations/configuration-files.md) که به عنوان خدمت می کنند “patches” برای پیکربندی.. - -همانطور که شما ممکن است متوجه, `clickhouse-server` به طور خودکار پس از نصب بسته راه اندازی نشده است. این به طور خودکار پس از به روز رسانی دوباره راه اندازی نخواهد شد. راه شما شروع به سرور بستگی به سیستم اینیت خود را, معمولا, این: - -``` bash -sudo service clickhouse-server start -``` - -یا - -``` bash -sudo /etc/init.d/clickhouse-server start -``` - -محل پیش فرض برای سیاهههای مربوط به سرور است `/var/log/clickhouse-server/`. سرور برای رسیدگی به اتصالات مشتری پس از ورود به سیستم `Ready for connections` پیام - -هنگامی که `clickhouse-server` است و در حال اجرا, ما می توانیم با استفاده از `clickhouse-client` برای اتصال به سرور و اجرای برخی از نمایش داده شد تست مانند `SELECT "Hello, world!";`. - -
- -راهنمایی سریع برای کلیک-مشتری - -حالت تعاملی: - -``` bash -clickhouse-client -clickhouse-client --host=... --port=... --user=... --password=... -``` - -فعالسازی پرسشهای چند خطی: - -``` bash -clickhouse-client -m -clickhouse-client --multiline -``` - -نمایش داده شد اجرا در دسته حالت: - -``` bash -clickhouse-client --query='SELECT 1' -echo 'SELECT 1' | clickhouse-client -clickhouse-client <<< 'SELECT 1' -``` - -درج داده از یک پرونده در قالب مشخص شده: - -``` bash -clickhouse-client --query='INSERT INTO table VALUES' < data.txt -clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv -``` - -
- -## واردات مجموعه داده نمونه {#import-sample-dataset} - -در حال حاضر زمان برای پر کردن سرور کلیک ما با برخی از داده های نمونه است. در این مقاله ما از داده های ناشناس یاندکس استفاده خواهیم کرد.متریکا, اولین سرویس اجرا می شود که کلیک در راه تولید قبل از منبع باز شد (بیشتر در که در [بخش تاریخچه](../introduction/history.md)). وجود دارد [راه های متعدد برای وارد کردن یاندکس.مجموعه داده های متریکا](example-datasets/metrica.md), و به خاطر تدریس خصوصی, ما با یکی از واقع بینانه ترین رفتن. - -### دانلود و استخراج داده های جدول {#download-and-extract-table-data} - -``` bash -curl https://datasets.clickhouse.tech/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv -curl https://datasets.clickhouse.tech/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv -``` - -فایل های استخراج شده حدود 10 گیگابایت است. - -### ایجاد جداول {#create-tables} - -همانطور که در بسیاری از سیستم های مدیریت پایگاه داده, تاتر منطقی جداول گروه به “databases”. یک `default` پایگاه داده, اما ما یکی از جدید به نام ایجاد `tutorial`: - -``` bash -clickhouse-client --query "CREATE DATABASE IF NOT EXISTS tutorial" -``` - -نحو برای ایجاد جداول راه پیچیده تر در مقایسه با پایگاه داده است (نگاه کنید به [مرجع](../sql-reference/statements/create.md). به طور کلی `CREATE TABLE` بیانیه باید سه چیز کلیدی را مشخص کند: - -1. نام جدول برای ایجاد. -2. Table schema, i.e. list of columns and their [انواع داده ها](../sql-reference/data-types/index.md). -3. [موتور جدول](../engines/table-engines/index.md) و تنظیمات خود را, که تعیین تمام اطلاعات در مورد نحوه نمایش داده شد به این جدول خواهد شد از لحاظ جسمی اجرا. - -یاندکسمتریکا یک سرویس تجزیه و تحلیل وب است و مجموعه داده نمونه قابلیت های کامل خود را پوشش نمی دهد بنابراین تنها دو جدول برای ایجاد وجود دارد: - -- `hits` یک جدول با هر عمل انجام شده توسط همه کاربران در تمام وب سایت های تحت پوشش این سرویس است. -- `visits` یک جدول است که شامل جلسات از پیش ساخته شده به جای اقدامات فردی است. - -بیایید ببینید و اجرای واقعی ایجاد نمایش داده شد جدول برای این جداول: - -``` sql -CREATE TABLE tutorial.hits_v1 -( - `WatchID` UInt64, - `JavaEnable` UInt8, - `Title` String, - `GoodEvent` Int16, - `EventTime` DateTime, - `EventDate` Date, - `CounterID` UInt32, - `ClientIP` UInt32, - `ClientIP6` FixedString(16), - `RegionID` UInt32, - `UserID` UInt64, - `CounterClass` Int8, - `OS` UInt8, - `UserAgent` UInt8, - `URL` String, - `Referer` String, - `URLDomain` String, - `RefererDomain` String, - `Refresh` UInt8, - `IsRobot` UInt8, - `RefererCategories` Array(UInt16), - `URLCategories` Array(UInt16), - `URLRegions` Array(UInt32), - `RefererRegions` Array(UInt32), - `ResolutionWidth` UInt16, - `ResolutionHeight` UInt16, - `ResolutionDepth` UInt8, - `FlashMajor` UInt8, - `FlashMinor` UInt8, - `FlashMinor2` String, - `NetMajor` UInt8, - `NetMinor` UInt8, - `UserAgentMajor` UInt16, - `UserAgentMinor` FixedString(2), - `CookieEnable` UInt8, - `JavascriptEnable` UInt8, - `IsMobile` UInt8, - `MobilePhone` UInt8, - `MobilePhoneModel` String, - `Params` String, - `IPNetworkID` UInt32, - `TraficSourceID` Int8, - `SearchEngineID` UInt16, - `SearchPhrase` String, - `AdvEngineID` UInt8, - `IsArtifical` UInt8, - `WindowClientWidth` UInt16, - `WindowClientHeight` UInt16, - `ClientTimeZone` Int16, - `ClientEventTime` DateTime, - `SilverlightVersion1` UInt8, - `SilverlightVersion2` UInt8, - `SilverlightVersion3` UInt32, - `SilverlightVersion4` UInt16, - `PageCharset` String, - `CodeVersion` UInt32, - `IsLink` UInt8, - `IsDownload` UInt8, - `IsNotBounce` UInt8, - `FUniqID` UInt64, - `HID` UInt32, - `IsOldCounter` UInt8, - `IsEvent` UInt8, - `IsParameter` UInt8, - `DontCountHits` UInt8, - `WithHash` UInt8, - `HitColor` FixedString(1), - `UTCEventTime` DateTime, - `Age` UInt8, - `Sex` UInt8, - `Income` UInt8, - `Interests` UInt16, - `Robotness` UInt8, - `GeneralInterests` Array(UInt16), - `RemoteIP` UInt32, - `RemoteIP6` FixedString(16), - `WindowName` Int32, - `OpenerName` Int32, - `HistoryLength` Int16, - `BrowserLanguage` FixedString(2), - `BrowserCountry` FixedString(2), - `SocialNetwork` String, - `SocialAction` String, - `HTTPError` UInt16, - `SendTiming` Int32, - `DNSTiming` Int32, - `ConnectTiming` Int32, - `ResponseStartTiming` Int32, - `ResponseEndTiming` Int32, - `FetchTiming` Int32, - `RedirectTiming` Int32, - `DOMInteractiveTiming` Int32, - `DOMContentLoadedTiming` Int32, - `DOMCompleteTiming` Int32, - `LoadEventStartTiming` Int32, - `LoadEventEndTiming` Int32, - `NSToDOMContentLoadedTiming` Int32, - `FirstPaintTiming` Int32, - `RedirectCount` Int8, - `SocialSourceNetworkID` UInt8, - `SocialSourcePage` String, - `ParamPrice` Int64, - `ParamOrderID` String, - `ParamCurrency` FixedString(3), - `ParamCurrencyID` UInt16, - `GoalsReached` Array(UInt32), - `OpenstatServiceName` String, - `OpenstatCampaignID` String, - `OpenstatAdID` String, - `OpenstatSourceID` String, - `UTMSource` String, - `UTMMedium` String, - `UTMCampaign` String, - `UTMContent` String, - `UTMTerm` String, - `FromTag` String, - `HasGCLID` UInt8, - `RefererHash` UInt64, - `URLHash` UInt64, - `CLID` UInt32, - `YCLID` UInt64, - `ShareService` String, - `ShareURL` String, - `ShareTitle` String, - `ParsedParams` Nested( - Key1 String, - Key2 String, - Key3 String, - Key4 String, - Key5 String, - ValueDouble Float64), - `IslandID` FixedString(16), - `RequestNum` UInt32, - `RequestTry` UInt8 -) -ENGINE = MergeTree() -PARTITION BY toYYYYMM(EventDate) -ORDER BY (CounterID, EventDate, intHash32(UserID)) -SAMPLE BY intHash32(UserID) -SETTINGS index_granularity = 8192 -``` - -``` sql -CREATE TABLE tutorial.visits_v1 -( - `CounterID` UInt32, - `StartDate` Date, - `Sign` Int8, - `IsNew` UInt8, - `VisitID` UInt64, - `UserID` UInt64, - `StartTime` DateTime, - `Duration` UInt32, - `UTCStartTime` DateTime, - `PageViews` Int32, - `Hits` Int32, - `IsBounce` UInt8, - `Referer` String, - `StartURL` String, - `RefererDomain` String, - `StartURLDomain` String, - `EndURL` String, - `LinkURL` String, - `IsDownload` UInt8, - `TraficSourceID` Int8, - `SearchEngineID` UInt16, - `SearchPhrase` String, - `AdvEngineID` UInt8, - `PlaceID` Int32, - `RefererCategories` Array(UInt16), - `URLCategories` Array(UInt16), - `URLRegions` Array(UInt32), - `RefererRegions` Array(UInt32), - `IsYandex` UInt8, - `GoalReachesDepth` Int32, - `GoalReachesURL` Int32, - `GoalReachesAny` Int32, - `SocialSourceNetworkID` UInt8, - `SocialSourcePage` String, - `MobilePhoneModel` String, - `ClientEventTime` DateTime, - `RegionID` UInt32, - `ClientIP` UInt32, - `ClientIP6` FixedString(16), - `RemoteIP` UInt32, - `RemoteIP6` FixedString(16), - `IPNetworkID` UInt32, - `SilverlightVersion3` UInt32, - `CodeVersion` UInt32, - `ResolutionWidth` UInt16, - `ResolutionHeight` UInt16, - `UserAgentMajor` UInt16, - `UserAgentMinor` UInt16, - `WindowClientWidth` UInt16, - `WindowClientHeight` UInt16, - `SilverlightVersion2` UInt8, - `SilverlightVersion4` UInt16, - `FlashVersion3` UInt16, - `FlashVersion4` UInt16, - `ClientTimeZone` Int16, - `OS` UInt8, - `UserAgent` UInt8, - `ResolutionDepth` UInt8, - `FlashMajor` UInt8, - `FlashMinor` UInt8, - `NetMajor` UInt8, - `NetMinor` UInt8, - `MobilePhone` UInt8, - `SilverlightVersion1` UInt8, - `Age` UInt8, - `Sex` UInt8, - `Income` UInt8, - `JavaEnable` UInt8, - `CookieEnable` UInt8, - `JavascriptEnable` UInt8, - `IsMobile` UInt8, - `BrowserLanguage` UInt16, - `BrowserCountry` UInt16, - `Interests` UInt16, - `Robotness` UInt8, - `GeneralInterests` Array(UInt16), - `Params` Array(String), - `Goals` Nested( - ID UInt32, - Serial UInt32, - EventTime DateTime, - Price Int64, - OrderID String, - CurrencyID UInt32), - `WatchIDs` Array(UInt64), - `ParamSumPrice` Int64, - `ParamCurrency` FixedString(3), - `ParamCurrencyID` UInt16, - `ClickLogID` UInt64, - `ClickEventID` Int32, - `ClickGoodEvent` Int32, - `ClickEventTime` DateTime, - `ClickPriorityID` Int32, - `ClickPhraseID` Int32, - `ClickPageID` Int32, - `ClickPlaceID` Int32, - `ClickTypeID` Int32, - `ClickResourceID` Int32, - `ClickCost` UInt32, - `ClickClientIP` UInt32, - `ClickDomainID` UInt32, - `ClickURL` String, - `ClickAttempt` UInt8, - `ClickOrderID` UInt32, - `ClickBannerID` UInt32, - `ClickMarketCategoryID` UInt32, - `ClickMarketPP` UInt32, - `ClickMarketCategoryName` String, - `ClickMarketPPName` String, - `ClickAWAPSCampaignName` String, - `ClickPageName` String, - `ClickTargetType` UInt16, - `ClickTargetPhraseID` UInt64, - `ClickContextType` UInt8, - `ClickSelectType` Int8, - `ClickOptions` String, - `ClickGroupBannerID` Int32, - `OpenstatServiceName` String, - `OpenstatCampaignID` String, - `OpenstatAdID` String, - `OpenstatSourceID` String, - `UTMSource` String, - `UTMMedium` String, - `UTMCampaign` String, - `UTMContent` String, - `UTMTerm` String, - `FromTag` String, - `HasGCLID` UInt8, - `FirstVisit` DateTime, - `PredLastVisit` Date, - `LastVisit` Date, - `TotalVisits` UInt32, - `TraficSource` Nested( - ID Int8, - SearchEngineID UInt16, - AdvEngineID UInt8, - PlaceID UInt16, - SocialSourceNetworkID UInt8, - Domain String, - SearchPhrase String, - SocialSourcePage String), - `Attendance` FixedString(16), - `CLID` UInt32, - `YCLID` UInt64, - `NormalizedRefererHash` UInt64, - `SearchPhraseHash` UInt64, - `RefererDomainHash` UInt64, - `NormalizedStartURLHash` UInt64, - `StartURLDomainHash` UInt64, - `NormalizedEndURLHash` UInt64, - `TopLevelDomain` UInt64, - `URLScheme` UInt64, - `OpenstatServiceNameHash` UInt64, - `OpenstatCampaignIDHash` UInt64, - `OpenstatAdIDHash` UInt64, - `OpenstatSourceIDHash` UInt64, - `UTMSourceHash` UInt64, - `UTMMediumHash` UInt64, - `UTMCampaignHash` UInt64, - `UTMContentHash` UInt64, - `UTMTermHash` UInt64, - `FromHash` UInt64, - `WebVisorEnabled` UInt8, - `WebVisorActivity` UInt32, - `ParsedParams` Nested( - Key1 String, - Key2 String, - Key3 String, - Key4 String, - Key5 String, - ValueDouble Float64), - `Market` Nested( - Type UInt8, - GoalID UInt32, - OrderID String, - OrderPrice Int64, - PP UInt32, - DirectPlaceID UInt32, - DirectOrderID UInt32, - DirectBannerID UInt32, - GoodID String, - GoodName String, - GoodQuantity Int32, - GoodPrice Int64), - `IslandID` FixedString(16) -) -ENGINE = CollapsingMergeTree(Sign) -PARTITION BY toYYYYMM(StartDate) -ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) -SAMPLE BY intHash32(UserID) -SETTINGS index_granularity = 8192 -``` - -شما می توانید این پرسش ها را با استفاده از حالت تعاملی اجرا کنید `clickhouse-client` (فقط در یک ترمینال راه اندازی بدون مشخص کردن یک پرس و جو در پیش) و یا سعی کنید برخی از [رابط جایگزین](../interfaces/index.md) اگر شما می خواهید. - -همانطور که می بینیم, `hits_v1` با استفاده از [موتور ادغام عمومی](../engines/table-engines/mergetree-family/mergetree.md) در حالی که `visits_v1` با استفاده از [سقوط](../engines/table-engines/mergetree-family/collapsingmergetree.md) گزینه. - -### وارد کردن داده {#import-data} - -وارد کردن داده ها به تاتر از طریق انجام می شود [INSERT INTO](../sql-reference/statements/insert-into.md) پرس و جو مانند در بسیاری از پایگاه داده های دیگر گذاشتن. با این حال, داده ها معمولا در یکی از [پشتیبانی از فرمت های ترتیب](../interfaces/formats.md) به جای `VALUES` بند (که همچنین پشتیبانی). - -فایل هایی که قبلا دانلود کردیم در قالب تب جدا شده اند بنابراین در اینجا نحوه وارد کردن از طریق مشتری کنسول است: - -``` bash -clickhouse-client --query "INSERT INTO tutorial.hits_v1 FORMAT TSV" --max_insert_block_size=100000 < hits_v1.tsv -clickhouse-client --query "INSERT INTO tutorial.visits_v1 FORMAT TSV" --max_insert_block_size=100000 < visits_v1.tsv -``` - -تاتر است که بسیاری از [تنظیمات برای تنظیم](../operations/settings/index.md) و یک راه برای مشخص کردن انها در کنسول مشتری از طریق استدلال است همانطور که ما می توانید ببینید با `--max_insert_block_size`. ساده ترین راه برای کشف کردن چه تنظیمات در دسترس هستند, چه معنی می دهند و چه پیش فرض است به پرس و جو `system.settings` جدول: - -``` sql -SELECT name, value, changed, description -FROM system.settings -WHERE name LIKE '%max_insert_b%' -FORMAT TSV - -max_insert_block_size 1048576 0 "The maximum block size for insertion, if we control the creation of blocks for insertion." -``` - -در صورت تمایل شما می توانید [OPTIMIZE](../sql-reference/statements/misc.md#misc_operations-optimize) جداول پس از واردات. جداول است که با یک موتور از ادغام خانواده پیکربندی همیشه ادغام قطعات داده ها در پس زمینه برای بهینه سازی ذخیره سازی داده ها (یا حداقل چک کنید اگر حس می کند). این نمایش داده شد نیروی موتور جدول به انجام بهینه سازی ذخیره سازی در حال حاضر به جای برخی از زمان بعد: - -``` bash -clickhouse-client --query "OPTIMIZE TABLE tutorial.hits_v1 FINAL" -clickhouse-client --query "OPTIMIZE TABLE tutorial.visits_v1 FINAL" -``` - -این نمایش داده شد شروع یک عملیات فشرده من/ای و پردازنده, بنابراین اگر جدول به طور مداوم داده های جدید دریافت, بهتر است به تنهایی ترک و اجازه دهید ادغام در پس زمینه اجرا. - -در حال حاضر ما می توانید بررسی کنید اگر واردات جدول موفق بود: - -``` bash -clickhouse-client --query "SELECT COUNT(*) FROM tutorial.hits_v1" -clickhouse-client --query "SELECT COUNT(*) FROM tutorial.visits_v1" -``` - -## به عنوان مثال نمایش داده شد {#example-queries} - -``` sql -SELECT - StartURL AS URL, - AVG(Duration) AS AvgDuration -FROM tutorial.visits_v1 -WHERE StartDate BETWEEN '2014-03-23' AND '2014-03-30' -GROUP BY URL -ORDER BY AvgDuration DESC -LIMIT 10 -``` - -``` sql -SELECT - sum(Sign) AS visits, - sumIf(Sign, has(Goals.ID, 1105530)) AS goal_visits, - (100. * goal_visits) / visits AS goal_percent -FROM tutorial.visits_v1 -WHERE (CounterID = 912887) AND (toYYYYMM(StartDate) = 201403) AND (domain(StartURL) = 'yandex.ru') -``` - -## استقرار خوشه {#cluster-deployment} - -خوشه کلیک یک خوشه همگن است. مراحل برای راه اندازی: - -1. نصب سرور کلیک بر روی تمام ماشین های خوشه -2. تنظیم پیکربندی خوشه در فایل های پیکربندی -3. ایجاد جداول محلی در هر نمونه -4. ایجاد یک [جدول توزیع شده](../engines/table-engines/special/distributed.md) - -[جدول توزیع شده](../engines/table-engines/special/distributed.md) در واقع یک نوع از “view” به جداول محلی خوشه فاحشه خانه. پرس و جو را انتخاب کنید از یک جدول توزیع اجرا با استفاده از منابع خرده ریز تمام خوشه. شما ممکن است تنظیمات برای خوشه های متعدد مشخص و ایجاد جداول توزیع های متعدد فراهم کردن دیدگاه ها به خوشه های مختلف. - -به عنوان مثال پیکربندی برای یک خوشه با سه خرده ریز, یک ماکت هر: - -``` xml - - - - - example-perftest01j.yandex.ru - 9000 - - - - - example-perftest02j.yandex.ru - 9000 - - - - - example-perftest03j.yandex.ru - 9000 - - - - -``` - -برای تظاهرات بیشتر, اجازه دهید یک جدول محلی جدید با همان ایجاد `CREATE TABLE` پرس و جو که ما برای استفاده `hits_v1`, اما نام جدول های مختلف: - -``` sql -CREATE TABLE tutorial.hits_local (...) ENGINE = MergeTree() ... -``` - -ایجاد یک جدول توزیع شده برای نمایش در جداول محلی خوشه: - -``` sql -CREATE TABLE tutorial.hits_all AS tutorial.hits_local -ENGINE = Distributed(perftest_3shards_1replicas, tutorial, hits_local, rand()); -``` - -یک روش معمول این است که جداول توزیع شده مشابه را در تمام ماشین های خوشه ایجاد کنید. این اجازه می دهد در حال اجرا نمایش داده شد توزیع در هر دستگاه از خوشه. همچنین یک گزینه جایگزین برای ایجاد جدول توزیع موقت برای پرس و جو انتخاب داده شده با استفاده از وجود دارد [دور](../sql-reference/table-functions/remote.md) تابع جدول. - -بیا فرار کنیم [INSERT SELECT](../sql-reference/statements/insert-into.md) به جدول توزیع شده برای گسترش جدول به چندین سرور. - -``` sql -INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; -``` - -!!! warning "اطلاع" - این روش مناسب برای شارژ جداول بزرگ نیست. یک ابزار جداگانه وجود دارد [تاتر-کپی](../operations/utilities/clickhouse-copier.md) که می تواند جداول دلخواه بزرگ دوباره سفال. - -همانطور که شما می توانید انتظار, نمایش داده شد محاسباتی سنگین اجرا نفر بار سریع تر در صورتی که استفاده 3 سرور به جای یک. - -در این مورد, ما یک خوشه با استفاده کرده اند 3 خرده ریز, و هر شامل یک ماکت تک. - -برای انعطاف پذیری در یک محیط تولید, توصیه می کنیم که هر سفال باید شامل 2-3 کپی بین مناطق در دسترس بودن متعدد و یا مراکز داده گسترش (یا حداقل قفسه). توجه داشته باشید که کلیک خانه پشتیبانی از تعداد نامحدودی از کپی. - -به عنوان مثال پیکربندی برای یک خوشه از یک سفال حاوی سه کپی: - -``` xml - - ... - - - - example-perftest01j.yandex.ru - 9000 - - - example-perftest02j.yandex.ru - 9000 - - - example-perftest03j.yandex.ru - 9000 - - - - -``` - -برای فعال کردن تکثیر بومی [باغ وحش](http://zookeeper.apache.org/) الزامی است. تاتر طول می کشد مراقبت از سازگاری داده ها در تمام کپی و اجرا می شود بازگرداندن روش پس از شکست به طور خودکار. توصیه می شود برای استقرار خوشه باغ وحش بر روی سرورهای جداگانه (جایی که هیچ پروسه های دیگر از جمله کلیک در حال اجرا هستند). - -!!! note "یادداشت" - باغ وحش یک نیاز سخت نیست: در برخی موارد ساده می توانید داده ها را با نوشتن به تمام کپی ها از کد درخواست خود کپی کنید. این رویکرد است **نه** توصیه می شود, در این مورد, تاتر قادر نخواهد بود برای تضمین ثبات داده ها در تمام کپی. بنابراین وظیفه درخواست شما می شود. - -مکان های باغ وحش در فایل پیکربندی مشخص شده است: - -``` xml - - - zoo01.yandex.ru - 2181 - - - zoo02.yandex.ru - 2181 - - - zoo03.yandex.ru - 2181 - - -``` - -همچنین, ما نیاز به تنظیم ماکروها برای شناسایی هر سفال و ماکت که در ایجاد جدول استفاده می شود: - -``` xml - - 01 - 01 - -``` - -اگر هیچ کپی در حال حاضر در ایجاد جدول تکرار وجود دارد, اولین ماکت جدید نمونه است. اگر در حال حاضر زندگی می کنند کپی جدید کلون داده ها از موجود. شما ابتدا یک گزینه برای ایجاد تمام جداول تکرار شده دارید و سپس داده ها را وارد کنید. یکی دیگر از گزینه این است که برای ایجاد برخی از کپی و اضافه کردن دیگران بعد یا در هنگام درج داده ها. - -``` sql -CREATE TABLE tutorial.hits_replica (...) -ENGINE = ReplcatedMergeTree( - '/clickhouse_perftest/tables/{shard}/hits', - '{replica}' -) -... -``` - -در اینجا ما با استفاده از [تکرار غذای اصلی](../engines/table-engines/mergetree-family/replication.md) موتور جدول. در پارامترهای مشخص می کنیم مسیر باغ وحش حاوی سفال و کپی شناسه. - -``` sql -INSERT INTO tutorial.hits_replica SELECT * FROM tutorial.hits_local; -``` - -تکرار عمل در حالت چند استاد. داده ها را می توان به هر ماکت بارگذاری کرد و سپس سیستم را با موارد دیگر به طور خودکار همگام سازی می کند. تکرار ناهمزمان است بنابراین در یک لحظه معین, همه کپی ممکن است حاوی داده به تازگی قرار داده شده. حداقل یک ماکت باید اجازه می دهد تا مصرف داده ها. دیگران همگام سازی داده ها و قوام تعمیر هنگامی که دوباره فعال تبدیل خواهد شد. توجه داشته باشید که این روش اجازه می دهد تا برای امکان کم از دست دادن داده ها به تازگی قرار داده شده. - -[مقاله اصلی](https://clickhouse.tech/docs/en/getting_started/tutorial/) diff --git a/docs/fa/guides/apply-catboost-model.md b/docs/fa/guides/apply-catboost-model.md deleted file mode 100644 index c72b05d4843..00000000000 --- a/docs/fa/guides/apply-catboost-model.md +++ /dev/null @@ -1,241 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 41 -toc_title: "\u0627\u0633\u062A\u0641\u0627\u062F\u0647 \u0627\u0632 \u0645\u062F\u0644\ - \ \u0647\u0627\u06CC \u0627\u062F\u0645 \u06A9\u0648\u062F\u0646 \u0648 \u0627\u062D\ - \u0645\u0642" ---- - -# استفاده از مدل ادم کودن و احمق در فاحشه خانه {#applying-catboost-model-in-clickhouse} - -[مانتو](https://catboost.ai) یک کتابخانه تقویت شیب رایگان و منبع باز توسعه یافته در [یاندکس](https://yandex.com/company/) برای یادگیری ماشین. - -با استفاده از این دستورالعمل یاد خواهید گرفت که با اجرای مدل استنتاج از میدان از مدل های پیش روت شده در خانه استفاده کنید. - -برای اعمال یک مدل ادم کودن و احمق در خانه کلیک کنید: - -1. [ایجاد یک جدول](#create-table). -2. [درج داده به جدول](#insert-data-to-table). -3. [ادغام کاتبوست به کلیک](#integrate-catboost-into-clickhouse) (مرحله اختیاری). -4. [اجرای مدل استنتاج از گذاشتن](#run-model-inference). - -برای کسب اطلاعات بیشتر در مورد اموزش مدل های کاتبوست مراجعه کنید [اموزش و مدل سازی](https://catboost.ai/docs/features/training.html#training). - -## پیش نیازها {#prerequisites} - -اگر شما لازم نیست که [کارگر بارانداز](https://docs.docker.com/install/) هنوز, نصب کنید. - -!!! note "یادداشت" - [کارگر بارانداز](https://www.docker.com) یک پلت فرم نرم افزار است که اجازه می دهد تا به شما برای ایجاد ظروف که منزوی CatBoost و ClickHouse نصب و راه اندازی از بقیه سیستم. - -قبل از استفاده از مدل ادم کودن و احمق: - -**1.** بکش [تصویر کارگر بارانداز](https://hub.docker.com/r/yandex/tutorial-catboost-clickhouse) از رجیستری: - -``` bash -$ docker pull yandex/tutorial-catboost-clickhouse -``` - -این Docker تصویر شامل همه چیز شما نیاز به اجرای CatBoost و ClickHouse: کد در زمان اجرا کتابخانه های محیط متغیر و فایل های پیکربندی. - -**2.** اطمینان حاصل کنید که تصویر کارگر بارانداز شده است با موفقیت کشیده: - -``` bash -$ docker image ls -REPOSITORY TAG IMAGE ID CREATED SIZE -yandex/tutorial-catboost-clickhouse latest 622e4d17945b 22 hours ago 1.37GB -``` - -**3.** شروع یک ظرف کارگر بارانداز بر اساس این تصویر: - -``` bash -$ docker run -it -p 8888:8888 yandex/tutorial-catboost-clickhouse -``` - -## 1. ایجاد یک جدول {#create-table} - -برای ایجاد یک میز کلیک برای نمونه تمرین: - -**1.** شروع مشتری کنسول کلیک در حالت تعاملی: - -``` bash -$ clickhouse client -``` - -!!! note "یادداشت" - سرور کلیک در حال حاضر در داخل ظرف کارگر بارانداز در حال اجرا. - -**2.** ایجاد جدول با استفاده از دستور: - -``` sql -:) CREATE TABLE amazon_train -( - date Date MATERIALIZED today(), - ACTION UInt8, - RESOURCE UInt32, - MGR_ID UInt32, - ROLE_ROLLUP_1 UInt32, - ROLE_ROLLUP_2 UInt32, - ROLE_DEPTNAME UInt32, - ROLE_TITLE UInt32, - ROLE_FAMILY_DESC UInt32, - ROLE_FAMILY UInt32, - ROLE_CODE UInt32 -) -ENGINE = MergeTree ORDER BY date -``` - -**3.** خروج از مشتری کنسول کلیک کنید: - -``` sql -:) exit -``` - -## 2. درج داده به جدول {#insert-data-to-table} - -برای وارد کردن داده ها: - -**1.** دستور زیر را اجرا کنید: - -``` bash -$ clickhouse client --host 127.0.0.1 --query 'INSERT INTO amazon_train FORMAT CSVWithNames' < ~/amazon/train.csv -``` - -**2.** شروع مشتری کنسول کلیک در حالت تعاملی: - -``` bash -$ clickhouse client -``` - -**3.** اطمینان حاصل کنید که داده ها ارسال شده است: - -``` sql -:) SELECT count() FROM amazon_train - -SELECT count() -FROM amazon_train - -+-count()-+ -| 65538 | -+-------+ -``` - -## 3. ادغام کاتبوست به کلیک {#integrate-catboost-into-clickhouse} - -!!! note "یادداشت" - **گام اختیاری.** این Docker تصویر شامل همه چیز شما نیاز به اجرای CatBoost و ClickHouse. - -برای ادغام کاتبوست به کلیک: - -**1.** ساخت کتابخانه ارزیابی. - -سریعترین راه برای ارزیابی مدل ادم کودن و احمق کامپایل است `libcatboostmodel.` کتابخونه. برای کسب اطلاعات بیشتر در مورد چگونگی ساخت کتابخانه, دیدن [مستندات غلطیاب](https://catboost.ai/docs/concepts/c-plus-plus-api_dynamic-c-pluplus-wrapper.html). - -**2.** ایجاد یک دایرکتوری جدید در هر کجا و با هر نام, مثلا, `data` و کتابخونه درستشون رو توش بذار. تصویر کارگر بارانداز در حال حاضر شامل کتابخانه `data/libcatboostmodel.so`. - -**3.** ایجاد یک دایرکتوری جدید برای مدل پیکربندی در هر کجا و با هر نام, مثلا, `models`. - -**4.** برای مثال یک فایل پیکربندی مدل با هر نام ایجاد کنید, `models/amazon_model.xml`. - -**5.** توصیف پیکربندی مدل: - -``` xml - - - - catboost - - amazon - - /home/catboost/tutorial/catboost_model.bin - - 0 - - -``` - -**6.** اضافه کردن مسیر به CatBoost و مدل پیکربندی به پیکربندی ClickHouse: - -``` xml - -/home/catboost/data/libcatboostmodel.so -/home/catboost/models/*_model.xml -``` - -## 4. اجرای مدل استنتاج از گذاشتن {#run-model-inference} - -برای مدل تست اجرای مشتری کلیک `$ clickhouse client`. - -بیایید اطمینان حاصل کنیم که مدل کار می کند: - -``` sql -:) SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) > 0 AS prediction, - ACTION AS target -FROM amazon_train -LIMIT 10 -``` - -!!! note "یادداشت" - تابع [مدلووات](../sql-reference/functions/other-functions.md#function-modelevaluate) را برمی گرداند تاپل با پیش بینی های خام در هر کلاس برای مدل های چند طبقه. - -بیایید احتمال را پیش بینی کنیم: - -``` sql -:) SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) AS prediction, - 1. / (1 + exp(-prediction)) AS probability, - ACTION AS target -FROM amazon_train -LIMIT 10 -``` - -!!! note "یادداشت" - اطلاعات بیشتر در مورد [خروج()](../sql-reference/functions/math-functions.md) تابع. - -بیایید محاسبه لگ در نمونه: - -``` sql -:) SELECT -avg(tg * log(prob) + (1 - tg) * log(1 - prob)) AS logloss -FROM -( - SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) AS prediction, - 1. / (1. + exp(-prediction)) AS prob, - ACTION AS tg - FROM amazon_train -) -``` - -!!! note "یادداشت" - اطلاعات بیشتر در مورد [میانگین()](../sql-reference/aggregate-functions/reference.md#agg_function-avg) و [ثبت()](../sql-reference/functions/math-functions.md) توابع. - -[مقاله اصلی](https://clickhouse.tech/docs/en/guides/apply_catboost_model/) diff --git a/docs/fa/guides/index.md b/docs/fa/guides/index.md deleted file mode 100644 index a37a8089b9c..00000000000 --- a/docs/fa/guides/index.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0631\u0627\u0647\u0646\u0645\u0627" -toc_priority: 38 -toc_title: "\u0628\u0631\u0631\u0633\u06CC \u0627\u062C\u0645\u0627\u0644\u06CC" ---- - -# راهنماهای کلیک {#clickhouse-guides} - -فهرست دقیق گام به گام دستورالعمل که برای کمک به حل وظایف مختلف با استفاده از کلیک: - -- [اموزش تنظیم خوشه ساده](../getting-started/tutorial.md) -- [استفاده از مدل ادم کودن و احمق در فاحشه خانه](apply-catboost-model.md) - -[مقاله اصلی](https://clickhouse.tech/docs/en/guides/) diff --git a/docs/fa/images/column-oriented.gif b/docs/fa/images/column-oriented.gif deleted file mode 100644 index d5ac7c82848cb633cb0f1b341909c0b955e79fcf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 43771 zcmd42RajilwlCUP&_GCn6Ck)lNN^AC8XOYb-M#T{XxtiicXv&2_r?kC5?uS{zxH1H z+=sjFcOK5enJ=?y%%N4I>YG*dD=8UC9$rH?L_~xe1OR*u2>%U)4*}pE0C+C|-VcDE zcf&8c;k!WiC;)z(4PO+24+G)TK=@%dd=Usw0l@PB@LB-83;OZv%_!a;VY`}^Gf*okB7sO?d`3DgM;hq zOCK+f;-Z4|w3M~ARb6e(nVD%7C3!OwqmGXDxY+2y!2w5m+q&B7u+X5Fot^vZW4Md+ z(}vyS-Jh$Im6zwomyQnjMH8Hi41RfeeYOGD)Pz4B-Tv8!M3YwYwPm9?f1D;=W5pU?&{g~pZllD-L1RhR`|Xq{4@}Lkp%zr=_NV& zWr`R+js-u6j%|-?`ej$#c(Ey#=It!!a@`R9(*3Q4z{K)h7=yQHg?W@9)gsHCiX_Ae1FUTkl83H z{>9>AEl4T)&j*TsY>L@CnNo1Fa+Z=H>uhmC`sg`Jm= z^D`ei2gQFslz*)`nV9jZic9{tt-m!vN(&bk2R=47cXxMI_s^{MPUdW%czJo**g4oZ zI9UF2usD0#xfpt|*f~@ETZ6c%v$2z*gE`=T03Xf ze}%(pY;VivVd%j2iIx4Ikp9i6pz#0CYHRyH+|DklrvJ;||4+rv>Yfg!Y^tWt_O4FG zrhnO~{xRjiC+1{o=wk1rZf|e%?^aZ{uy?U{wy<}g5EJ`Xt5MJ^7#dsJ{X=BMzhzDSm#pZ2l>H|mY#shu7B_XWbTc)Pbh5Xl_*dY3mjBro?*FLo zzhzDSvoAdVQI_qmGi?8K@Bgd2|4sUvL;n>2Cwc!?{wMuS?f&Mx)8FJqfWJIHJwDvu z-QHYZUH-W^KRZ1+J~}+u-`m~U-rD@VvA(vtvb?kiTbQ4lotd7RoERS)9T^@P9O&=+ z^|QC9yQ{OKy{)ySxv8&_?ryFw&Q6XF_I9>5)>f7l=4PfQ#zuxez8mQ4 z>FQ`}X=;2^S5s9{R#H@umy?x|mXeeZ7yBwIA}k~*z|Y6a!_CF{9G$T-PN(EYVG4a2zBMQXEvHqTE6*`jwcs78$JI zvtqFu91yIFL%kU27f4S(gi9zIl^*Q)d37{W6i9l3{u9JQCM(EQi&|$eX-^dZRUZ(O zk)aZVhvpx*{*oH0llAF-?U`7rT5W`cN#+Rx@x*?KeTpcf9(q&F5#fNhy_Jhg863*< ziZZzH4UKg0_7ws$*o07g_xcS-)u*`jwBgWSq}!;t)Pf+w7Se?aIwo9i#(q}|&)-rb@(~`2y@0C@HiA2fdsAxnDos!rO zOau*jm6N#+sBe+X-^5B5Ii3&Z1^FyVF16`3pR2^+sBQlCk6+ zzLB_V{UHOB86pOvxa)xEU}k#odq-qZDFvPHcKr&--=(>Tf1A{ADq#Nyn@@oDR1X7rt$?9!LJzF zxAAamc+x*i<+tq7K#GU2xFScf`QO*Ze4`zI=k!KkTm91KAb9%^?`KEBU&@OvkyEhi zAN8j`w+4u=A00g}VbZsM09<~CK~ha|#3a-BcIp?O)dt)(lyocl zpttRN@BKnVrdhExO=|RQl z(`L0^@3U_Ihey7jsAtj_{dDAne@eS5pQuNL*`7tGG}WH3g7l)FZ{{ts+Adcd_LUH)s zqWe{8U_dW0dxv_z8zU+(u&bM*ug5X?!tdqZc<+ml5?+9$X9KWf2F0nDx?d?ka^I)i{bJCW z0)1=W#H)je(&oN3`t8+0uUl0`P8`ggEBg`aY^me)Sri><(+Ebv6)ZX z)M5<7Dt5IA83LvZGA**-a^vDEN2!@4ct@1xIbxf(O;}BdqiFQml;aA3>QF3B z_aF^Yy~@S!pqHd7oeDLp&%=JEmu6s`j*UymqrIk=RnSpQs{B%b=UXDF%cq(eI9rI( zXDQ{%raIdpVt2PPklY?dP-=czr2v!{TfO z*|&79$Rhg4)NlD-bE!J6j9MdB^Ob&TrRuh&x;Vocz|MwBw>~<~z6Y7w+=dFB3~Rla z!}+q*Zxyl%HU>Xi7y2p_GyVIlc}O0$6lvcViuT!ucbYHaL^(0%d{mnXb*oQ3 zv9S!XT8fNIDk^<#X|3-)vnVv%i22P{B{FQbm$I=oD#OMB4^yidTeErK1O0cf_rf3_ zcj?F+%~wzBm99ZkyYAPa=WyGPqd$;}=6i62 zPC~vfFJVwMcbA%0gWk0-Z;UOp-M_B(Qo~w*e9`Qlwyg>`K3>0Da_=7w`2>6%Hn<;d z?i=g72>12bB*97Pl3x215xw%AVj0>mgY887d}u<`1?%H8b4r}jHD>W98rFK@NOmUO zVdf?pwP>Uc5QY4nvxANm`_=^u9xtLa5h1wrUuFm&t71R4j5|d+D{0GF#3T`VNYK{C z^fB)+@p}y!hBD_9`R+={wN52Wo9EI>VMXXb#gy1B>r{tiW2NJXwPEto>Nf@M_8l*MSIwo_Iy>$} z`;H^OF7MSFp~jkk5B3J|whdiHu5Rv=BOh`BgdttvFFM7GaNYDx4CU@V?r}%3dHWUx zoM%w1@*-AOZ=1G@XIO3RBC)Q0hsOVY$oR!6c_Dq5>N&Pd3bJHfP)gs`k7uH1IH25am)~?t7$K<+YG}>Kf z)u1VGg88Ox#@f~)aDDj{W1y{j=e51Y?ehY{1b+>>+#TEvGjla|#k~@K>h4dqcH!m! z^b>d7eu&NI>^m#hI*GsMFcZIr-azv*Sjej(Oz5VK^+VfA* z^w`{Uh~bv+|LFr;@cvQcyGrMavF~FX=F@KO`+m@u*F0b%Ip8hSH$Ni@^wPXy$jvEcvMqZ)=8VO4|!d2I~<9fAdgywQono;t6xn3L}{eAlvu0E)J_%2!PSKIS_yfnu40+5gD=(LqVW0C_*ttX+F=o$cM%;hL|CG~cCj@1Bw{xd(sPF(PYC?l95Pr8nche2 zg@q^-2W#K>opXkkB?-*jMYd^0?rXpyU>C^7T?9)vWb+mxW)Yn6K5}nAVzW6)`OaTY z0w{DBydv%2Qy2bn7hM7iH>VH4v;YGy|;fO@x1eQ*(?qf@#G1V49n4W<|cQLe{ac@iFnk*vvHG_Deu_Pt2q{$K0 z@1m!9qUU)+(0@b;5k(zBqZg!uf6>dGYbDHr^dF%K>cx?X1c|30>9zfYeIAG=EO7^x zZ~=`Pqfa;iMV=xj385#5+=L8BBu1Dgnt_4OcX0rsIINU7D$O_-1|)jRct3D5FS-kB zOR^79JhvzGi={X7eR4Pv9Lm!Y|3L;?P7ls&hN>1NyDTPCfuR^JDU94uZ_8wec6<^; zd^{So_72KjoJuedUy_%odth)Ql_0^8bcGf=u9dVKoMeceSUc}$3yxmCOM6OAtR9N6 z@=TLskgZQilCt#Il1`HEOWT8{BMc_R>?Ykwrg!6~#1X|Lq=XfwI6?2@QngbvTH?0A zF4-w5p()UF-gvD2l#-Uz8||zZOM!LtcqDKtsb(gvXXepR$_qG4U?Fq&KJ!mWJf=>{ zzD#!HP^vGXPrqln5<^-dfy@|D(nLy*PH~#LN78_0269+NRAI*Jfs6+;0sQcEJ?-db zUKtXdj1Q@4nkDdrNuJzJ%RF{QPkJv02j1YdmaLG)th|(Dw8L0Lj2xt;{I|oIj|{0- zo~dfYIhXg@7c!|>R@oK2`6|Sj79J@w7_oAx1zI}U@0Q}emuBi6LSHjxDyA0dG8Wm( zI#z=DScs$8ta7WqbxIsPgQH|i4IVO7!wWT2odFNAraDDtIt4C=&c>zr7<@%a zURl`0MLNS}U9Dvb$)#~ErBX|U)ShLo_u%sIa&x}2nxb;tp~8mY{O*S^KP|5+i;ADb zB`ka;0*7#oWWLB0*%HWb)J#d9sAcrQP2Pe>Y1*(fUTNhPC~xZ_o>MmI!m2X*P&!Mt zGUqU)d$HtsshIJ>;g>A5U$(4*(YdNLf7GkIcDSO~%V}CCx#h61)vIDLHFHjd=Y_8mFkf6Z7F1D zIFG`svQ@kIN;W$0(BqWR=KL`D5+iLRwHU@(KcrRvaF~dkRu6xOMi2BboiR1)@h99zq+eNg7lgM}$tj|fLOMwD zUbov-$n~f^cE2HLYmdm0jOadX>kjnl&P&Vt(B5^!-?;DIQ^?r9D&4+L@^fRkeLfW4 znUB}Ok>1Ysqhn3d@XN8y1i#v9+Rs6f&V{s2(RQ^D^X-%PMuc2ORHk;#V%^?+)qT`m z`}Do!rM=kgZKwP_(#u`9pkB?Cp1of^-;aB~g8Lff`l&|yq|55*M(Z`0O_=riS>=03 zNCj{w2mGD}fTV-|D}Zn1>EXx#EC51y`ygax5Hbn?Z1n4<_qId;z{i77$PgBwKfkO; z?Y@hFxVxaAgpqth+j)tDng3Xw_H{3_Nu*Q1IM`!eU^wUO<%?GbN241g>=IO!xpS@C!1&R33dG;DM=MvL`B{nuvy)~jSGM#s%+bciBAU~5)I+vw4t>-g+{xoX9GTv=7u5mKChc!iI>!CL` zj`v*2qu)dHJk_VSp#3~gjx`fRGeZQKd*PqQ!J5x}CGh8XSn7GEr5!%|s0Ry=oJD$u zNv^^`6|;yPF!>BvlAwxm2A^ujG>z?y<;gTf$2iLJ`~}9+DCxpijB(G7sr<+#HkKu; zv4zseQDf3+Bdz7?v}xeUg6ZnC)`#UD!KIsp1-Fhdk4US)XU|~42}s6lO61~K{l&Ns zi(v|jvl)gnEUWP+vsX{HQ?J+B1Qw`lmfTmDe^xA@bB||!NC##tp9#POq~}T>m%pt} znk)2vQdp|AUK@N~tHEC%6U`wy zn2O&DAJ$fUXX~F<3H(-JzKb9HHs3ZZ7SAl6KQ0bsOdfP>Yzx3=tUH36KWw~x8Ea>m z85mnr^O(Pnoc|@b)o;6TtI$7EVOHMJ@FKYVIdcu8bKyyF^yPVhdwl+N=fIdPjPi6< z-)@rxdzBP>m;Q8T?!zYk>M!~9AIi+Tm<5|TKR2yLUyi9aOoeHnyiJ00_I29qVIQ7% z9{v$JyE#3*h&p4#x%>h@1!@_arWwf>gvoGR=1o8S3f^7${J z-_1csog-6(@P112ahJ@a-l+5Rmn%Nj`PXa*NJO3!K1r%v16N&5kp-un-dU9Wu@~MI_P=zGzCB*M<$yob z+uu5_A67T)y4pXs2w!5E9wkv-57-^NC%;b=ei(p5@F(uJkFV%)t~*wOlHp6=golQO zZzc@RvvDp^25+Xx=Vre@kguN)`CofS-K$mJPpn^0!LQ$U{`oX0UJI?qH(C z^83^9>C58r0oMA(E2XPk@;~pYp7AE0juBtpKDu)Jg!O?% zP(k1J!%194$JlDesu5|r2!v_vZfdjQ<=N1Vm{swVRTwK89Mf5OhIsmdZ4-<8|-*dQ?}qRuV% zi=no(!YAJLr@(`wnQ}&pmEkz_zwI|g1UcDKpy7A>Xl*S1h)5Qw0GGyAAo~#$esnG}EeiKGMyL){Fhh-6qA}FWOB{DU!D3hZDCEEW|61F!O!B>VgSdStsG$-rq66X8M1Bm5%N~5 z^-3bjsygI4Vs5E&^`@tVjUUA~x{ep&Ru5zFpVo*xc#;Z<=Q7#3oGvw)w9OJ@(R5&o znOLbNe1(-#<8ddJ6k@89mT3`;3_AuCSgm}DXu#LvC}Z*)bt*4SO5d!&nGlHVFsM9A zYdHckRkS^0^Fzkta95n$CB-mZdPIB6svEutKSRbk9@4>`UVqA5X4D=>rn~j5^}oQ* zLRq-dFw;)H%vv!~an`6C8(j+tWMy1Wxwunbjf=03IZPX0u5!-=);$g`g)+UX31kQw zUD(u(OyB6Q4m@PsNemk$*d|h3xf{xecwpR*pMWzGj@AtO@XXr!D{wJ5%7wZf)hgqd>8k_{7hu^rCH`g#}e3j-=nxd4qB5R7ope^ol>qE!4KS} zOcstvv2&rv)JG!PuR%GT(70C1xIGQ|Y@b-@GxkVyBv9PBFLC5Ki?ntnYtj2&qbew* zjAAUDP7qJS)|X8?^HB?}_tG{vbnTu3-M+b(;!ru14r}d<@OLkP!MWKdaWb4T6EV`< zbCB@+zIWE!VpNEqLM})Q@9NPcrKugwZq?wjh~HlIbY0myr+fVXOcZxb{N4cqHq-YM zkwthp6r+9Q6OmP1T0aS4g&N3{GxjH^(Gzoog{@mx-HB2(F{10)xIN(*onK4uLH8;1RvUx5=d4Ki++q0^&e_ zKz}h8eu}t9P=0zFADOTt`Qu}R;$*K!+20L#{gj4#mFT2Pue=>r+HQ@8Q72qVbKXzx z1L%dH0*{dMksgZ`-TL1}n~HwmCU4+u;*1Z6zg?jGj6knT&$X)7mKB_uN8S)gCcNk?)C=xF?R{Fi4Gjdp8n`Fyqz5`Zwv73+1 z`tU=LoQbB>uc0~+h1q(Q7*^goU+rjXYb(>D6|}~gR~#8Osd9itX*~+egQq)5zZi&1}hnTd85Ew z58=ZM8*izj#fNEv&P$e3zvq#4@<+{*+ts|dNIg8jS@S1GvST+#pB-H5lxm_v$3SO! z-2e-Zg12rL2@PsH1VP$8TxOIp^P{`|2HO41q@Tk^Moq)gwYx1kYEtX4_5{A)b$?5A z&OU%FbKT-i@U1!9daZ0zz%ds)V_BUF!mvzbs8Xhr#_JKmJ|;6pOG6s5wR!$=yEt`P z1zIRyAh@Jv-%6K80vlOCdp1T=8B3o^{H{yumJdbXM3W#lvzn?B3#8A)^Pa1mRgVJS zwBfw49yiW9>&TtTX=1-2v-4W}4m5NASJNV74TXZw+j8y?EBW8JTjQTjUxwROb|tq8 zPkeP8KNl`E;=4iGPWIT$A7NHB9#RM|Q+f;P>tBDnmq(E3SrLpZZFIUdve=y%EfX!T zud)6l!9M9UBB>SdvO+p7MV%!0sPe&_9u#oA>S zlHmS1D_;@bJLhbkC-^2yRr5^KlS|S&{vEaguUsSpqov=}2gTSe-FZ=t)oP@NhUCq2 zk-lyw#)7>VbW95jwZP^P-&2>W#}&M6K--b;nd1QeI{fsu3rXlKu*!P_)Bg4+Rpwdv zcY!VRGmoEQYZsA4Puo=Mw3eHU7d?*ja8TDyL${V>>| zXCaoXfB3@H{G(%HCr0zX*c>BIy7sD3As%s82JH5#-s6>dU?FPcj%V2lQL!c3Y%#T4^#B zff-#SyQNV3v6}n0Hw4X31{vL22le~ezhb91w_w`~B1;PHmGFC*3Z4T5;@$bPNCWYB zfJZ=Xg;>mpU6^B|#2NBSN zf<)#o+OQ%c1;Aft2)Ncu7j^VG7Vn-+vQDl81w;Mrq25F(ro1U0+9^?sP&?0*5E*Eib`mB< z(#RXMe}QmN0jU3ga3leY2p|C96}&shL7JI3dNAK<$c8U#ve-qoLI)=t;x=;Pg*S&M zCr_3(vhTbzY=^^!m*k^ZHN!;Ess^-!eQrK0Pop(z9w2 zN>^ux2BCeTSJ&Ftd&lGQ=l9V7&JUD8CJY^sV#AL=S+K?a+y~hkb@|h)OdH{&Q?>tV{1asW&zoV_Ua}Mrc5m$YyC`Aa8 zlVjy>^Za;uz07@%)caN*7dGrE!zaOVmYdEdp?#1 z{+iUVx=z3FO`Yl!zRSKFsnEVb=5nR!Xez&)@E=cgW80775Nws!pO$kpT1_vs_HW0k zxSkQ-)6eaxXIcgsG_%iAg<8JZla@bYD^3|FC3WXoupWYEIXLZq(xdqeg#5PeP2d-@ADe-O1=Ql& zj^gy$uf7h^>r(X&a%$>(|DGo{^}5*A<>_hpN*X)Rl1CaJ%ej}zot(%$WX7qesho0DK^y*r2??mjI0O2gq=R*i2udssi%Yei9!u?jMZqCdIIjwV^BYKv^X9W z)y}plzio-j(&W5odKbabX5|c72xpF~P~_B_4>A^ql|fjI3Fk{sJqgJ{L;cM*;}!Hf zHePB~JS=5%Z#yQ-C6)3jW{o_Tsj93{S|)q1=DoLny2(B%WS(Jn7t#8q>i*%n{VI^M z7WTEsV(DWnGx6|xi2&J%-3y2A&qY{L{%_xpYb1SzcOR-a14kM@C4~_@q-REO2t{te zBxD_W5-5G(3|%Tg9hpU6Fa@tbm?^|66F;hk+a~8TPab%s)$*^V+xRM`np=WUSmp+% zN!{nWKbO^k1%fJa>&xQ=1tD9*t?j(lMNlT~_1?&aDi?wSt=2mygT-H6AusOa0mfS0 zXahB~o)Ee4qlE`O1pWZg@gHT{L_F-@gqNC;{2xX%+a3rCB4QPRK_mSKvjsiTG}Ui0 zO*V`UY1tQD>6}pGvy)5r+E?)+B$j3d$7BoY)J_J>`WG$7u-=s1E{wr2I<-q1Njtnl zE%1cE?LsJAjM&KH6|#zoDBsiRAzx zEp7r9QAIQ5RqNwdQr|3c@IRfKawQPq`MURQTswGRs^Zo3Cy50HtN5mq4uRutI`?yS zEHMK%)$%>W5JE5@IzP4Rezf#iAVliyZLQT#CXFy9T+o_BvWWX6ghxzmVhIzr5Izox z@_W6BcE!x;z|i_C+*#&2XA(rAaMQgs6cf1i6wmh?Hs~t&$9bn7MlB3gl&g)Gad$HB z+aQ|IERT`TO4t9$&D3;PV!0F6Pmx9+84DzPLz|hv7LhfszqTp~qA>TFN{T)!fAXPOH z8xIgpn*DPVq&#s^VU)|i*Uk)Wk#n>0$9SEhQnevNP{Wd`_hOK0(6(kq8K!Qn$e?;i#p*HBFJfzOGeia2KZF=p1U3VJV6ABv^Ff0J)c*4<4Q)zis z9Kc?OWy*ZV<-yS7Opq@8)T!2`Mls64I+}W}Jd8|p4(w4BQ30q0zD8gNA|Q$wFCzjQ zlfe~HrV+*@5zPIcv4OUZ?OPk;jg8_cHMADY+tAMX0*U#HZxUal7-zNs8kJ70C^6xz zMS=YWl6~-mm2g1t{ir!@UFyQyFMFL zVgD=1!z4(tK9^u@Uy}b}DkiEvpHAUGR`+2VI$mGMJ$9hz{Vn# zE>1=@))y(9Sm{14!TyHQG8({ zLfUu=s>R)QYYSy^AR0vhH8CLT{jgW(sfsgEM*s*pn7?V>uw13iQ4x<1yLQhbEJ|E&H(5)!$}qx*1>^(IjJH3&Z6?MFDFF8pa8lOKZX z@H5lV_8kJuyQ`49`3UDUt?l|vci_C)5pXUa@yx~moIA9CKx0Y+W2I|y$F90)VBTO) z8)&n?iS?5Dn0e2ngBHh?L$XjIsn(`5AC<7(oycqJ3(?)CO>UyNX4yAvn~s0SJyEz$ zFU<=G+~LA29e?IM%8S?rym<)>O)k&b#GY?y%Q;c) z;JMd(`dOKGXl$9;H&hfJg6)$6x6dKdI8Xil_8hn2?@pC91pq|$?rso=`z?sTz?RQT z)7$4vt0GUH0v)+IoibE?W>=GG8nrKY-bm6w@&YeZ=nr^>yGd1;IW|m1&eWR}=sk|2 zFSg^okgT{!-~+?+U*q=Q2;&_t65cM-aGUpk;q*mU^F2=HUD|UwU-0c;cs017=|Nyn zcIoPK>l3}@UUu<)w#be9PK)=>El*tg&3ur}c@XitU^aTkPZq&7YMycNZr>?^yafR$ zcb@OGRB4NV(*3}KBHa>{0Djld_fX(xRPUk!y~e!&HI%@w?m?-M*V`22gga0o~<_3nXXc73QnL7{`o|vzF2XlQe_tUfB zh_vu*CIq$)dX||2|3NPCc=f_U-L)W&APAmT7@SjodIH&d>~Knn zy_F4xxkm$ee8xrWy}x3=u}qSJeHao=Au7U3h`~^x2bXFYj{d+Hfteb3`f!4IZ#9YV z@Ufq1Eqt3kkah#2wa_))0Jh**(`Ij`5aZ!y2*fG5m?o~=6(E8KncsH|yGhPS_QHOn z%hqQi({0cflj$6b)uTi`HYc$frdbgp0j1z z#{1#dC$`*dG_r2MPhQKpnryZlk!(LTY=sSc!ql9s4Fs*w)buOBAOjgCUv7C104K)M zP}2pj$-u1RLavM=`d=3rPywX>wT6oLXAOk{r=W{6vCOn6t+YAnbYPxvv5k=JIpm!(nawk+$>wNXt75rK5AnB0%vR8AZ;KbhDyt?#!8 zS)H{V`JsEWdj$B0dIp7a13_W75wUEV3B2-&l5sT2P$`WJfz({4tn8!$#k^9ULh?Y0DxG3|GUXw^;-CqI2pwYnYpn$OHhRY*y}L-cm@bPT)Ru;E=;=$F7AWV<&y7)+lXV>dntYW30X&3Ya$DQH5gpYumkm9QP%47xSpbB z#J20T2qUUCYlz6I@6g_eG#o7;BM&B3$O4X+%aBG3PmIn8rih9*G{%-(5!SrEK30-6 zUFk^`ga7^%WwZt#!D!U06tjWuXAw4KYeE!l!KgTm?(j~dmgOCte%*;EA#c|kCklI^ zCl}f6aw25HY($bDKe7Xesf#z@ff!^S*+6*{_Anq34+tTMK(9MH=)G(+x=4oQ$Bh6& ztKt9@Od-MzVgD0Ub$@Z&R}>-0I)fXLJgm^15JC?gln|~%3!|vlPK1W9s)MEbKxi12 zJ@Lx9%{c*xvN?LXXPn&e_VyCVI)E8l^GxJ-@-Z1H=0;&;sjyh?Zud3xA3+`^vE~Ly zuYYJ3W?D9)i$MENUTJ>1nByf+dAC8-llz9f)dJbHc4*ZKjl%qABD8QTol3v;Rbj^N z$&h*0o?tg^igD}xl3h}!2pvMUQCLx?&Ag6vNr|2NijCKzc!&qcf~DE8h1(mO(rZ(tg`@o8It&5}OEswTWq|=qbCp3qd{Ac$MooxfX z6H)mv3rk)Uhm1@Xvtp^4QMVWWpC6jXUF9_LC`3eFu>-9lYnL7=SvbS1iGet8D+umwLjjn(|>u*S{X#vu7f64Tx_aepud*QpAjJlmpC+eth0RSGG8`$^Jd6z==c9Aa3X!1UaJMF1k~8z!u51 z!8#-k;_J_Gx`6zkF2#==mgKH|6_qxcPeujK6xei(NgO>SXCoT?#AC0~82Zb;EDjx(pclyN{-v#NHwGVguax3udbE!C@}KkvuFu?p=j+<-@=5Hm*el_h^dU0{RydZ^=9ux%ao^#;_KDZiNzI)m%4<0 zro({}=9s)o8c{; zcLe%|VwvYEVx$!bYYJeArHd6HyLCud)iqkPf%@Y_>0G^{YvK=60F|8QaIGS=m!^1I z=YDCS$cqC#`{^Co9)?Ey+FYgcDid=oKWcQsC9Tn9v7uv|7SWAICTkyyr(CPp+S|#d zgcDtdu;R}bOM4x7$IP2HI^S&%-**YHRJiyuPE7#C2Wk=*ZMq(}i21ZF#6oRiCmzKT zS$Q*0BF)m!7XNMz6D!7|Q*IB(&KK|HcH5zE|;t)O`11iHdz&v_@hc*~ zqtK_80yxZfjimi#bj-GbzM)w}S{sqreplqr2UY5#p){Kf**#7jC%TyR zefa*-CoJ!sh*DyN>w4&^XIBnwdfON!y+%QN;p`Cg3gxOQ@8$7U(`)X*Afof%8jnMX zp=^;w)oF;;yNltc^*kZjJRPr&?UQA!%&EvDF%@UMqT9FWCJ;5WJHI7N zwC{3H+>d*|TxR2R>`N&zEao2!o8~ph^y6b6A3I4DQf4Q6Jdb!QIS3K0~anAms(&qwqNFaK3?hhFB z*IjT5+oQEnbF0EdX3-?I3UhKdw-oE~P!`im%mdPHxCd}63mOOPBLq}By0uLK=Md~t z@l4NlKag-5#2C5DA$gvLxuFv1n!9O9iMvT}6B|vsmj@6B&O4qaI;Y~aOohphq=#qzhaPCn2DQx zfQMKMrwC*o48H-dnUOI|I}}(T8K~N+&+AH=eb@Y{aL{k=d;{vyv~v9A?2%wqQfOu5 z5%N}2#vsvfx#oW@HJn`l(^Vw)@QvYU2R%sqxUNm52y2i!h@N z2j2uo%-b)`n{^JF(9(ejpdPQ1EeAYIhCt>jzd$M$&ehHOhVIb`7T_tj2&w{$Rx{+%V@J|#(MB2w%0&j!~U^J8Dw~|OY zc$uXB7!XL`4lG>;k`bwZQyA#6W^1FvSOyyylf|gH8>(umR)wbQkHyqKsWy%n?zO2N zxT_g?#M+UAn~%pfwWL~>tJ|KY-jCt-^`lg(N}Cu%|7LOf#7NJ?(oEwhPZxU1^u)=^ z!A{KM%pfZ#WYkDzi_e-YmZu#ToIcBlrj}cQ$oq)UH8suzjm+F7w@nLA8c0qX9*TJT zoi6p1Hk+&|^FZK80)hFe`RZ&}b*kgDLeK!h{x+V=v7&{I0qwZ1$k&nyPM!Qx_*0pT zZQGO0G~F}^S(6eyKWH$}x&JiuD7h_$4dZMh-3p$RFGQeulMwj~*( zVwvKq6{|j`N@^BsTxR84k)r_vfG;r+6ot5>cCk8(wSjiFN0&B!%*R5}P-N7oEtWHM zEs%jVXNfFUV2Nl*qzo4=_SJMW+rshD%TWoS-UMz;J+2fUJXIHzdt8CM_V+ zD#v5X16t*!Q!5+_254mj_K`$+3<~6L*xsZR2R7uETLF0{vb51S_nh%r0Ocph1z4(z z5>g2i9YsAA>~{NZw6`=2&Q-#Lwo9);=314pw^iXU#Z?oQ$jfXO4u!0BB~}%+(x%y3 zk0nburMj%8iss@goT_`&5_+d9%=W3u5g}-E45(#B6ap0cmj)gQe{WKf~HG!cuKa+~^2f@!UFo+@xxCLABw8}qosXO%cV@plPlCK~5xxO_01RQ#*iA=AaYihSdX zgjs9QB5Omin+??i*`hcf)nXIXrf`%c=We>_n|SBoIE7 z9Xlu;R$d4h(k)g2l?hT!jhP;0osH>GEv|i9#ux3BaFAJPon;jsUeaQT(kc$Hb)Xb& zMn&0KXoVP-X@BIc8H@3KN4$!2b1 zMz^?iPitf+;w98BUjK@zG^tg8D^D*`RZp8&*I&GLGxRplt=f0kzKqs^4_X7fu0GN? zyld(HhPVC$>n@xvD3+~3s41*$ydhZcev+&~T=;%gx_)RnsEy2y!n7Vq-YyFG!PRnD z$}CaMXh4PeuoirK(P-!S?T~1urxM*z!D>IM;7A+ga0Mm1Dw9T6SaTf~=zpDHcmQ1t zKm(u$Vgdkwe=2&QaM37So_FSGi6$8x;Y!hlRmU5>Hf)_7iZq{Lu;ziMbT9$dCnzeek5@q!BD>#DS7&6AFzu#s=JIu22kX5;IA zaG)NCP}jpsZ+bqJ_qM|Ne!%@SztmazO`o&j@v`awjpmL1Q(SN005zr zEJ}*2hSZ43Vr$(FVmWRbj0L5lCW}|ifGJ3n)haUul_)4LiIcjA-p5n0;xq+-!6P4~ zfOfU01~i~ZW#&U-M(zfHT)Xx8L%UOz7eQh(mKP<+Wt4}dt94+MWNUezl@+;0juZn_ zU;!zWkaOq7C*AEi7CJ&*sgbH#QGQ6DFLE2ubwr&mACvHqu)7-pQZILm8Ym13op8KU=;v zbuwiPpfcWM0Ki_apun{=ucnq)=!(PO;9v^HVz@V03L3YenlySu1+4%nsh(#gAi*0= zMJk?cRtuFaumVu3=Jad}1&$SGt63Qn0&5t&e3liE8q-!wHqzDS^Mo{GeKDxGO~Gxa z`+3(5Jyu(EqYySkKjVltc^Z-l9?RQWh}Au=;-;U*9Fw|9h^j)~&Mg@~r=vG{0W*rB zawpl2q^S9j25sVd!xv9i#uI*@7fgHKwmLcY@#$*ZyH)nJYfs}_*B+0^&f8xqpp7p) zn)9NryP|l~7<`z$ZZ2NO*DCS9Pd*ya-e2urs^UU7l={_GJfRmpX zv0m!g3kP-e1)RL(Cb;%t6RlmH6N}_Bwzae1Pg8i$1PYRChu1fZgXebU3`JoP8*Dr* z7-%H&E5CT~P13!UUUh<7f7q;yVtn^;In=oXv@X~ROkfLqi)~PiXC@TsS~Gz0K4oG} zwGWoDbCr0~G0{39e#)eaDElUj|8W0XRCqpGI#t>Xo<1_tcn#~oeR?dha!3~&8R1d9 zhOL%TgKIlId5j`(mCOt!H>H5Io=R^jT8X3(YTA`hF%|Sy4W%B2oI*1^W4&mLQ=g0x zb1X3}T;zavTXHTiwl75@ez#m8Tp@`2F&{bR3-lm0AF1{D`DGTDiE&KK$9g^%5+RgJ zh)ey)ES4>oQlD5z?RYF^j478fmr}NPYAoUW9m={4-H4(ak#!k75bH>@=(W$R^+0FF zR_dWLb=D{;z(xY^4a@Zgky1thv0TC_(4s2Fn2b?H1VKv!h9b3Y^#15l=MNm3a%(>vKJ(97>!) zk>QP!bx+4}-PZyY@q1!x^}3l_cuTV#owNlXr`h+AhObzi(9?GM9t!Z_=-@yMs)*s9 zs{8#YE71D*7t_jMPYBz!)em8E7+mi`H8y%XF^ z^zjP5?eiGR)RYhh6$-$J7Z;<@ z6Dv;aR}SZ~d#q59m4;1Ef30)byTHw;^^okc@%f?~)T_-Q_SyPQydPQju`Uhff30)$ zFhPKdRYn=j9I}xjP8mKM-0b4()5^bYK6C1(s#u1vH9_F2_^aE64h6Wd1j6Cpzabhu>XxT+bvC=fD z^Ba;mr%^$-a`y@}l?-t{YcC2@n6SmxQO#!i;DKF&m-h-H8`d_E{%(GC+jL2!87(|v6#rP70l}c@uw;dMB31qn>9C;htuc3w*)tcHrV|4 zi+?O$vuFQ2*XpC;*7sgZbNd;T90)^Pwt5$jV!gKdQn`3muTrtk`P7`S=qPd@uHrE) z`=zboHmUjF*Lz)y1ga=1L%U;ssXgqiZu^uP z@cXEkWA13AhA98T00RK%BtZ-S7$72me;8nZ|C<38tRSnUko>H==z5N*4ooD{0{>xZnSUuivcz;yR^EtUcIuh{c9(Ft4o22DTB!Y zUs-MUYJ0y!hF(?UIFZVpfq_~1`eXIhp2=ZBP@hjiVP9pwwRWdBY#MYTG9V;h&?_3! zuKqL_W+-lAci4isut+jVPOD63T!}mkN7-m38U&#a4hIuwz@aSMNYIC%&K+FR8OTY2 z;Y6c5k-UdViG9bW+5s^mi35kDX35l4wyL{dvYRF4YK6h@#e7_*43b`Gkov5gDuW>k z*zR<{BM~rIRo3hD`uv4Jfz7zngFLKB4BhH5gQ&{R%zL_6-;*5<`-xD6<3QS<#p4xQ z%;l`-c3U+RKmG~<%Rq13Y9DxXxG^dOrKz~aNhKy+aFc=fqXYSLaOkdwCZ_9_BO;&m z*n5Kwg=>Y`xX!Do)$17ysw?UnKOwKj&GBrd_)UfP%kS-GK`fVt6U~nNL?n4TjYr=P zn0Ix2vMCU3BK?Ia%18_y6=E#kKPZ%W!P5lh26|{#BN71s#MbD*LPkYh+E>-J{0QuV zG5yefQ#I_U%yLcx2BKg0qKI7H^%EXv{i6q7=f6)$J%kXce}M#IYZiURNk^*3Q1>DS z2_v|U3kG3jCA;U_Ge5N!r|A!j6$cp6n8PQJ-I5GLIv$ckfodofm;!YPU&w*lsg@@Z zCM3^N0!l@Oq1cwm#GARm%kiK?zfmu*)e99@NusP@ys(dO1p_jBP)zxR8f zj^$_us7(nK8)%*wjzzE(WJiSFnFMC!elq~O3eO}`HZ~;sg&I$1( zJI}MQzBX=4i?N^0y#eU=t8i@SFvC&(N8ADiy(_i97y%w$HnpUFvulzQL1pvIWiEZw z3|T>}@AjZ5{DknJxeA-GphB>;ZKOGY8Bl~pUO9knRxjViv#!D?Pf}A(gmVIUcaRYT ztjW9J3eBI}Zx6G7d&1G3*HEkot)UYk2wFA~{yyJJ){cE9doJBQj*RN-6n7>U1wi5& z(ZPSbOBG&?fxwY5juJ3X@7=>WCY{-zAz(6?S-+huHaCF4^mXfdWR?gwyXpo0LaKHuyA46!Qz8VWN;BDuuIr& z7FFonchLaA{C-7|zhHc&?Wqwi4x4-E0DV2OFq!b)z%4Utj-sH8_RdHKCXKQOGRn2| zK786~+f;;MwBF;vw|r=S71L_>V2w=@an%YYchpY=uiyhrbD*eX1~bas`kt;`pO6r; zTtc1q`p5)hC|~M*ausJfKba6;n=W!3WpFX-w7rBlcMLq^5c%jw0W>0r9R>)Gb}}qe zaVPXGvNkV}F#vm9v@9R^fS2@3szh3nbI}nlnGpM)a!ewb7{LfR2@L zjK}nV#$0uOt);@$PL^J!lR{-^Yi7!ExZETcyw4ykKdwN&Q&D$;NQ{xjwf|AV?T0O)~!Wk$YI_Wu8O%D#5( zOF6mn?9~QhLo#H}o;G<-+jZ{Tw&8OV&fdIq`OeV(;`U`1tbb+Ufjm?dDMq67cfpD`gM)`}!W#7frA(uRjPHA+T@AUpPDr znN%uzETF%agkHyDYplL-6ccM52LI1g;dm0hpx$dd3i$-)Hz7pJv-{#HM=~dd%5mjF zsZVJ-FW+d7q>{O0j`1@c*s74lQiVdPkXKN&5G>_JrQ&$>%9UFB3W}?dCc4@}20t~} z$Q4S(bg|6}0_o|hZ6;g!nd~98>&vB1T)NlzTo!vwGpX&0 zXah=r%gTGm?hRUF=yShXTS{eBnnThnH$BzO!^zKjc#}GdbOo{ao$%Vem>i6!xjB3& zc;+xBwY2`paePmBm>=En6+R2o_W4_Wzaz0D>#|L@ZuzXKj$oFW#{>t9P81lj_58Wz zd5sKQ3mG=So(lydN!3?`O4P`VxXS}(MH01;?prbJgBtTxjFUx=-o+#bfE*lIKH;@O_BrC@y*X)Ean?C`|GgF*!Y6_JUo{l+ zpG=qC*Io?OKPOh;|6?yEB;cPDt0lc#=PrZk2O7+pPh`Nx3BX1tdfWC@9FtC}>rssI_(ez#s^a za3zGusMy@3!hE7{<>jTrRaK}BR^Tlet+R8I9WmX#B*bg$`0$jJgTwWHLBd(t-90Nr z@a5aTF0XF}hepv5km|-K!9nMR+TD*U{=#C`8gO`2MDn#gTWJRPBBRI}lQF7Gb^%zXL z)A|1WxGGRnMDK68MFbV2ax>(VQ9)BrKeZexi&|%O-91&bAAk;f}ZHl_w%N(^hW`biECa>-A(KU#TL}W7y_zEMLs# z<-0{wK!y?hrY1&kgyAR6_IpVPOv?0Eu(*aOy04DQZ&i(N(WakB!9NIAfv$y&P(Y{q zF^W~4MGSQUk+u93y@>7pO1~dZu?FtU2K)oT;(q>?P2@B8wHn zaHBiwC=R0gmcmJ#3+GD?Z23Yh`{Ja2jQldW?lw>N_9sf;GN$mQz>Qs4cFAy=9csgg z$hUu9aSJ|@Ah0vKmpC@=0ZB81xfOL=3!q#N)Awa$YRmOQN)k<@u8=;pjPHI z#Wl?5)xx$^!?8QU2qQ>*&G#ohpC1o5O&vQ-wVxg-$aYhNnM-Lm`*RW_4|`6MUzXuk z<{5!Vy_F%>VrlK_4ccgJF+1-emCPWQuAZ`n`L>bzE3);|xN+usS=(fxLX*sFxc!e^ zmaHS5MRo}U&vl&_CT(EuFXGPHw4jE~nv@t~*zXyOO42vp@wmmPUl_3p^b&v-tbjs= zScXBFq$OQa5J7H5ogH!9mQp2t*F=7p>3Wo@`Se+(lk-js=dY;G>yWMKB8zgT4*kT| z2BTRmbowC_9<7_^A=oap8GDa6>t6p|kEk5s9q;S;@tgt%pft-B!#olcW$lu<5TW?H7iFyDkzByd7^JGmTCoZ zdR*Z@BqiKM==HtPrlwp5vk3TGs1PQ2yrm zyMN)P*TRO=w}jj9?;EEUaon*)hxv(HheiWTlL)8A5H#6RI-$uR%)3r$VE(6<&+rgN zWvb<9QB_uIKXmA4?R1pBJ-)l}_$$i0jh?F-p0@JA{sQ&0)lPniehypFMDTIrSz9SO zWE1h94q}zce@On_h}m(A{A*JG?uCQxFl7?gWl#(Hu4T~v_boENcM{dUIQwP&ZTOr& z@ylatTSUx|U*Z4|0Hx6fAYgRLF*5L#I&N1g{-mQZsLCtV2y@M8XI!2sv6_uDpHu#He zgj_GwJ0lAZ-GNh-fP^`j{4x{st%+w2+D@}v|*^;2JRpQp|}-{|npA+uRnrMF$ICJfx|Zx8GL#5|IWUbqEip{Uz13#5 z0L0OB@S0cqNEnHT`k%-`$o~^QhW*c=iuz|zL44+s{J+A-v`_^%-Z$S`^Xw(0vA-5O zQOmUnBj!{VC1qvO=Bw1A$~+P9nURGT+TuYr=+TBy_Zrhq({gt{A zICY-VnA32Vms42V_)rY;{tgKV0ENY;s>>A&szyY`Kke)93!=RpUWBri=?y_5WSs)m z6ZP`%$7WfJ%i;1k2f%(JYe*6S{6QEErGS!71K=5`cKb`z#Ku!Va9C&znmmhnJWvhP z7cM|)FeHN2{UKKpb#IDT)Bsnx3H&5bDPIg5&d>%tlQ*D-j)+KVjO*uKDmu#otU4); zW{1f+E2U}$m!;YJX6tw5V%k3l(^e|f`dJFaE6KbMW+Q!^>7WD?YGEq^dg_7Iyn=M= zizpKrx<0N0rZ7xc-+WM7?P{!!ptC14Uyf#b*xk5~xHQ`C2FCOUT{qA@ZqpMaL-dVFs{ss)n5PvuV;sK3I< z;`oB124LVn{0%w;hT&*R(0~A>KgI&+ zSP;$zVYnmid(l_}5``g*6sL*+w}eN`?-c2aq(ABZuy4o(6R2-F%UHE;yKLl}E?Sui z8XSe1X|8Ko`^vg%Cd$ne;X6*N7OP2vh%o7;v!|DCWS-ZX<#;)Z5v_QcO@yi1^1TA# zb$vEc5|zCEvf~5SHN&V}4PhaRos^W)H&WnDG*^=Q`LNc~MnkyF3{3}IkHrha7-K{k+FDZVo0 zUapU_dM^JoOpnPhD=+xFVODt(lKHE`X~3$c%Cdhb5%99T%7FB_9kk;I1c#mzK*ZHB z2!yjz*$)SxQSAqDOtcY0Ai7JC27^J--S#5qj44KfybqG_gLVV!`XLrAjQhXI<16sD z#u{7K_``CQ>Ggz4>DCYN)lnJ5=tYrfB{E-J7HLae9Jwr0JX58wS7ffWhyb+++w4O- zE3$W2zVM8!^AqMefiumbx}CpYvFl`EX`Sj=y|`KJ6G|$aif5m^lS)!S2&$URjkTLw ztXwC}EM+QRS(mE1x!Ee*BP%FDnnO+Sq_mt+ELQ3KWO1fqXKd)?*rh`nHJJ*3O{-HVaCuy`S&dj>pPzJH|kamQ&Ckh-r977rp(@YOmcL1CIXB z^}qJIt(zh@myMlF?+pCb;PHas!L@q?WA3Ysgng;%dB>LFAkfNd(PgEhVTMSeo7#%c zjK>9_A@X8}K@+2P=*(B;ao@-{!B+f}8+0A&BnVB2(P+h6P!buXD6;lHRj}znKB5)> zT)?5HIkj%gE4Z-h;36s(4)&$Yhc!3tjx&TqAY6k+5WBVBJtN{>`}MDD3UDjnYDy5Iq_#Ap>QtwYN;HtSzQvIbLna_^@PX9 zY(!d+8O`sA0h*S?5QiXBrHhfQfP(Tocz9#P(KG2#(8fTob81db%OTf^y=;2nOcsph zmGV@^+$1X+K<@dhH7IMQNaYcHCr36Q)>o(aiCf2hhAp)R58KOIr5Is84R zMoNAwSQD&JIsNahE@bN>C=pP=PW7AC2Tx(mYl|uX$Uk<}t2HND;+TsvV`KwyEXO zmFo78Xp0os8qckkZuH3st)J4V6H8d@i9;IR-IRqyZ9G5{{hrfXQcB7+$l4=1SEISw zzQzUCMtkEfGH#P(E{^*PJe?mQWQkvBNgnDjt_umq8zoaQp zlo7J;lJLwiGsHhTOvG|UD_kds5o*^cIqAd_lMd`NE-%&Qj~ng(I&I$k^3Xxtet7ef z^|c8ych+<#Gz}1s5{Zb9z@}nONn%Y+XUSkr&dSWi%E?E=&`l}KFUR@X?x|JQ4~Y&7 zXtrvMZD|kI?eJ@E{ZTjM!9NfjZ9g&EHQ6+yf!d=sKf5YFyz#wZ>dU&B8r$pps>50H z?mynVqD?EJuD&Y* z8P1jgf0+^BC0=(@VF-*z*$o6-ZNOU`cCIDfOl=wnKme!oI_>}fVIih&lUoJ*2cFph*?%l?Ce;TPqlqh6`> zuKJ2mf-68DJQ`JEqk-S0IR2<48oU0GPaK7M@Mfy>rpTZ~#O5rnuKA%dc56nW_Giv| z5hNAjBoX3y&}yo*_p16q&*M}pL2R&8n6cuh*#2=gPZHSieq8RmzT)@5!$es#2^A%o zh(uhz%0CY$lmR)6W#l2Qj(yaq!6!O)@j|PacB6mUb!;aH(y|;UVZ?Px^E@f>Twpa; z%Zi6B(=1E-ear#&dwkDj&b|gXM+~a_=c!)${iDT6dMy?z1}~aN>%Qzc>w>Y|HD?r` zajuZ$`tgn%oT&+BVwEPE9_Q^8xeAHuIn{*hMHlUJ77I?jP>(H(cqw3oxTux51WkGL;84O|PS zRXtE2oYlPyQdD)L;M!xht+_s#b%4_JR4Pi2NvE=zvh^-1fzk=BO{Zin%1!5HMcB$x zAZDp0R94`trCWXrx0OrN#7KFCT%(qyYwbwnb`=_@P56*Mkd}PtNa6j$8@m@zvkPxF z^KulQk=C&X`WntW=yS%yHk&>wgmNmD?<~0uzK}lQK&6KM4})E$1yk-dD(&%f{pv5< zn?{}|-Fx7BPcU-=i1An0gaI#$a)5w2=OXzqma4uQO?F;+*xs11_4pojt=^Bw!c{Xm z6D9>)ZECPRMScQKp7SEKH`+ZK*Wswta?@$Y$C@WI-f23JS?jS(Qrzl%9z1(?bIu5l z{ugbF?u739G(yZKqS?vP%Oy(Tq+Hu=6d;reo^Cz+Oh766gD=`OzA+o%(HsIgG8`zz z84l@1#MA6U6{yyceX!;z(2{N&L`9}Q%Lssxjjr`+7tXgpZx+JYV1)YL4l>w34zf7_ z4n+K42N~@Dc94^!mA*V=b;T?qRR#uadkuOjnUw!&kxEgL&CF`f(#g&1U?8xU`>s&; zpBAa`w5FD+oX*)Udxz4o<(1X7m4?ZwskXUYRr;m%qvMm)h|TTG?DpL`Rs6%V-%rm2 z3R_pdZf@t8OTY)kza(<8h8Jq*C!`!sPbgBUE-29`p^zQik3m%9g3-96M_rv$c~VwR zDuD_eE}26KUmV#1;qiu&X($x)xZSl|PAC;(K)w*69JJ(w@>S@tfZeVF> zm8(=cJzZMx=uHX^RL873ynsj!Q;&pp?dN}yr%h&t)w`#3b-&g%A3B&8UxfBBG z-clvYZv_F`(Dg5j#$NEVZa6XN7v?)Z6zYy0H4vs{G#D*hm1&T8qj_x~1geedAe!at zxe~?xtBf?3>H72lB=F_MC{76R%$P5YKx6+iQF+W?0}jl9DnCqF>2%LcRjNGSSNmg? zJZ%djDMzn|UUT1`Z@%;>WzAGGI}>@}j66l()%_@W=#ULF!mQ;P6w}9{i6ZmcjHE?= zSdXT8sLjN>S(*oK8CtFn`>K$$8H5TMD3=|LaVbaOd3i#iFRn>K0c`qtoCwJ-u!vN5 zPpcSa1`wu0h2Tk*6}ypcQM*eYSzH$#x{g|fr6@&fFFR3OP!oJFMF)(4lr}G1)UvE% zEblJ9Y^dWqxrCoHvOcX{zjwRJKBIXhVA{Thsqxy+DBdrr+~1^$`E#yaQ4aRPW7V7A z;u#GJM^<$^0MF9Z-*q*-D}sTpZgclJj5#QH9o&a`aZz*SiElnepxsGNN34wn7@A(5 zTyDfKjXoWnpWuBMJ8pQpW@JCHhnVbZr^A@yEc9X+Th73Dnz=9%j|O$b_Y49N_*|*# zW`X9lB6+{=v}9xRoUVw167;b^;8sD<#z&YDS$4ojykJ#US+YOHr>*oc(g*3k;vj~i zi$7hm(znILME+}=g~dh2EsmvX?O>^&Mg7q_4MW!lImgn3FpH_9QS$<+i1Yz5gjhu+ zq+cB2tFya2b2l_QiEA$m>J?3trd~5R?Y`iItVfL(sJs_Ue>TY*s!f|jtH>e?|tM|%GR<@WXx#d3{u8#rga`{;I1BD&>$Tpo822H@58k3h=P zG5Yy>?FLB-Jpbt3Bl3CdoWcJH%ReAT!oQ}?vuh9lgQTzz2;2B|B19?J*3S^I(!ZNe z)YntvTog1KkqePeIUo#i8$ctJzhh(1gG`YZ66To4a7rrNOAQEk9NgPxU27#AkPRV1 zH!2|64a2~h|EcFb0DHqODl`oOu<|I_=}{2IRJ)UEcnHMA7#$|^ofqapKENJd6CKTV z5aM_lz!g;=S?gxvcak{3*ERoIS5)Mc8k)nm$Q$8{oPlHQBgFn3BjBk|R303UWFBGb7fS!EZq~|LvDx~J*0?&Y^{)t=Gh_P%^0e))qXvtGGD%R^utwdz zB~fIwP+6}%tDc?nnl5d;h)bC^FSyT@^h>2hV{s&oK%J|Tqq$5BeXp*@`%|F|%F&tb zb=!U}597}mYWL@dik{Xd${*YwL8k|;897bd#}{kOUOo)He&nwrcdN1%F?(3qzPd1{ zJ72)N=U6+hW~gm1Dkh73GQ-D=7qkVk>0Z23c{3?RGe2-_|Elt zxZBZOR1xSJ^jEhXS6tL!&7EJ=_CeBK)(v4=U)GONR9-erv0q#^&WX`pH7%)GUp23p zRbI7hxn5kg?uOD{w;iThU$>u@RDND}T((?XcixWBez#>@vcBnl+2Fb9!92dW>3zPZ zWpo8#+1&O)FIwLYKu})Z8o`Tu-wvX;lHUy@s8`*|W4poMju5pT-i>{P+Axcdl}6u; z(}5M;PrkO$v2Zc2!VgYyY8y9Bb3MaP&G5X@vC;~F@jcELpzE?O$jBEo&C80*jx9<{ z+cx8>(E9vdF*DI+TQ%_8YhE!->mFXy&#ZPuwMgoI+7xZdX5X@(L}1%?+4SM~Wp&Zb zv19s1z`AP&_k(lK5YLx$Uz>J@M!SajSlzmZ>qN;bhxs%=>Id_g zVx}+mxlH9X(?xY>EYqdLh zSKZx7uF94kX<0Vu1lQq8d)4#vR#$UY>V zZ)dyg%;tR1?4$v$Km#ip3&FE=du%HBU|7Y$0Zf{kkb&@g9W#jx+zkbH5|vQM?*4v6 zyF*@LR-^r_ut9@UMoiwVXe4R#E(*|t2)@8&0GK(iqQ_kn0nsuKwtu)7FaZVMz#kK? zpnQ!B(H_y=AOA#RlzZcC9>m&<^yU7`d)c41^9A2p108QGML~6n;vplEERonns7;9f zYWSm+9GB9%PXtaCQE-fp%X!`><-r$IiAzo>W!xv1$rRJ5k571b0KjXb$?43gC$$!s zA@A%3;BCgGwFa0W-7gU#cm<}6a36qU?*>%WFK~1f5dqrxh3>EDo zS;@@dxDnzgOzuP1WC>ZtOUHC=>jXBAO4-#0lx|z?lI{&cIR_CXRQ))p4mn9AN73K- z!+_JA0i$dyRq%Xlmb0q6i+ry-C4#F-als5``RPr8VjP|!?gIU>MBT=i!G8k&WrWF= zGYjJUkErl3GxGbFH~#ND_&=GEU-KaB|1l33ADn3trjHuAr`i36ZeQy5nAUGzcz-P- zzd$$!1O{mnlDmdcL_~$hQ2f;QB>SiEHH8oaJQExuI9I~iEscycsiZiG5FV@=%+EKk zq0!KVqNoiB$ka_tCHVzvS8%=Vri4HRF{G>@D2Ke0hz?Al6!FJ&oq=eS&7w( zOpXkR?r0v2Dii2L`NABTKP;fk>VDC|WE?Ulaup`tHB}pqR5GSnWE6Y`%38|Vyu|UE zXJ}<}HZ|`q+LP+z3rat;L$PImU_tFP`huB{l^~M2^A!mws0TH$i zVxhRJ4QEqAJXmZSt-k|$3q?f;w({A-kjUo@TrE@U{iCoB442&F0`3mRl4$kU`UI0h zkw7@{pu4sOgQ2LXc6_@)fJaGnq!v`TQ4tVVvw{M;#qA)f0?zFlWC*&cw7^^6RT zS^&lrRz=Oy6qcsUe*gkgqi!Hg*Rx#+Xq0GK0)eHI*Pmf+v|ExnqodHDq`2ZyoT8;R zM-DWK)jUquEg3(~#5TD#R&rvuJW0`H-zYO6aiWi#r#SQixE9uT2z!_NmF6f@~0#}Y~it@AX90*yigN1e*nZu zF47v<`5Gx0p_>hzY6;wK8MnT>4d}3ihauQ+#SrBGGZHE#vC& zg4=p-Z4B0~mV~q3v~NKcx;)t5VZB}&eh})ZHzjBd{A?;AVUcnyOD?c^nC@<{`Az4z zknVh~!R>Q#?+=OVJyYek_^^@-if}eJlQVvQHR?3MS|(rm{#LFGrSDRyZjR6}rn~sc zcB2@l+jGk;{?(4jRQvZ7WRhLFX^E~dbN8h{`O5+CUwG-$8J_uSViGC<;{d z0>?GAtebYA{?Z5*Ma6ivsk)cuLfI(WQ9z8xG;L<)Ime)&Oq`DZoK{goX=R}gyxZkl z4w{#G(_O-}w>W+dT^%GXMN6%kA8h8+u{s-#+9wn8OH#`3my?!H!3&jz*F5zSk^!f7 zMg!1=5m1DszP^AtGUTN!aKo}_o&`pUeMkHe*0MpeyLgclr(FPyrhsrFV?d+`F|mpU z2TC1dV3}8btBHjq*Ak;JXEgpOw*>k(W)%5OxZIl|NM6-q6rUkw5^N`^o+G|`4-3zn zbt~#{T1T3wRpWqfv`542AGaEf3rUQV5;FI96Tm9R6pq0)R!(jf-{WMX{4(lr-u-gV zw~Ji%R?8WI(1nD!V3T7R9BGlBv7hkZ_yeHaoS|H0c}SV$d|-;gWnT=VM4l~5UE(q1 z)CK!eRzsT4P)S^-rIg?#G7(b7er&m=)K6eE0s~9vysq7ntNtb7eTG_tc|>iNsE9Fl zl10}y+{p+bG9%Wr=LJp;WjC*_{i3GT>4FZ$=>5ktMrl#;Jz}Sr^Tjj1%|AV*f3Kru zY%8!Y$Jx9oqT)Ud3f@6qWf5KfJ4`ZXydX5nDt_u0E}Wq-koB#F)itZ7sWVURN#XH$_jGpM6M;`B&fal@#beEQhQ{mt`1CE`d$=!!=E1 zltmv*Og3m(Vq(q-84vs-Glv>w#mKJf&`) zAczdEd@~RSiY6^g5e^-PXc*o+kPH~-D@Xx2jTb?Ra0-8Z)aeLt@!7d}Owb~{q!W2) ze>@O?KMBXQrHjh{@65<+$IQ2_Q@PfT)#+>3&bRH`*w(HS*&ENzw_h(atv%1vH@7%J}9}{K)m-|X!*7QteM+Tp7%ZExVHZkzP|5c&$f+lAUa6Qrq6)cOVD2O2ILu# zf_^gnVfz)W8Hvq3O;YQWs~YP#T?9E>{~^pF!0^Bir**W}HbGILob3T`PP!$@?Fd98 zJoOI7Tq5q^tpN+Xq{`;m^@tTG6#67oQSRd-d;!>(>BT*Doq*k)jR|u{wvV4TH_$=p znyurlhh|*oqnlCBDF4NJ)36!*(_C)MX03FOnB3sedMS9sP$;&Mq7%qx60+KC^#};KDGBNB66wx? zp@wFL?(Xhp2M>*>;1g@_51_pI?uW9-(9ct$}UuMBjuCdRLIQ* zJn4yvM)SU>X#b{arZn0gS9fici1QV`_qcf2t z{=zF?wFvWt07U|3JV>7=plAoszcFDATETv4{w|%4GYrq7fz)sO#r>+ZZ3RgE1sEMY z0meU^ezWl*TjbYdqA{jG`{B*^RB=1PE_#b8gaU`C%6NMQ6q~KPR1#Ixgj~?4ULYza3 z;~DMc{Os%-2eFbX;|1*P!w%`Ks{KKS)L!IX{vZ;TBjPP(BXR@hg)W>)kHoE%M8Ym+ zMJoq-B#xk{_rpB#$vt1OeLRklLo)*jIywU^+AF|D=2SVLS;Srt;)Op9pq~eZuZr43 zQtT=Gr~{q}Y=?0=1aZekXo})ugr!)q`MonF5=ai`oDARl5k796N<$UI6_6sj7Otl( z6L66913NjH9aw)OgP$DG5Of;&8)5mH?S9e-xTH$S@$1CmniL?Q}>XneP*J$`(k?PD`!rOZSlS^;s&9 zSonaU51TTf!9*7ClrNu{-`ri$0yaHEjh{FrO!8P;GDj2@o2icjbjAr#cE~it$+CEm z$fC&fozj+A3rRQ542;SN5w^%*&wQ{?ugZ+>jx?zdkE%L}Do4xMqI3|gQif*dST^TW zQUrL(lGlXAU}(jf!_ylK!;sc|k7PaxVm=V6h{*1&FGIA?8B+603@m)+d^!R_VNz(s1#>M@|$Jak>lI4Hrz3S+&nLhcbkGTEACcmQw|K+{RSQ7t1 z{y7SyMbcE>8w$)NYxSA$E1)EE4gUF0xk;b7jPs-oYtFiUFZpsU{`|0Z>8g4AI>awL z6c(-J8{`1TiAf9-NlAUlpO%uum64g6nNean0qawpu@4VefJ{wKeO&rvwQ9NHRbE)Io3>wNbF_PU z06MqccCNe-{9gFOdDHz-dBA4*sc>i*IsSzL<^*!vLB?9k^P%tv7Dcs6DB*fUOf6lu zVG~IV@(9ctnW4!v_E)G(H5EJC=`>!;(s-)WJ@Pa`>)l!7MON}cK1cND9PL%vrQ#{* z`ll~v=1Mq=2wcHqCF5`+zITzGWM{;+v0dh_n3=K5joA2#=fb~sR$CS3O#*=skX`}g zN2v=(MwQM$@T-;Q#FQTYreh*#2_7h%Z2ALeX5QygD({qMSsMKHT(L&Fo2*Y4%)Iwz zXemp|zFg=Q&Y!Q_2;uF^RdPZioW2{W3I3d)uwg!XrDhBv_{I@cBvcr2r*x2Q3{=uc zf1DkjP2!8@6%52({r!8$fVsEyjtptvB;g)eat@2?G~+2s=tJDtTvU0`WrU*o7T?%? z^-v5;8wvxlW*4Juzu+4chcMqym}}+8`mILdvKys_)6%WQhf_wfm&VetHy8Vjii1)q zpR;x`h9|<5QcU*9%9PPC_myP;0KDzJB$>a}Hi=jutMZf>5dZsRVxjeY3_^g444NJ} zjLyl&wZta*HTry6Iwq}nOuUgDi=`82r}{;bn#2ifmVC5hIUf3C$$q9x4+mPhJGwDT z#y{68kJ6uD`w%logI_jHP6N(NMO^D&;G?9{1Bj+}<7Wa=|HqN8$PKX_?akc9QB7e6 zaH@V$=x`DWY4Iyi@v8jM#BSF43r>YP^H{F;PbV^m8ue99G1T3HtVm zZ}7YH18j`tCFenr!6}Rn?NMZkZv-8~V)osE8Q+mR;`*4eyvQLlWTQr(MDWIpVfl8PLrRf z7|U~LhBY1XWX^sYPIYg^T1)%0hm>vY13PzO0`bLn-_e84J4_jlyY~GcTMa-5how;kysIV7#&c$Io~`K4)2JnzHLLc&qHpefmZYDePj z9Fx#Tg)5h`{}jb^IfbA zPih+8`mGl%K)#Z{4u_5uHmr=-#m7}ZideXnO5(A~h)ez6=Qtq=kkC86O}uYEpelL|Zdo^TOZbWth_K)nKkF-W&=h?M&< zIbETuBNiCkE7kk~K+%`PHuwGZ*e?@jgKqa;(0}Pt6zgg7zsTFeJdG8S++Bb@E3@g< z7;oF{ewG;SVoXhAq@02UNRj_iQA%$7N=dpE_STeTjP8&hINX$FooW@qjs*hC9>nm_ zG98Omf5azmaW<%z`6>}LYGXc#$-HmZ>Z3ox-Ryz*QeR5uy3E5fK5t+hSSqBe@KMi< z*MpMQjA(iNqicw_e)n64BTfd59$SGDI%bEBOv+&6JdZyyjI(7@frEO_4_2hT=0je_ATS6l|aT9@HE-3A5)( z>_(E;l@+O9q9>bG>gCn+Z3?*KS!2>GWp1koWuCw1eSUDh1(MXAy9pA`$bRlQ4KUJ) zuZk;W`7vx_@TK5ZJ-GEWG2|gqfgr6WzP(VOtX9odCnFWwK@lkvrtY7sRl4z;K}-@7 z79G-mg?y)g`YZcaiQ4&?@O0M;GZ%JhQ!Rv_ZYv1C=7>^4BT(_CBd_4y z0e>zUL#jbLIm3m#dfX>33%!owgGPb?dv#f&p^2cb9vj`=^g{g z5*4r(Em9-~1~%GjG;LXRUx+WGOSGjT-dr-Kf9chd%Sy0ubu_^wkXJ%_egs0^?7}MF)RLfg#uB8}!BqqqpeE}psLIO`zIn7OVl zVc=@QQQhaq)|>vhC`UDY)(P=e!9lkSTeD!OmCS~Ib;N9)_JaxL0PpPwjpygk3MZpG z1G3y71=qC4lCPhJ2?joWbq{x~n&#`(8C)a#>F2Dn%5!)3&dCBrk_X?mdshk>_U5gJ zenS{Zoly~q8|>O)L$w-t(hIW=s1f=FU*XRMt0nyv0BIyXKzbb)z?$U!6S*+)d$ernb^Fp9hda zt2aBT+mmtae79N^TiSQotBXQ<7n_dH4|2Z2wF@T$_A655PxjegcVo~dd!=JA9Pqsk zn5A#NP7QS2pN>-i;8_6<);}D`#C)EP;PuYRGTWD$7@b6^hj@kja6ZsTf-aR@`s94u z{K{aTIABjcORthuoh&W(gShd<^Ff!2v&?&cZM5&$bxB*)NDquxg|{>#^?)u6+qJQN z2z$yarMp_g{A&U?i?=2;uJr34ME_Xet^WD3eyGzm!MoM!Jfv(_pX1+>NH|M)g+{w7 z`juW2O17Q9FNJ(q^azPv+><)7ni5a)C%7r|?!W2M9LgtZ_6Mqq`4d z__r7)7EZesh%Up%>aXmX9B!%<4sY>Y7xAwzb1l}3*fR)#A<=J%BcgVN>PH~wopuF~ zaF7;DQI=&XME@K+LTW7{pbCOcp%iuJF{$h3DyVzHs!L?=5iRN&;K8hD^wOeJ=3L+Y zK?L&lKDfyqP`m1dpX}ux>*cBJ&8_6)r|cuIuhJb6$m452o*+nNH-M=B>R!DQBO2IJlsM2HGa(@SFpkbVd;U{4S-L8NfcFYWN z>nz2%90Ti;nCQ%Q0jqNA=w$L^Iq8{Y_{oUtBUXH!nE6em!KedbJQiUK)Pu^QA55}4 zK>}s3;#=30KWBR``A2`+_q@>pqd>`NGQeL}I9N5_FS4Mov17bJP>*V_XysU9l^8#E zINl&%$T2_zlHzXU8zq(^BN{6k6N{*pYBt1Vn()#33<#Y}q2++5Rj2Sf!nNK7qVe6HHfN){|q)8ckSNCbtm+y$(=xstlK{#ju7(eH8Z?AHTk_lQ>z}Su$&ehPK0N+NXBrlkE?+J_4 zYTid9^p0+Vi2y>={2PL7mg=5CrwyQfX~ZkEwtxPlR%KzPC~6@6a1 zex-cjbw@#2fXyODF4|Jssxhm3Nx^jv%?3_hdUF1aRx)Ny(cyMc-W34~3M@vg=jyH$ ze)iD0no_~bC8DY9fCQH!;)Ld z*;C35rubXQI!(eBIQ7bT+g-5sh2CzS)lNa(RB7T`v`;m(N)$NQohImrg zhi9?_id0xqA=YU}kW6vRHM_`2wj`NJt1-AgE_Pg?PQst;Xp-y-G79GJ5$R@Y5N)pt zaabi5%hcxJPB#(LbL~+zsLjG`#_X+PHW;_`rH*W* z1iP|Tx?)0}{Nu{lV6TuqGT>BF@l^gJFCva|u^7OcTxc=E3-xK;iy2-Xe-A1|#Oku8 zr7EYvkIR4Fm3k60^Bs9?y@}FwsEU28JQzo8E+-Cc2s&1|%VlFbI!RO5yD!GN=Q>sH z+n4n{CKQP!qTl(}?6c;JOF?oFS~wONcdv!tvitx7epag89#&1I*FX$dcG;?m+T*s3 zBo$8J-%P3^?F9CyrOxlu?V{cq#0q$Jhbs|4W3GXmgNW^VFnj)Q!vt1oSIA<2LgZ>Uq`+pWP5?uJkSHcJs6Pc?D*6 z%1jmhms}O?ykqSFHHNbhIlcTHS*$H2iy0tOej#wEX`pmEJhJCTFtioJ&e_)COFD+V zf9T9J4;rJXNF&X~PVJ|V?x_~8npQ7RO1D#*)Dvc#L|bNWYKF6qsA#_BDaou#op4?o zKg74CCp6&9-tC0tCH2d!svMOk$@v3hP&c*5Qx(Ni*WGH5(|G4iY6q8cFNTuecFuG7 zd%-M|Cd#MK9`@HQXT9EozSny~T5Zj>QSIhB(sXwpd<6&|jmd$=c|0b#BoamJ%f)1m zfo4MmT}NcB|K8)uyytl(z7)8!Hm-MH1}?Y?)|QNmm>(VAla>%7idx`<4~{s0sFIP! z{i*!L%@2cF3?)C$N`#3ADQo*rI0pSZicfln7!e?fjRB}gDZR@ODRk(&W5M6N;ujx` z*w3Y~;|EE(`|;HB2!cw{;RK@NBO)FnZR;hBzlRfz3v1ejG3rfT1_2pnhqdwxl+6ZN zrEJw*fSUEAbhCI#Cx!ZIMNH@5JI4_jm*gadQCz8^a>MaP6~dnAVgVNr1Lt@*WDNB* z{)Tpk@uD7Q4(3i@1*)8B zxfaV=FIpo+?5l(ySf7FF)kaXK$dO8+o3H`mX{EfV&B3NNsxGm~+&x5c`Xg644}QAH z55%A8WPO{uH{I`2HZxjzG2^%_51jM_2qp(VWCl}jwCXITz0};2f@a%OLdB^?giJlY z<%apr^5jc4T=X_slZ3CQ`Cp9mSt8Wv-idxZ2=&E=pylq`jN@oaT4bBwI<91Csw;K; zX*s^K>Ja5F*y>IHIV5IE_h}fZcIe}|8YO+N{kEIIS?ux1w>1wj;1)Ua^TwIMO;6c}xJuki}w zM)%+V%hlJH$(O_Al5su-$`rj-U?Q%W z^HHl$ssbY#GG{;6Rq*=w2bGZOT#G}3uR2?=yiB>4zedw^eD-p_XvQ(u{eAnvn#RL`A z0U(#W;#|e&a`NapKWr@`ZjTHicRNf0A2bYGmN>XV{o1}X&K?f@MQe6| zykPhRp6tU=Jonm(+d+SR-)Fbr@valR`ncx9h3k~0`d#q(9C%fBDU3go(MK02OF3*U zA>OYeu5{X`N;1#-6e4+MPX13E;X=8!1W4@}Puh*cBpT`weY%C@BWoWHWeM%dA4y5f zA#&~&9#Q^FTl+nCa##``VLQFnUomrfm`py!n1h)fqCv{?yL=YP@#W7Oej&vboi#?~ ztOr|>uO1F+0XXeD^h)9**ofBTYAG|oGn`QhUSdFn&ptLI#}rXxVmNa<^Yejy>8e+8 zX@PoKn}Wi^I_Ry@Ic27tN1boWSyF>JHjTMo`47BUNi{{Pe3r<+f%bhn8Px9D8&S4t}}@YYYM#|(PWAMYRXn&)Ub<-REA z{BoqmAgAm6Hz=*!AH=)7aE;2vRySd}n_5_~7St_y!TyHCxUSV$&ETg5gK%DeY}l)G z)xYPTNjH5};}zA*y3eaM&{mtCT{dT6@t+XpDDs;gTqWcCo+x7x>GD>DWl@bfh&&({ zYpHkq_oQs9J*vt zH35dSG9A`_V|7;8s%yj4znWv!{|7w8rn)|D@~bubW4)c|%=#SkS6eau|A2=KXEs)P zL}&#I*!cw54F}Ntf^nYq7njDG(P4T9pUj+eF`fpD9zGZAem!%3=%{S9P7cOL(HQbm zMTLfWFLf6QNYkD9xK^C<;Ah&AnYW}j0AAjiQNe2JROIim6`bGWm* z0mgVOkwx|}+;43faU%x(uw5X1`>MlLaMS)ijogKasCz)*_Xl<{xmqun&kfh+A3GeY z!A49=2e1iHFrA0u!+oMe{AC+EV1VbLp(juBq!Id?pUz2ZQsIj}m4QCf%M_|9lb+Nx z0mt_e)o(i~X?Y8;PY-T7yTP#C^h&ADBk8m5PXSw$#V9EUS&R+M)N3?U5n+OZJT_v! zGG-cGaS&}BzEU*7jTT3>drtEAn|G-J*TMHt2A;}z4(%3f7F(o=s<9`3Ac z?>kc1vszd<6hR-wUuk&P5r6oL)3^=1;w33t-%Mi@{m17_Cu@DS5OW{d^dSAILp$~d^l-12k;H6DF zt&nunE}589(SyHh>po-EdDG*b!Jl?5gpmk)Cps<+R&}F+ihVb4l!KQ057%(;M6ox? zxwhrT$6OWK5@4~q>ubi+FDt)<#|QD*as9m!G;T+5M@Y?%t62UIkt!NhT0tz;_p0>I z-G9BOwJfS4oDq}eTxhbZ`h=Ew(!y(;Q1dXU`M7rB_+@3oDqpDw@_B}8mIqd+7%**p z@%=p5Z7Eh+JuZy&DpqtjP4vNg@H6ctbB0vtvOfR3{D-;lFIInqAV>4=;X~BYcQ`Jh ze#tY80W2Z{2v^HElE=)zc0XUPh-GNj;zLo(>ysV$w^*Z&eix@xjuiyTOfWP@4<)3I-QT~9_-*ek?hi9d_7-4wIX`-> zM-iR?j7l00`=S^XejR2g=E{7`uFvKM9aop_3gPm%<1k`rDvo@fibYEnt>)kfTnp!G zg~_djv#&*XfS8z*VbJ@C8(1`VxS=L{)PAObKBZfd9K9LDHD)YYbq)71JfaTpBIGzW z&LO6JA%vEKC+8lPxfYpF93umAX=E_|@3s@|e?;;B%ga>!&$iS5vzJ+hGdhMdJUP`? zIWS#1K0j8sfCB=}PcMzm_04u}*01dDVq=5qRu1-#HqSt(=iBN1->&mF7FO;*CocU% z!QSsY+0W83LkID6(2e35Mqtsu%>!dUE)T+&NO|R|wMpK@kTQZ(DFye0Tp85;m`6Qr z*&rvgL;2kVbn1CR-kHt^1oj$PGfpg+8^8j@ z_09~M7MRQxPPGb`>*C{?gvUFmjXIQ3i;65~%j?GN$###k>$->P13|aw1TGiamcx6$ z#^wDe_?Wt!zoNTHKDe$_W~vD8FRyuy&ZMw>FO!n+ww?7+Jv=?nA=>Y7RY}6A zaZmA(M>F*yuYO%z?5$#bcRem9-#6FGX?fBz+-S-1dk#bYdzax$1|Oyl*}qu*rW$kN zW9GRJ65T{20cM)HQ=lJChd+qlGq3~a$V~!O$tKOAoS8;M!E%hl#UZ~F)~3xc`>RTW zK>gKL!Gh67WeL6cYo&o(1%6g2BYqL@$#^E>?o_b|hDwkZCA2gxrUACLovdR)mczsv zf`r@Q_>t_T$wgCDiqOGtrv9OMU6qVl;Zw2r?p?BP$lZXP4q6(!9OYzcMvP-j7F`SD zWny@oHR%?oM+&$aVggaYk!#z>W$plplj>f56D;x^2>}ceg#V)hO-U+eolRwF?n!M! z2)M4g?boSJL(Oh(Y+dp7=~)X9Es(2jQ1$m&-Qv=3eV(r61d`Y9FOZAGVhjjhIc zY(3A-a+j8j`^Wg;Y@()J+tvZ)yxmlel;~p|+E#KnF{J%A)*c@XDT5_G{Q334Yh-6w za-@GT;dW1oP*WVmIQyGv>>-YoN%FE_^)C^#?bhE1-Y-d;e@+r58+x|JD0Kdkse`OY z#DrsSIXFT*Yw{1+TT>rX@I&n}zWp^idcAKhi;Es9_xKxwT64ZpV_2p*=W2KGNtieM zk;VcwX)lWh)QWJEG{B*%5+aq;2#UTh!o}_yqMUY$u9URCVy+&h?c|Smb+i1`yvsx+ z9iid}HXj>S?Acl%@*g(+dJim-|5qGg%%Ncf)XFpQ*JAoK$E_ce+K(};c*Js;eFb;> zk5aIDiVD@}+_XkkXz{xXsK4Etubx!AG;eWJSSo$y+#*lOSq|@xW0v0C7Z;$u)F@s2 zG+TS9$jzGiJv2_?&5Q4oxX%o9Qlq#KRD?^k?&n%CGzHmOF@6lvwQDRjm@UcRd=%S# zE3U*EjDP)Vn)MujKdBHwwK6y>d?k~W)x3?>A*1eatrb=$RDnhaB6BTl$(Y2g5}Qz& z^UD31f2>v|hC_)n>~j2^s&roRD;k*lqXJdGR+aSK)Ta=pCLEo$O3@RaX?G3uRP209 z{_B%Z(Gu+?okZ2&Siv;Sn|l%Gs+lbHQ~z617cMCV!{6z7w_A_NlpM?Z81eDXco$~) qJY$4*-jJ!BvYK-1zD1l~R@#n!&EI^Hv<|qfE`gEgcs~OHB>xwH>+8J$ diff --git a/docs/fa/images/logo.svg b/docs/fa/images/logo.svg deleted file mode 100644 index b5ab923ff65..00000000000 --- a/docs/fa/images/logo.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/fa/images/row-oriented.gif b/docs/fa/images/row-oriented.gif deleted file mode 100644 index 41395b5693e973a41ea0827844c7fe546e3565fb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 39281 zcmd@61z1(rm61VK6_1f;uj(%s$NjdV{!y1QE%8NV?n zvea*{v(Gu--q-(seb-*DbqQn5@s8&m^U3?UpZA6Ea}G`=6J%theFy|#!vfey0PqJa z6hQJaaO)Dl@&KTx0Q|LpTqPjH9#CQfxEKMg_kfx(;I08cZGbXgz|Ib!!3ApUfI?G% z@CJ~=2*^ADc58vJ5I}?jsEP-M4S_K`pe7Gkumd_EKzlwgSqDsO0kyS&q9PC*3j}@! zGGhT)EMTq#%x40wB0#7w0OJJKVu4|PpfLpKmIY)40e*fU1$>GpV08}=5dq8}0U;1T zM->jpve`WKn8XyfL`09wL)Ms8`ww$3?V==1lTDD z3LwB20bt1)*ckv0hk%1#;CnN$;sfkf0f%G2!8c&13s{T+PL_e~0^oQa*qz z6r}S@GU6{upDGzEfq(NnHZxFC(NQxdRaVo`(&HuHtf(O;)l%gpf5$4$EN&pErl}=l zVW{@nLgI~zg^miRD*0nRQXVrdGhG8+HDe`GGu=;mMqFmRPJZR9E+}Pa7Fasr{1oZx=RtV{V|vB&BAg|HV*64g5F- z{80v6f`)2J#`=bD^z}boyv3KA`o{W3n)(K$f`W**MoJ^Dq@twg$psHqNE>H?jz_JbmB;fY;lcji_uZZCt<8<~wbhm7rNxE$x!IZNsmY1)vC)y?p}~RvZ+*Qz-LS6Cj`p_J zmgc6$hWfg%wKdgMl@;Y>r6t8hg$4O}xjETcnHlM6sVT`xi3#y>u`$t6krClxp&`LR zfdT%0zCKWIFHaA5H&+*DCr1Z+J6juTD@zM=GgFf<#zuyp4fOSNKj~;|X=xI5;{vySTc!dw6X!XqN1qGMv?;u8{+ zl2cOC(lau%vU76t@(T)!ic3n%$}1|Xs%vV$*3~yOHZ`}jwzYS3cEP%Pdi%ch50rl% z{%k(zYC3FgI$k|FJQHDTJiEHqY+~`*!eHIO*yOXdiPhrwLQ4~~nNEy?`*Jmo=%0sQ zz0e@$wRGGv5XIM#h!nVlrI6T?H=FKkLwJmF9SutSWn`v%7$rqA+RI@lb+_Qv%Q6n- zCks7&8b0Dy*FR&9)H8Gt8^n=f&LlMZ;PX=d7!zYRO2Fr|usX(|{u2M<#lGS|#G!8D z-GG&ix%)@{5^Y)5z*He+1X@?S@7oF-sgt6dNk7*$x9h@2uo`CLovpY8X)59@hp7)8 zDy~ivE%bc{m*x|?<|uw6YciczttKGdW0*vsg@tDW&(LSF5SRd*la%R zX8%I%ie6HfygNZiv`i}lyDPEjC-h0#pliu8sw-Ypa>kIw^>|Q+6Q|CZJ5(D4DSHz5$oxbT#=RLwBx#l5+;rD#t zbrQ*EtpEv~&#I`eG_`Tie0ZK5Z^wq`G;+F%2K_*h^u$?4OCZ_mSV9vG!Djg=Jp5e- zFF<`kYx$v4?bl`61U-uD@l__3oai8b99k}lpBR4^d-qeb-%QZ2vl z4mmQ$14eZ&bJF0#6~x?a9_s| zY0H_Jk<#%z>7 zO6$9RvKrVSak3sfQVrcG<*u{btUcH}VXa5=5!!B|p2Xex#+*}0+Vz;pnziS}YwMjs z$(*W(!$$hY`<}+QXTV{T8_esdG(qz0xF$8m>m;Whc6M6Y^6~88JFc?r*=c?L;|D*GQCN9=ZX|Mx6qSG zYXfuY#7T6~mb#~zxwn{3MX$&S2`(k&5zfvwXbD8Vu9%6~-}&;M!zIqEem8RmGr z5SgcxEZgpnjmdAs(LAHT@pFG08IMWdB)?4{^ps`C4{Ak8dHsIt zljr&VE|rv21Zl0}hW>zmbRi`T@362GlqJxcCy7$DOjugEH^{+B?Vehh$XiDL;P@-| z7%wkh;*VzuskyYkWO~%9G?E|G{79WyKTJ%O_F-s)YBFO4XRBH-OL$nKI_t+Y@h8eg z7(Huho5AKSdR@Jd!<5tH8$v!cD=jpTY1-)EL-mk#S}uAMO{!XGOvzefBI0P9&4U@P@=-eK`Eb&!hC*f zO?7jN_g>Ni$x*t?>RN=dy~@Rflt1V-Z9QbW(%DiH_Lgk`at^P1_lwO1Domo-5 z_rvJPQzgEfznGzGXM6FV#Xp$TTMX@2o90?)=?k)qeqK#Bt@+}uue2G_v!z$$H@mK9 zQ*tsCug9K?wV^*nG(Ce4D{W$r-hSX^KTC>x)X2-Y;~!8tM+I=SvD@qf8CA|R-sbF3 zVKfTri=1amIqLWly&K_Exkxi<-r>Aq7_Tz6MEAm?+n(4kY4&tMqWO@-a&b50Vb#hj zH1mN7o4vqyKbGVVjs`r5P4ZQ$)>r|qp*W~Xc82`2itX{}=$TRR==l6=>b&xU4NX^E zo@s1=webg^8Q?;4cMJ0|&&>6h)mp&X7ts4qdt2hrT~1;9DEoAtWb?4=A>ZyEqtz@n z({Zm6pRXTN_z1VeK@!Y*?}eM?_($B6qQU6Bq7dsfIVRH)+^7S>!5?em8z%+8*+z47 zMWYd})yy5+z4F^}iH9DABFtTx|*k?WZ;DN2yK7 z@Ez}_TK6{t_CxHB3omR4g&g?~9r-m)=HEHdEjr{pG~YJ0!#OmMZFYjV*w%_CUGJyX+_55j}S>{%$|S#0K68tPeI>RCDD zS$*hPi|eVsi)ph9AE#!#<;X!`sJz(ZN6zV-v>OD5(J#pwg zg%6$K0HAX+&;>K-QYdt#6uLG9-8h79;rr}pdSBu9-Zk?%4D~rK^*J5#IXm=$5cndS zd+!Z#w`)Q=+y~^Q(EVCy{91+l>NWkU-2AFj{c2kLN|*d9(EZDo{Kz@| zuM>FP6ZX8%>3NUSyQ<#ZJJo~v$iqe`AcH2r6+IxyEg(`eAfYATF=qfjr#n|!KsHAp z`}@GurGTu|K%rsx7X(3G_?{m_ZMO#P#1CEGrMbx6xBb*2_Zi>i-7}Y?+rbTNt}F3Q z%4s&hZD&0hXPIG_JgSgH7u#(0V0ri85M)PAO$SmMC*D%~2by2Lu{+rjgm!6!dI*Q| zgaTG$uAx)6jf9rCtlo#YYB`0Lg?6)tRt<&dG0PT11kvNmQM2!(wb;W+o5N!Tqas?Pri7yh--iyj zhH=wG@4JU@4TpJ|$8423Y^6oVHkm5AMT(V1eBz8P2@5KIZ`UzwgUY2|HEdoJM*X!l z2rE3c$vjRHJ)|`(0zEypKFti5Q0rQ_-E~^ZwdLrAwD2_ZFvhl+{-fvzZSj-B2@K^x z3=?evt3@Re=Y7lscCG}T@Pwtbc<%5-E|Elz@E8`EM6vWl$&rNZvhW`-oMqDD zE)m8z4=25sjl{;(SF*6}YK`b7uz#2COn@076CS63lEAc!^bNe)+4Fn+G(TiX>PQs?u4mt zR?_@1Q{z1x!;e#;BWZCf30mcG?`a}sS6tMG(n-pbR6HV#j!h{r!9Z(n8BToDBaZYq z{VQQcopzieZOV7)3|!Ncs}{+M0~t%N(jsYXBEubam+imFX4wo!dz7cW^+->^1kyuR zvJx$9dRw!cxU%MHv)9tICuP$~mQvD=v!`j324qvuhO-FR)BLGpOK3Cdf-~~8G9h=I zKhZ{F)8=B7XY8S6HE(Bln&wIhWg25<-j&V#Dx25FrGts(+C}TSD4RNmnY|~HbMGW= zOFP|nB%4krpFt;w4ZnaUBR_@q3mIMF{X01v+&N4cIUC_w6ru&x5ovs@`CJhNFWL(z zxwDtc(zDBx6**lBwDaV`gYLA)z4OfI5Q!&=h{ZX%eO0dLrd+ICg`J*zo~B%GZFrIP zXr6p)F4AF<&Ge^%CPjNHJ-cLGo&8Vw{kmhxC3YFF&FD&}O-k&!HLt1b-F61_gqL#E zxr?Rcicv(1{7!PDT8eM46yu69fDl{4@N-U(6n&dPcBT5F< zOKn9WB11m*Jw1!Zp<_T);o@@2#Ab0&}t*e~g)elv8!Ij-jmAGD2 zJ)A&emT4Q2Aw5 zo!F|%##7hIvXX$lYGAaHjHl|hRg*Jao%WSiUKI+^)dn7`^3056@zd(WlZH9&;QNua zPwC^HU#VxOFV`z;sAp|9UaP-+rNN56p>@BRim*lBqk)4bPwcc|^iK7w4n0er@(H;{ zg)fb_yc#E^+tfZarA9V=s%%sE)JA&xHS@z)mD9Gu$|l{@m-=JjpS_xe#d2M&TGU(W z3^SV}#4KLZ*ZXFs_B@2MxhB_PblSQymfKF>)j8H| z9@*PL-_@6y%y`-aI%NAwMkc=~Gfl&Q%y z)-C1kh^mp!^i>A^Q~A10qKSwf{Ze;l-q24jM@%%(R)_OUDOdGW`~V88qM9|M>Sc+> z^1UagPp6L-##ORL@^nU%7^a|=13j%>=r(;Xa>OiW=B4-0uuJ9^K* z(;L6R+KnVTL&C72M>|+?H#eIhSxX`rC3;W?tWQK2W~(Nd&&CD#W@F`MGe#FUp>y05 z1H5{(5SvlVsL`)iCM&aMs1z3VvSzS&#^m(IUTw?=WLJtRjQKInOXAE2Pt5ga%^zn@ z_lRFsgmNZkO$N&^3h>PU42uh=9p1#Ftk_HAYqOf!D}n2iCQ+F>jPW$kj(q3@5p=k? zD~LaO)fBq)D0`{MyZ_qwq6NcPNx&4&#BxIQ@}S;)zr^wzK43l?x(;GJE!-Q^(D8}r zaUO+L2gXG*?-B0I(YENt4u!QGqE#-w)#auQ4c?_(=xA+aH>E;XRduG-#v-h3hH!nx z45!?Ladw-y#$sc{40~OlaowtO&VOS4t;BSA_4d5Y#s}h^k*@Wiu14a?d19uONr@Ho zk4s;o1~#B0I3!y}$163uo2X1{%NzZj*_+MT6Xg><*?e1fPgWnUFB#x8A1llm6EE?{ ztUZFQdWlWGmYedV93c`|AI90BgYDgeZI5p3k>lTgg;?#D)ND@yJVe_Vl>Ewbx_TGu_skwi@MY8B+dt%Ic;DEbT|M9S6vN(TpK|A_1 zJ7G_L^C(9DSgB@zl<9cjY;R(6TOBs91zVpWcEUR=`X#pqeU92l zmb`(r9orSqZ+)H|zHEDh20QWDJW+68HD)@kVEkb=x!GKEfSNrQQL{Hoay(CRmZX1{ z0KOE-{VArigqSnFs^$EvvCzh^U#kC!+5R&&O(K0Zq68qV|iQ>L!J6ZCH{{x78eZ{+;j z9sMah@c*>*|GMzNpWo8|Pv8B&F8#kQJn)~X_J6&4;QvLV{Y(0PU3lPs>oEUym-BCw z{$Cd!0As+fExifJCZ{W;Z!Z*V&wm_INH13)LRTmtTKHtNK%^oY{F$&hC;fl$nco7; zuf_k?8~^|NiNJrtCjZGE!~UCZ`+xpTR-vHFAX>>z3JR{CUJmZwzF;%iBf!N!*fliF z8*C;A#e~GgL?gSN}NXQTf z>RWz010i}spO!?!)=yk9-KoYRS+Y4pnI`S)7BzzXs$z^0Ff}rZPU4Kwb#@+{(1iK@ zhkUS!-IFi2ruGgtRxY+~b`DxjULJZ-AAjdSPxoMBt4jhupM(UV-$fFFAc21%$jDET zehILPBz$pM?BX)`X#$8aWE3tJ3Oj!J*co%1a7Go!YG(zYNCQ>XdT#gGO?%Kn|Cavl zJI9jX*OvWqH7{PWcwO>r3KCsbL5>m^L;m#$2;>F0$IIXz(`y~zdvw2LcdqAS@$ znQRe>7By43W@09?C7Q-19eNd$TcuVKIis70l{T==6Tr^?S0~?03#ui29nr}TLW6`v z1cyb&1fps9XorI=g_QK`X&LER*-^=PnP6|fFuyPt1SBfU%K~ci8X}q^Tbiq5@!1ib z{2mf7U6T8lqhsR}^haSU5FbYwH>tn!ny>et;kpz?j>6`g^|(40q7pe>5>UH9d8oc4VY)VYy>zWo2!> zXLF00nsxu+@aUL&Xsc^y8Pc`7C^^Myi*}9ep^SGNt1C8!x+Pg`e4{7+Lxn&Hxl^+b zsi4OdjQH7>07~)r56gZIm%N$Tr4(E9DLW&Gb!QsVj22s>`2;Gxgx^43#5{hx^Ty+b z=IdngD86UmhAH2|U*;Xtw-wF2N*DgR(6Q2|Ht2e-l3u^v5lFM8vzub+>JV1h}?-* z+AZFdNe^Ge{-Lr=f+qu)nTk8*hD_`g`*UIm8Uz_`K%$_ffF=PN5a#c8C3ayzf7unn z+{FLkGJ>5V0V)${g_j>Gl7p7y|H?Q!GqiXMt9j*9+rInade=eGB*WqEP4_3a4b|sZ zjbN|7v8ge>?~h~Vj(56{2Cl zM`8-l_W6f|=?e9w6J`u$>L2?KEaYNrH;%qYYuYMXJ8Lo-dOR_qdI>1~Hm<_% zd#A>r2eRX@n5BqreVL3aFm#iY%?Q&JBU1AfL8HwzZoV5BEUM2e=Y(+TYx9%ZLzv~W z&Fm7AJ0m!ZW~}&Mr@(+HUR%tpviPqq2|@w1;vpm+PxxJXI8Ux8x-(3AT_#p~^ocyTX*!hX7X6Kuig!$tH8v;*ouZ|=)AerG=@dFj z^CpmeDMX`rhVnpdeyB6iVV?JhzHI6@_=KyB{7^Qg9~>cn_`qTUgg$%gxAbrz-~ znKQwCvcJ(;|AL9+%<^P!(H16oW@8DgAPGg;EnXWUu^lH6nzX{+>YhO1drC9ybVHhB z8l6HWb=u{catqHisLtrhF67eq%I>$K-N;>Ovs=jB8N<;*P44AT@gka_QTC?Waf|oC zLw+Cc%e_)c=Eu_HmgvNL>B8gxrJ3+Z^tsj%p-@!6yfiCOj!-%x5>2Jt6y>9=Un;cd z1if^Z1Kdk5w^i6M!FG>qZ0#T@mz`Z*Kwg-=gQ+*v(J#O^zzh;>B;x526^RlQ7Uv%p z9-3sGVwvEV3C;eJYm%Rr?r`VA*&?Kn1l1ZIcepvq$GR@JAmUCI8omRy6&bU`Omo(! z)oKH)XBzK8)>fOrL3v>&mmu`W5J|`m5(>B;3(eEE(XH0J^xxuG zv}SrXd_8)YvF@Z<_NEmKxjOb|6sGpQSO&Ca0P+g52e*Axs@i`H(`-nE@Gu>o2@he6 z5zg3}SAUOwaEt$sGE|D2&6hn1m{r;$g^j2@Ls*SQ8qx}=#ZCwqWpF-|)u+HBo(I%D zO@DL0FPSywiQ2U{OE05es)>k~f1w@7ls8qs$<3xd6shqg`PoVTqFsr>Y+KZw1BhdV z+3w1Cdr8h@jolAI4*n7Yc@+C|L!E85#JCg9Q_DDaZm4Ai8dcaV9>n4ohT2(Pw0dr+ z5rmE-zZohPT8mx&Z-&~PAR2=(R50wMRFgR%!VVfRU4OW{R23o#v*A72UzqeXWwPb@ z0aj?UFcMqf^z3B$HmTiN+KVYm?!5_(Hj{UmfnA@#2}~D%Cq-&R_l`3zh=?02U&RWIazK zshJ-jc&Sik58pt)fS^!MT58uQH`=f;|F}fYgruaDRNwRr=E&$gR`-m|khGGbtnxCS ziprwk)SCFJ##E;KHiyEm#a+ow84>L+)Un+U8oEb^$Lo9JbNih;C#C}@o5z+$)>qf2 zYO^=Jn2^CV^5paw4BW_=cZG6E1+GG{X?a2o;_B^gP=1eC@~NwLMQ75!68d?z2}+FZ za0TNDd1C=4qlw zJ&WTC3g4x99w2gn5vBc>tUvly1>a39woh*(C|hwTZ?8Uep)mkJdlog-b1C_j&{a8_F;@--&>)AQcvo| znS5ETPjLloqD~LT=R0k0GxA!0K4_TUwej7XvpYHbD~&{z`N)TaNF%9IryS0(Q*j-y zK1T#z&1vUT-Ki;;yCzH1uB0wBGj5c@nltV+$*D6Q#4NQlp3F@&vtEw|G-tgz7HA+S zG~btIefUw25|A;`wdVYI@!aQJ{Y4mCRsAGThgAcgNC@BamsW7U7bq*zniTxr;Yck+ z7`m()`Yv8LDV#5#ASps4+?+B}!9+O4haD2P7;Uw`OdazCeS0z1X@x*D&W+eZGv0x5 zB_$quhM|?{EFq$m@ymRISE0{CueO4rtUC^qAFM)U-abknvv3{XWT$`b-_D$AA#NS)siA*& z-3At=ZxwB(u^(Y;^W`NI&Kvx5J@fu^$OY%;FEOK@`co5dhRHWJyMwP7BoGa^)X6qA{HI;xt zr&lcH(p_`5pV_K8*eN?d*SEB{)^+lLzU8=BA9N-Vhzw|5-&BUz=;LeHVmO{KmKGIwW^}dC3I`kM=PO1m{vKQ$OhhgV{ErqF_l5%i z689uYo+_^^kPUD7tunCZ&J#hUOQLCMK65`xO+%WmSZBB3OEGX~s zsBEq7&+aNn=!N2?4w7{Y&P>g<52u%od4HQ!GB$HdhD?DTQ6AB>8Gn^XBaxs9#CY zb3RlTiD11w(LnH;ymc#z*a-EWyP{nWyb5@x9hcUZ@-jgrN~9>gKSQ*DrK+tcV=zat z-Xki#D6=O|voBq~y*O*MM2bB#LbN1%{I}!+1&dg$H1|HL%SCbluL9`3?Hv)xMP!)( zN>{Al$9?+pN6Wp*j}s=WELax@vY*vM8RS@X2a2^|15qdYOB1Q4&9^qfIn-w}{Ls*d zWjQxSvCSH6rVW1%2KGRK^eyM_NPJCQFX4c;>T~V_KsppHoq{;e{+k;=^`xx?PC||!#~u^;N@8)dB`<*;(ZB& zxkXq9(z!zLhx*Hc>JR4DAT;wI8dJw=;aY3}j6W$ANJOadOqIa9oXIL{05e+$<;hsR zq3#vp4%58Za&MByQ2O^w^KI;uq!S&|Po~;TnYeFxA3fCUwG&8@*1cWyanM$&hK={o z?vG*?i$vm&Viukw+eADc_#0?usqeLF>@R4h`o{@B!O~A?29;*Y?i`wd$J8rqOH;qc zR94dZ^O#EHL5d8<)Y3RNs$@514_au;j4NF=y0RBbxR8oBTe%yB7uSqgf)DS8nX)ha znM{Ho3wdkYRevUy_mqD3xLRkSQie-{q`!h21fl>c38+Gs?PAH`kc-q8coC_WRAH|k?Ms1p!VwsYu&Q9cXsI3fU>f5=PySN*=d3bnvoBH@d9Grqt41E1;ppoJJG0`TmapBh9iFRqW z8K(GTzl36ge1udZgyNXG2HwSY5-TX%t$J{?rtnj}@ zvFe_7B(S`@LeyrI))y`w!{<%=mbyP3*Nc4ILQ(x~hCqtT4~w9+p-8n(o6EGU41ERq zF)Kq46z%NFP1k3}GfMOouS?loymGRNx* zsY)({dO~bUu2&7wm7U0;W=if<(T7SNWZ9wQ9!%veF$T7iHEG-ix zzX_jDtYW`OUf$IF?k*cmduIo%tLJVDIG;H9?!v+lVk-ZqhsZ}4A#x$sKJGk3CjAv6 zMJZl@Arb|5zuh!XrOtx{SC2mI^IWs@H63;Cl6-|We*w!T(h;NO$`ra39P6V^?ZK?e zHC-7LtF1`C!$>7jk{5XIDf?&Z&c9$JOBAy|yBAA+86RP1C;R*7-3u^8f@LR~T|B&d zfqEDNmWg2Z;>R_&DF;l(7Dam;)FDMDWC>#3n*2w&+8U1}M)ySa$Mal3ntBdaEqK_c@ijDu=%aPV1bHC?AGu zpcJ97T97msyzEpqPYRLOGgl22vw5EsCKcbB5H6Yjo+|8FdFyFqlKn#Q8PYi)KM$FcBfMo}Kwsvs9b7*vMWNh-= z4LsOf_l>EkiKW$*)wSM@vGr}z+4%$Hp6$`yr61q7_FtX6^N!_q+=O60bpu;rR#$_T zueUIfK69OFcxFsBeYes6*5g}O?i;EK`;x5cTCwF(J`W)O$VM6Fo%|$(O@p!H{pZOx zO9n|7-(l}XF7Lz9ai+u{3End7(mfPqUf}wdEh02fw*pMDw5k=}puHCi$xsrZ3NRw2M%t z^Lz`1pT7pKOzU(neQRoLk57J3E7~FBw0yKbyym88gUNXK(GiEwOwkEL0-wz3wyY4D z9pNpvI7bpjd}V72a+!EnqGMBK=Ns8FG1&YK2n>TG;p&s2)LH0*nU+}}zB6=H=#v3_ zRX;XjcS?F65W}GKlOV)M3J{m#R14%)xUU){YUrLA!s75gF;q^wj3P|xxqGsgz@>8# z%;3D;tqr+>aMF~TAiPidds}&qCK80>wU(>#=}HWAC-*XAFcA6DP0;&-xr{cbc_N5R z|KFu~NdG&{L;81W-p7C#en(6q!W$ScYnBS4XNyR!Pz5R^qQeeu$t=ozSa2sg%$dVdadV<37aM3puN_=~YE&jq}Eet$mj zjkMN$kc@iTeDDX8<@pdrm-`E$s=-8h4>%ES`kw=#?!*W9gNT#kcE~Y*cgWRqk|E=vA#x$T3x& ztavh&oh7&DRHD?#85E-xMC8^Yc~%%yAM!*RS01*pWQY}XJeajI>LlCzWZ1=o(wX0V>v~jv&*PKR2fb{A z9fggT&`I~cy?Lj**Ds@ywKwp=bbW77(UoC;NHs)ve^@gmYk%ZZ$@>22YZS=AnDLl8p?7 zJ8o^qJv-P(fjzm1RtRt`|2#uOi?zc#k5&+z{|n*<*F(ff5MOEe8d_Rwn%X)+>PB;S zJ*=;xf1nL)5A=?Gn;09NnVxK)@17lA8Ca{`=-XOeYTcQd-2VzWIsJhExj@7{_A3J9 zYRQeOcfCp>5-F4iCvF@OVO(UJ2%!r=p}{m3&B{*lTxcPo{|liBU!4p6j^KYUbgi`g z@4#jDUol_sTp$^l{|kXcE}aX+Mofk{Potfbw6muJ*4y7E#MUs<_-tU{+r%{4)XenU zd|&rc3+eFa=B1|9<<-T#mEGCHp8bQP)*p$NGud8F@#$Z_MDqdu7M<~YPZJJuVmT=B zh_zK|;_9ASlSC0+!C*;zW`C6}{SAK4T#G(aV-zL+bwzO__NvapW!&*DRg8C-y2AOY z)iJ%ynAStv6jD)~i@fuRsDNEC=cVYhbCy;kYSKIzf zAUj|FN+7pg{<{RS+PZ#y13`be@Vxx?@C1^E)Q+)~8Kl1)zf*L$0w$1-sC+cBj(9ji zO3oMxZb~lK^f;7U?`VU}7h+nW8Ao#EP-SPjv%z>Lrg$@QhX*!GO5Q-yzcYah{XK!) zMJfWLIoN9alY|C=6oWbjUcSV9wN*GHNo2fJZDR_}Wkl0`!8v)nt$5zDDMzj^-_?h0 zG^Iwk`_bGwM`IX0r|{F?3c&M4K+8cjt_?;I7-+tIZoNSj(M|m*J(4hKfP!sw%&?+= zll62jxK)|jLVKaOU{h&~OCu+2FOTDKrSTb@W@b-M@8srg>*C=Jb`f2D9eo0vf`UE3 zlFUCcFgh|KJ}$;H$u}V^Jvh@YJ0LeL)jdBprr3`7qU1uDa4xu;bHW%31kvGjCU!eN zfyVinpWBVV^Al*C@uEDUex5)R>W~nZmJJPv2+k=iN{fliD*k6D(EQevtj0k!B^OGk zufC*vxv{mm{eAc93=D7iWNGyf7>P!Q`yBxXAudfBgx|s8HM0}ij%)D1zhm(-^x8}# z^yXuSH5x<8R(GoV!H>!wB-P{4$ZLCX8mfsz&}&(ITgKDqieWcb|1NyZ^SS{t z{Bm+rbKzK1baGUFNqlByO>IfVSFjr%hljwL+QGsEaavD5LC@gm5Li+4QrzpBgHcRP z4KJE?vH)DL%z-Gd@8e_QH3X8!wvD4qX7@qnSk7 zTdsk56)ByRuYoYKvoG1>_z!7@_uAbVxr@Cp94M*WA8>X(EeoLPh`ebjdt=3aNi>Gv zTm6m2gBi(K;uxwR%rB~~0WTe;Iz{}J-((Ax>=eG2qVJ7V_!2%}#`aXUND&w>#y+{Z z{=rjXws-zGIN_}*)$G2w`Czs}*?S%Jy8ETM(z%WYZN`;>hTjT8J*C)c%L*1@vAm05 zQH!7>B_s>T7OnPYeQakGr+Be68m|H~>9i=(A1SwmZ7@WdZcP?_rd#AGFV&u@i{ox4 z9XH?mHt4D0V{^J^vN^71E14x|ho~rCJdXW6uUp+Ms5vLsfe5bE@lDMMJM_O0+#Mvv zDUgl?c18a#uOkz2oALNPuNxvq!JI?nb)^YdjE6LHzMROybAHc>-BlfYMfO8g1Nitb z?)gc9UC{urJWUFEYgnch{LCgzHRQEFMp778{BUBpiU%iUgseJ73iLii36a-PM?XQ& z1M|9(VU1WP91+bp8{+hoIFF51%>;WsOszzBiE_=P0Fm(f$-$Uy%W>gP2(=RuCf=vV zr#Kwnk4;n1*2*yb)er^Mau=!W@BLKyzr#<#7x@?bRL8&Ir-=UWQx)fepjp}xo=0|K z;{SbQccMhln`;FH>ziAu8(TZTv$C3CbzQyn-}+lYPt`NpH$FNtJvGrb2b&pM?q980 z@7-KlY}uZi*sCG^Bh!FKf%9(8zpKXke+FG}Jx3Q-$)7mJf{F3^Q_>^ry!bOAhd z$@*kOyyI|vrpcQ`FqzKW(CH333rI~Y+x4yrn3Y!N8DUxKPTF&kt|HXauFTnOaD1W z3xl2cyv^sEqo-yE%Q4qeMG)x1FNOrXn=^sNZ~w~|fDDN^$dxw;U*M*MO=z+d=X}?O zM%j%VDx>U58J()+L2J{Z?8#WSH0?>-I~4EDJj0RTb!W#c4tf*LOr)AlpxZwRHzdk=`cAtI zY~@eomTEPIi)ZH{NRC^julYeLx>?1uE=H={U2aYNz`8Wh#!MgUV z`2>MfgR+2J1hg!>Tjz$43PC!D@gNs49t0H)9&}^eK_o9RB|IcGF*zeJD;o(sBrmrZMr~+gP;(6i;)uLXT9)o_{p~D$0|P@N<74Bn$*Jy{+0o&JiKXSP z`Gt+;t+k!Swa)KThl8v8$19tsXN_1wx~P|h;oblLqcD1?J=T&39(M;a+V2XpO#AZx zUX)zyAmUHnmj^29E%<0apfB|NkUvjUG(?Jp8uRmzzibHNkiSCQo8pKRtNP}R+J>6C z#+Is^7-HO=Ac)=#!zc$&MjRXwA08b6SZf@<(?KT`W zo*W-EN-F+Lv2yYL*g-(RAM(QW?myGDbvpeM*QSekTL+~;yr(1wB!5jJ$X_ZT`RfIQ zzBJ#x*oSOWx02IH6&AHabmY474>rhxY6*&jgHRuUr^!#q|58R||rF{$RQFy0*3wMC1mGTq&7u20dIs4c&kA-E`_ zY93ob&MyDfsE)P0^60NdwUmsYGd_!fk_*0G3z-wK4Ze~axvb0#=-QSPJ!sMoNj<6S zWa7N;_J+oJ6P6#wL+OIuX5DDdmK0H*-9W%s=84bH$!S3N>KO~hd?1+Ss0Bz}EpDqBk}{9|2fQsndAu%syc@*~P<_2AYNKiZ#&>dyh9 zub^hZ+g?=Y|2U`lURw_f4`xfQ?9_xxl&T{q|edC^5**Uov? z7cS}7bdtb}-;j$tkh*g+$R$Sja#@fu-G@LRn!-$J5S3eUdQ36&@7gV`J%RJn zbe#RyEz)jm9NZy}XjiA_ijpsAojKrStSO%~HLtFA9n7-UvR5>=bu=<_cC+wMAe8$3 zEKpwpvY#@5jPY~lh@e#QyCXtG(iKhkU`3bL`C!F-DmYm2;`hM{jnauy%D)+`p!`26 zRidF0A>rV(63D1Z^t}-q8=aZ~o;>QG9hsF!nvhh2?3Wi&kXi+fECtpCHDovC)Yi3x zb_ADpdz9Bze*M-J8{Y%POZiqmFxd!_QRk;7i$;C==7$z$Haq9SzPD}c@3tTHSFD!c zacLr`sE(-IX8bK%v2`w2v7~eNnBFhGAc7!Oj>o26Q5k!qivsXy`QZZG}H6RfPR^_6ah6gs~TtsxKC- z8Sk-#n}1TO3})FHee?g3QpKzQ`IAybT04P+7_2~4G8Zjr_+Z7amh{OkD$dXB#TECr z>;*-#3(Q_7x?*sB7_Efo|s-s7C3$ z0wyW?wEmqU?V9VX@_Yqp8>VV^vPWO*l*tkg9vqoV+S*x&JHB<5ad)$Siy`p4iNP0# zl?i40$YOGT5Cg}KA`OB>+SzGI3OZ0G&V5tO>}f}c5GyPaZ*I|LEn1s##*ny z>fGMc%+k^D;r@@og|lPlzIS>=m5!Tub+Ipe@7B6JAVxiH#*KgEK2Ix=$!0)N488Q= z_1gyq5?AmGC`(J@=j&u94sW-9Q~d110@O1J%Jp91QGv zk3b5ZGXxR%1&0TQMnnft+=C|gP{hPUrKFQ0#tB2h@-pKSOYnT*1BF!u71fm${&jgx z;Vlv1&SNMId*B09{qO;*-sgCVgJe!{&ZFb7vccFhkn{Lv!SyHS(N1q)eD(ACRBjm- zzH&s3*5uYTRXnDb^~|_GQn@)?NrK(3Fvz}^xjg$EdX*J(ti)yJ8V1p$)DNxTNsQxAd<}%v;V4bYIpo|oi?%b{CWvnnlM^5kH8hFw5K^;nc6p3 zMY11#ow#vFdQ}n&!FiNkM?yuK0SVY8jPj@k>nl|era6rC10`1^uo^HQ2@jQ6!;PYZNbgNN~BA8ePKE5~&A#Ky$V%)!pq#o1Bc-PqL{ zYT>KuZyMm>BdmR4|GVwO zT@*NifC~I4SdJYMFD9nuR%}Ti`6k0%jDnFn0e#BH?rHhLFu;q2Xytp_tE5+coc^?6 zXj_&d#+4O9u!Ylc;}ym|-qH=`NZs2q5ZOq+?k zx;v||xVmmj;}9GY+}$;J2tfiAPH=a3cP*?6cXxM}!rk31NP-hAxFpad-+%h~PoL8l z-F?@$`*J^f?zQHe?-+x>Ku=eb+j^;5-#VIQUDPtO#MHdFGu{1krVZsDf#lca#qzK@ z?0F}#>*WL-<$tXlB17^&3fTXsoFAx!|Ee58kN=~}G5oKUBXZIIk9L9Mb_VuUyKtW$ zC2%`p93!-ev>W*eUCKC4;uvHSFOB?X?a$d-nD|Ag)i_C5KCLiW!s$&xiWo4BCYe9{ z&0d08--zhH3E-}vC;m&h{e>3)Jbpch{ta4utyvhT$$wyGIFDMY45oC><)N?^DqAgpp+4lTs6}DLYrSgeqa+4LW$|R-Vv>xNEXqxmt zHNjK=sz9Ir*bbuoUu_2g|LnD({x7zJJ)izlb^iG!{D0IV2K<9KH13UJa<>1}2`hw2 z_=`AbL681H9P}2BW2||J;QGQzWST3{YA$Ew`;p)8DK*=^Y;0zFE80fFxkIqN4xpKy&8+Tk* z%Yn+np)49a`Y8WSYW+7bvwWx585+}n(V73LGW}@i|I{N&n#|<>Lu!#>2>!p3S_S@? zzjFT~4pcE8uP=6DKOeusHGtPqIB&ct>rp|x{4j^zA2-6^wF>z^#|r_$za1~6Vt!#>VMuX)Sy6aNXhl?QPJKcnd2>NEsH&l(ysNX2TJ7)gGNh_HIyp5k zK0Wse@lGs`EUb*KtxvxyZF+iS?q{f3;QJD`n9VT4IOXa?PzHN&Zr|X=?<|N0| z|M7qOd-0cmVuS|&+Y0M{4V-{h7Wo@Ep`65mmtR;=TvS@Zm{M64B+SY3XBO4SDO~*; zIFU$PFTm2<|LPCXPn-nz&oqsPlyG68a~TA{nfW>^uAo5pZE>G}BF z_HHzs?Cn>mS7F~1GC&T72U^lWha2sHm7*^2T(jE`nI4H)bo>Uhog)nI`r{s}5DN+52Kvh)gXLm9=q+Z} z_-!`?VA=C?fk*2_bY2=Cr(em6&^c~zi$agU9e4y(B;i1?^FfKs?NUHYftdRlseG<3 zT#Nf$scM4|=;B7gg0$hh!1n3McB4Z(1yTM}&qMUFIUXz!7rrf-j`>!NJ*N8DADdKKCN z?bNuWWla|T9`!JrRj%(bhH)Bs26*Yt>Y7CLjT$eqD1Iy3+LS}8lqw;40DU8`&Vf|p zSP=wcQzSlm!&o8Pp=hB8PlR;uGdXHK@vbyjZ6VT|e$otBp1KSvM=i@#hs>W1WKN?j z%U_*m0_n0u7WWc#)1hoigxx*54$OS!TWJq`kA(%aZIc)YXR_0#4$O5kjpz<@hacQa z17P->Ew~Yy0}P@`=5}`yFm45mUP)>V`A~6+RBL2;QF_BfKIn+mvaqs4B_$TOHwEw+50Cf-aOE zfN9y64IAls`wz4^bV={&ys|dZF3>MHDj8lnMWDU*%YWUN)8~VuX)*hw@=MCyzREFM z38SQ>&xoSAY8`mx%InmPi=08Gw^21=3xupP(E||(+gsK zC7hb_rK?50VA*bdNZ8Rq#*koUNoLqFzh|y-qqE)|g%8VvURkOyO>E&&wx(*lraUcfbPw1uw(W9MUn8C-Wi~9% zLt4UCyNQxJR3AyV^WAECEsGp}KdL_-^RmAmm7vZ66Q8hhXA2qb7@~tUCI#p)m1gAw zRxq=&0K6UQ&=wZ+npqR1DfMrZFy+zO4azSHTED7Vx#Pl7u=x#hd}~D(7R}H(F8#u_ zcSED&rM*_e_^z_#%e3$J@r=d9!{oC!GGQHugfgY@&DNtMoNHS8(hiB#BqUhi%Xz8T zF&g&syb5j0RkFpo7G9_}tcYc#%w!``AvTgJeK3kf8u5xiBE>`qa|X$|UJ(D3@K~rv zpCRhcLu4$cNX1WrB*+A=7(Yg2U!3Et@^($sd!=q+%(4q99C=FH(TB|A62HP@EccgV z!_0cjGrZ2Q@IMJCIf}}1#l7WrsW7Ig@OPO2;~1w6UfBM$4rba~nZc9D$&88&43sMpXXkEG%yd112irChv!b+}3R>uy6l6~fBc|DVm)5V$<78Z6 zgMlb{Gv-&GfSp`ebYW8_MG;?_5DjBWuOBw@Ir-9vvylXC9!`o!3cCQYYJizneOwk7 z!xgn5P}){8Fsbdj2pLo+Ys?5k@ecZpcy5oJyH;=(rAUcTCj)k9I2;9%Aum9#91)u` zn6$L9h+X$c)|fDWbo*VY`pYSUd=sNeKJHDaw5}CE``bcFxl1`mNPrdvJBUXV+~yH^+)iL`&OReeK&gY%a@VY9bKsY5`v+U9gq9sI*H>%1woZwItm%a$^!M ze6=@KorY?>OCD{A&C7|Ic+|4ftqEkjWRkOD8^zAJ9{bcH!|$mV{!@PAF+Rl+XX_fd zksIhcCZaVmFrD_RSv7Q-EfCQr`aTyyu*gtU!B$Z5BhEr1tzIx90w-2@9D*9Tu_(MO zA7NgUjX=uGXWaVQH@F@Tp$az`vZ2N= zR0&Isr6j+PlrkO{O68l%)hCbD+8-Bdqnj$t<$h?bKQ2M0nyTF=f9O3uF881}*M`ZR z7!f_KjL0|Fr@Ty_n0_b>dfI}*fb<~BUjT`IZ6hf_`tYVMLO=Za zh8Y7Hpp*X@uKnwuz=^4!(cYOmvFQA$^q*e?Cv@g$d0R`mQ;u2-zwXj2@au+?T&8ya z+JBesNrOLm#G-hJC=B(8;im|f{{mGFK(tbjBoEH}u(2y}H;?PZdYy~c*(Nn?kCpoD z5-*vx9EI*R0~P{enS4B2JoREFj|=`-(u&A@2%SG!2^7oGJabxppPJC)-lYwm^h$pE z`oqutg=|-MQt!+g+iSy(tzY-TM5yu04|&hA49&AwHU7_BV$;vu!Eb`Le}vkO8{2r~ z@q21Z=rhyU@;Rix)t229eo;Fnp6{WC)N(rAzE49lt&h-w!6V~I=Wyw|LqzV} z^-f0R)=s^0-{RPPl5z9rRAUGKpz2|W?kc>Rw79h?j@~J+>h+?UBaQmepuV)Lx(g){ z)V_BTtlugg5oF$%s@a%u#cIu+b0S3x!fVLxX4vx9h)~U>+x-hPnlAF#|FaINld3Q|D=g|LUHp;_o-+y~gN=-+-0 zLJ%X$KL8R~DE5^EXNhYRxChhhTYVV`q6N`I3ccssot%5=s~W3FMZ^$sVX#*zrlT|$w*{9f<7{zzV%*!a8a1?NLU@k%a^yN zU|tYdK?Ag-p-JlDQ5FG79EI1KwS3rx8W&!zVv8xlZR3YJ9B$V)Yf z5|0r=@{MW;r&bLHAI4l9BG=Q$b}WK7@4#=tR;{VAsIH=2sY+W-e#npjWz|Rl*GNNr zl`;G<#FfYk@%T&Acomu`+YwbPjer$=>otuq)RicY`=~3rK&eH(U5~(OK2UE73eJNR zo~18JLtM!ch*T=%78EW504EK`9MH$L0OFw2L}1fE&5*>hP$MRQPK<{VYFg4duP7!1 zsK-3^7@{o;NQNf}QlyXBX-V$%2qzQ{qh?UG;!~z4@Mn_JW%2ZHcYFz*2BdP8LD|p; z@JR(YKoV+~60|MCgf&$?`4YuwqTz?UUoZ`bZ)0RFL3KBYVEiQg(KsJX88QY#HVIC~ zLg^%n^g_f4DE^>KKx|z}5K>t(G69!5f$!~Z>?2mn@6tqFf4PE`*YLn3j=$l7#q9Z&iR|oU zb>$URui=4Vwe{^Bjgao1Xo|n#frI4aL%oo(k(tr)k;%m+(ERM$^x*Pl^V+w$?;Be& zUti-H_fMKukB)~=ui`sy3v2G`uR|Ul8(yS;65RvTdz7Btuyg>229?3+3Z3bQKlKNL z-Vm^I9bQnfMO|;(dedFv3t08Y_bF?}@3F_80K#3Q(Dz&7XDq%i5K!VzTYT!DZG4d1 zmoDH2Yej&@j6atY0!22s7qgX%midep9x<%Bi?Lfdvlvw?RT~}!y_!OAndh5&UmPe= z(xXch3Ppd$6Cv{!eKmCXa@%>ctYP5jrhE+F#>dv!+fDfX8RY?2BOnE#(@_MmF10!*aJ7^$|CgWZ9+cG<~fiPtu^fBJ; zU8<&yByW#u$Nk*gJ6!N@KfsGj1A2}=3=W{Jj| zX$RR-$)kI0Mr%i~n4W8+C5aq1-)Q^;meS1A%!?V061#&z1%_TpD|29EAURSn)WnF< zTRDhEc^S(i{c&inR$l=^K4swNh>~dOic%pVjpdMT`g^#_;xTEfw;y7)PHGFMN)4Ej z5mKqZ28zvc8$^SaVDq$>=M7U_|OgpUozGOx*`>KMLsgwJM|TLsiaXf5}6QAm~RS3OdMP z2^1wf8lg@REY49l^VaOX?*z+q+fGSc^o4u}7f*Uifa~+zZZofvgeM{`ay`+tEaHWB zG7Dz`WLb5IE>1p*1&TseW}_O$EcN{{Tt9A%XYB-7!gG+npLQxNc(F#2mR8-ithJSv zxAgRF7_}BG9p6?7b=kIjJ$@FTXe6Y){}CxSqJ94TmM5DJI~sN~x8YO(G4nlaIKZ|m zl&Z`9>(^%%f$Bj2ACMi75|WnVfSBjgub)V#SUJwC@!58-Nxe^wVy@)N%NvOL_hSuP znYA(j=wBU%aG1(222{mP1oJ*`MK)mw#h$p#%RQ|M4$#ZwT)wbS=x5zLn4^aarKuA! z-?;xqv+rPGN69o<;M)4?k}j#;`SS82vo_m-CtQ++;Y}Oy?hbRP@%hb+BHS*)tMI>@ zrO`;*4SV&mzL{HUL)K8*TM4ymKVy7D^fNy8o2D_`a%aqwn&BUG-}!U#V_yyhHw%Aj*skOENRtk& zEcnu*!`nvbH|sdK7D(v6B1UHP!`7_O@TjZlbAMi`Xyx|osaI+{odlLEM2x8N9W}Wr z{$(skwvgg|EVQ`5W!#tnHFXe$6hNRpKG~%Z5q~vL-mCsatp1W3TZn%Q9ZdwO2#$u~ z)2Ossc70M-ViL1C!#E|mAskh15vyw1n0j%%cCAYhCQxbuz333H>N5~iv25Io$2n~( zk%r53bWBXEj$Ok=a-Xk>e2Rs`UaV5}a71H>f34QhQilp7YHvuncQ$i3k_u9Km`mcx zVJml1aQ?gkVE|TyC$}%SrD8RwIVi&}#%H1QruwqM1wQR!zcM+@Hs z6s*`H$CO@Tv)jZJ$ArlxHuUz8AvkJsBfrtD`_6SL!l z9Yi%{R=1Q^36HNDyxwIBBAC?L`LtV{`DlA8= zpJ_6xX7cyXJQN)AnId;ij($)|Gvz9&l;W~j;m;SXMmi;ClSzO7=#97ND~}Va>Wd{T zL$7EgjLW}XX~~ONw!p^eqK(c{r7*p($qh`-!Xv4&^iE%hM*m*$PN|AS%IZr3>lCIGZ!@^FPK@o_ zmRp})X1rJqR5KpG7QHr!k~7C{-2zMX^6zUbZLJ~OYZHEIZngd_HWU`^b6!GjExw&6 zW@K$JU|Q}bc-vE3g~zQ;Qb@}(v7K6g*0eagd)a)|nM%#tA~_jX2g_?W=v_2Ve+R_N zp13q^(?NpxJRs?zJE!=6L*c6NoN}MK%DD1LgP^v}c(VgP-$9`oAbI77kUD3r6CZqv zX~Xnky-CS=IL=b@WK!I8N{QLtmma2>jhMQw59&M-{p3BP8||u)@oUKxp)W1w_Ya

)tVttp`^7_wBmx@aUR{6nxL3BQiH?(D^G@ zN$kT@%9iZI-k_!q6v_{0Eki4GY&_}NfDY2uBZlwm)8vkWbJ{K(_j#Kcs@ymAwZ0E$ z-&K=rA$9x) z);f;=DwA|g$O!D_B8j`-KHJM-8!stRZcW$otLxmede*+mo&P7ifbj+YtbJ32Qh3H? zOBkd8MY;g>I}^cu{j#?0b(VJYe+&@(*`kZ3D02J6~*fFJY)R z)EYq9c;5p&p8?5tE6bKp2QL;G>i*P{zLfVtHip1EH^=b-U`dG|ZK`j^5HQP)a7CTA z{my(o%wJd|g#0IvM+BI62h;(DsNIJc;D;LVg_>xDnt6m;q=s5SLTy$;?e0Sz@WY(= z!dx`|6&~mX3G-R`6CN0V|2H{M0~G230;Ph&Uy}n@{w4?FgX8(Yi5lQ!4{&NKI2{7c zTmfg_gLCo2^ZCLHHNuNM!b?-b%OT;FE8*4m;kEb?^?d)E@W7Q&h6DIF76Ho+kz=I+ z@9s&5Adzx>kyHSn7*dFTGf)xcx9+03OWzd?N1Z@ysP1g{OCxmXgB9t0YN#j<%7o!?NgFNTzjo4QAYnRl?dZL4`AXxZCPDjM^R-Q5WCgNMDIly_;f zW3stjDT2Q$LyGzXNQ}?@@iwVv$je|g`aPd_VAzX8kh=wBIVsUCQ70`?njkL4AO*k0 zDon!Diy>)3(w~UmC-4Y_Q0m0&ZidI_tjC+6AQ@lDAAkXICIq;NySqryxfw49%9q49 zNX4@}IJ5BwRvHAnZw)jw&oFMu;2I5V0%yvp2hhI(2k|F6NhF6IC4tL;OPa}FG%XOr zl4Hw~;|a_YkGx@)v(iJ;l5^AI!#uNEeUc6wm0MFX&HZ2KQys&C;J}V63~8PzSxFDc z4iD~IM>aFJsVM6Hd#&D>lDP<@X1Db26e%k{uJDFAOUht&O;ihR!%N|`h#Wq@33?_ z&rHsP^e(A*RlH2$lEBaAE`So3iB*@n)k695BFxnE0S4qD2IrZwOe6JTo}zZ;8f=FR5mRK4n_}T$pRa2cC~u!ahm0c=9@E3JE5_>t5&ReYspELe*V&q0i%hz z4teMN7DS9W#9^h!xcP#IF_BhfU4&)&w-tSiuif-p8R5!4ud*j8q!fun#&BC-uL3{9 zayKh0POoy*mSpsja-XqsX{q$uwwD5h)q;7m(v~$JQNqG@tEAL1B(aO?i^pn>0QjF; zHMd?hKLu*;AFELeYaZKb?$hC5GT=1G;iY93~{77RDL{Hlh}_-Gp%6gl*hJ=GKUfLguLAl z@k8zTqwXBG9$lg~ql#YFcm4w$vExOm%p_imYt3_U6;&Gjba8#PB=R}cMo%g>T2tMX zJ^gBK<{NQ-_99~83~^BzJ)S}BtwbIJYWd!zM#)9J6B8L96-EoHrirG;`aNB?FMMt) zyoK@lD~nBm>-tRa{2O~pi48B}jWo*NZyGGxwf9{$?WyFCk{a*(^b}MeSs|<*6<@^P z30`iuXpA>&WynA7HkBlNX$WC3EdprYw1M8WgW%NaQD7RG)XbCAy02w=7SuXX)C1u) z0vDyn8`VZC)C&gGN3~^NGhc_U+j?VV#|C7>iZqrg)Ux5;2r==PR>w?fl&{63#si2iO|t$*AI)}nHlVazEMh3Qk2ZoM%tr5Hf>B? zS4>4=MxyBsw?Mr)uTS4_2xY(AOPj$!Ldgrb5z@&9g*_Yzb4o+-ui5M6&Xs@pr zNNa2#DD3Rc=qTB39~?j)aOucRlvX>D%G`yPl2tS1?yRI{0^`U52bpO)hJZgIn?$Yf zI+`}#jTi4&^Mw0PtovQwb*~IC6|VDZMtoR;Z&nk>xA75&7F3R9>PmF&>2lR1Q>9Yp z9__*G?Mxc3fH9zoq+(a?mWG2=4tA5_z))2VMOpX3bTBdaC|^`C6%nI)Oz`mJBW0cp z-!}H;MSNZ~7-(P`>NpvY{naLKBc-U*5mbcLYx1UxS*`zF zE@t;JQ@^UlxZu|pmBJI8jflQ2B2Dv%$;S7gnrUT`^C8%$vgY zoqi`b8RI&;+%ZC(HHAeoxAk@MDY6}&WrBrd!qjypg=@xSWiFX|zG;1mylU3BV#L03 z*2H3(CT`|~?%dw?Jb~>3k?))!3HoQ>G4aj?{NGdb8$$;43xvd>Op|lklZ#WoQ245* zyJhE%E#`Gi2Na@4pll}1))zpIB(hZtKXT_4b(gON7g2NjEOEJL%_DCz+t^#L17>YsK2D`ANc$=po4-u@h=yU1C8QH zi$f!894lMf<)aH5d+X!N`@8#t?BBkv{n$IdzKN}BZhQ=WcnU9n_IlVcA0=hg`;CCi zIr8eS8G=MX3xmUMDDI1<*@=x*r#=)@KjdhvE=MY5nZ6K3Z5dB8VnL&xZSzpSPZvjL zydO#DY(DPvNfoWJ>3{7|x?2=hazP(mWSRxqC)}xEpp9QKMst*0Ly8}s|+FXw#3R*Xp)B$+bjooxZ z#fX$*p`w4C_P_F79n2(sh>`D{xjvqYF}APv{orgdUE?jun(cl4EobixL0;?Q(OSc* z0z9tH>FM$zNt?ZX`X|k`A89zbG#4M6)+At&HMZ0r;NRxhJ%54ERfZP=CEFv(@2z>E z0hM*!cR~sU~K=W)cnbNS4J&B&Nq2048zySF24IjssKop;vg96t zq}43LY00t#+h((jF5Bho$Sm8h?~O$+P(E)wD+pWQAWPGv?m$M9t8t((K}iJuAIp*? zy|Guz5>`XY(!%g#%d*ty@bcpF>M=%~`kvasSIZLV3S2E4=%9+k=C8It%L*>kv+b|pzwOSwJ^tCO<%v-R_mb*J%~o)?5jmdd)?4`=P#vDU2p1K-z~x{(vsFJ3zF zGAb&om#JlH`mdL~G})7rC*bjDN-8hMn6qJuZACgyah*?GjzYE> zuV=)(zroJ}=vW$OwGuPA=A|WdUD&0?Sh%Zu?w@Lww4HS?de(q)7fTzozgYUw*(y2K z@OcGKI;^!PZu{|DY&p7m+ML6Ouh*V_ZrSoywJ_?r+d4FcC8R?df3u>q_f^nm{_MXk zN8{bF?*7OR`SgqYb1TJk=S9cakgr`=`~^w%RzOQN|L`{C*keUE!>VefF#)ZWF+S`o zJ5@3w#?Orm*q^YNInI7Qkn!#hK1U9cY@EkYrFv|KRj^ZSLksJ>?JwKD{u1+_^80m;HyJX+P?7%ou>}4Z>r@sh7x&DhrANxs z$VUGvxymP{IZ~I^LmMi4E%ktZ#x+`R^8MykbW7)KTCHKP*G2~qQvJzqTkq9=JQ%Xn zc7-?n4(&4H=QC>jD9F=x=_t!oW3C;TgpR@JPvlnv;=S+ot$^gaUw`S8emwc|Q?H+J z71cP9p-mAP|3jrVMwmJ;xlo-))>Q^}l>0zTTr9}0(5QXNgMX^d(JxE80L9Ql9H-AN zB7ke;33cU@==X3x*o7^MtCkUQPBec}<=x-;`ynqF9{kbh4e>Q z5@SgyLHQZRBuOcPJQ9vz$3}G|@k3EnQAcoAzcE(Lil|<*VrT&>Esr$6`_nB51e zZ2|f|o;8l-*0~+#7qfAL&N*jMgB`Al!|_E!jMP3$ee!TTM+F?huX466$mri_^aNr@#l{bqP@{Nt2jkAg2QBO#@G%Gj zIj;ktCg}k)uBdU7j3Bpm?RC~-dHXr+WDeN_xFA<1)95MJr;{CCb8fzyv22ny(kFDFY&}OpR82mshdj3 ze$dI~NiWsgiK0BT9?ndREj72@R5?Ud=p0Bdx2@e&drVg7-Hk1GJ>Jv=pj8?m%B=Je z-qwQTDvj{QSB3;`>tdrSO~@juN41h!>&K3$`LrsQ`ME8)7QdU4XjfK-s#U0>Rav%J zsaA)}hUwc{s!F!6lx9ujBZeG43wnlc7$r8lv9RDMYA-H?SGZt|9|(F&t4#ZqYuNuV z!!U>cnjy^{p@L)SxPu_4p5N%YNX>|>8_^Qol~^W|WrtAFE?#z-=rZ?>8 z8b7l;|AeP_-}y9w&4!hI30EUsHQK9Vc2%gdYaBUAV1QQfnn__j?W!+xa^tYUZ6XkV2( zu|DQmXL0sfH2NsN`BsIZtqmXDXMSbV{J7~}E41rc%EugMSaXPj^yH@Qy!mZT+E{G! z&-hs?D{do$(|nfGb!Z*;x-kaHoedlK`I8j4hTL{t16ahdjLBUmZN`~1Mg@EkSfaZ4*xz~zB$A6ndWO&e7` zV!h*^oFuu1NMhot__r?Fre3S4c81End??L+zJ6)t^*-i_@oE*haqAxMyzp-Eo~B`Q z?V!uPj5zq9({$#V?fPtPa{yTpQ*fKh=-jdR)VVC9cRTdrvlrX#)eV2{F?dyOGt=d> zVXDBmOhkGZQ}XfLp6%D#*2lXbw&x#V)AwC$WcPtNANMIdJjUtt&nK1zb;5dnQy4&_ zY+!WX4=ebc`FH<1ROo)_t$Du9>HfXr|LNDx^z%ak8UArf_wV)^|I53VXD3%GS!@a= zC+g-|YEwrV6<~nrcN9mO07O85@f_9VH!A!*GBo@^BO_WD_&^*G_1k+I9ZKq2VT=wA zS|T8l9}X3}6D8FgaQ7#Wnv-f)B#1MgL~{;!K_ZeD(_ zotc?ITEVMs@s9@!u&AW0uDRtk5-O*mwEYhW59w@r^??{2iy9oCnx5&OoPBk5te9VZ z4bg~MZ8NvBO1EOgCL7u~Uf6;dVB_-d#4{kkz;M(2cseQWYBTUah0tNl5#ZsTeVZSy z`Bv{%x3H@Sa3C#khsC*s+Nq_gx3{9+EfuLZi6R!HeicSJnTBJt8IK{Qq^P%dKN}-!(aK|`i4t13X?ZAZL^{D#?8ZlrqbiZ{pcT#X~oDRN^ycrdTkm? zgovs_^l4LVJ1&RX%?odo9#dLgP_YpR^AL#a2n;P?U^DF83`el`>wnO;ogbH-7G14< z;CLCF>l;7qD*sF%K?sxZWvEmyz@dmrhdA5<^~2^IX-!LCW_ESM*e|Opo`kT9sjkb6 zVhVC^Ed(gW;aoT>ol0$*H~KUM9L-5{-sgk&2!~fx6e-YfeHBAsBpfvDBUiDIZ|(UD zYie@b#U$3pNF~|zv#gDwi5c_*268YrLkjaAfCb_hY#IuRmN=_-#;^|T`E2x=cdRNp z4L=gpKf&-4Sa)Fcq3DEdv*M{yk6rE*UcJ937ja)hcw=mWc{?W^CM=3uv|K%P1k~sa zEwKhs0LHTDG9ae|>OoJ5_LUS|6xAi+E!0pkI=`-KGdF#Z+TY3`9fEnPrH*s+zc9bt z>?VA_rBtqb9cb)u+MgXJ24h{;&ZXFQ(|+;F<9eeR^9htkZb3*1v-in=l1=9u)An!N zR1kexr8LJ#-`>cK5a5@Lg$4ZzIW0)oP&GD?oH;lO7qOQD3d#pA0VVkH?`*hsE11tW5>21OlLD$>QxOc3~9qsNSI0Dei* zA4Dlq0oN<02ybJON0^%1mfFXd9!GG%1cy{lFD&A40luy<$PHE~o*VQ8Ra0q-oiU4W5!FRrg+3k*#B3 zJPgfR^22!ekxXh5(Ez>y6b!xr*rUV6jECiR!b~e7ZZ>ms+=8d!1~Qk)?`=Ba;yOj` zTar9L9M1ccJYjj`B)d^a$Nm$4Dtf{>$9vqI9r|&aQ*}BHo#BudZ*4RbTO}u-Us*8B z>AjOYdKNB;xkNV)wOjF9d`Nq!DD2p%ZMqZ|YE}ssrZf)-cfsb~XO>E-eA;B94>gr9 zNZx&HKKt>yL>R4HF^o_JDpDnli-ew!NL;CqwU>aGJ4B^Kda+#Vrd-2(vRtjjE7VoX z#r!llhEXeRN`uh8!a}Y>YdKw&m$)e#Un~}!jx1;WIjQQ;MODr*v}LJ@&oRn^Ieu)~ v7W3xsMK#PyB?Z?F%i(tgi$%JO=~vWuiP{KWj6`T@lyBoQT_wn&pg#N$h3a7L diff --git a/docs/fa/index.md b/docs/fa/index.md deleted file mode 100644 index 2637d22638c..00000000000 --- a/docs/fa/index.md +++ /dev/null @@ -1,98 +0,0 @@ -

- -# ClickHouse چیست؟ {#clickhouse-chyst} - -ClickHouse یک مدیریت دیتابیس (DBMS) ستون گرا برای پردازش تحلیلی آنلاین (OLAP) می باشد. - -در یک مدیریت دیتابیس ردیف گرا، داده ها به فرم زیر ذخیره سازی می شوند: - -| Row | WatchID | JavaEnable | Title | GoodEvent | EventTime | -|-----|---------------------|------------|--------------------|-----------|---------------------| -| #0 | 5385521489354350662 | 1 | Investor Relations | 1 | 2016-05-18 05:19:20 | -| #1 | 5385521490329509958 | 0 | Contact us | 1 | 2016-05-18 08:10:20 | -| #2 | 5385521489953706054 | 1 | Mission | 1 | 2016-05-18 07:38:00 | -| #N | … | … | … | … | … | - -به این صورت، تمام مقادیر مربوط به یک سطر (رکورد) به صورت فیزیکی و در کنار یکدگیر ذخیره سازی می شوند. - -دیتابیس های MySQL, Postgres و MS SQL Server از انواع دیتابیس های ردیف گرا می باشند. -{: .grey } - -در یک دیتابیس ستون گرا، داده ها به شکل زیر ذخیره سازی می شوند: - -| Row: | #0 | #1 | #2 | #N | -|-------------|---------------------|---------------------|---------------------|-----| -| WatchID: | 5385521489354350662 | 5385521490329509958 | 5385521489953706054 | … | -| JavaEnable: | 1 | 0 | 1 | … | -| Title: | Investor Relations | Contact us | Mission | … | -| GoodEvent: | 1 | 1 | 1 | … | -| EventTime: | 2016-05-18 05:19:20 | 2016-05-18 08:10:20 | 2016-05-18 07:38:00 | … | - -این مثال ها تنها نشان می دهند که داده ها منظم شده اند. -مقادیر ستون های مختلف به صورت جدا، و داده های مربوط به یک ستون در کنار یکدیگر ذخیره می شوند. - -مثال های از دیتابیس های ستون گرا: Vertica, Paraccel (Actian Matrix, Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise, Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid, kdb+. -{: .grey } - -ترتیب های مختلف برای ذخیره سازی داده ها، مناسب سناریو های مختلف هستند. سناریو دسترسی به داده اشاره دارد به، چه query هایی ساخته شده اند، چند وقت به چند وقت، در چه مقداری، چقدر داده در هنگام اجرای هر query خوانده می شود، چند رکورد، چند ستون و چند بایت؛ رابطه ی بین خوانده و نوشتن داده؛ سایز دیتاسی فعال مورد استفاده و نحوه ی استفاده آن به صورت محلی؛ آیا از تراکنش استفاده می شود؛ چگونه داده ها جدا می شوند؛ نیازمندی ها برای replication داده ها و یکپارچگی منطقی داده ها؛ نیازمندی ها برای latency و throughput برای هر نوع از query، و… - -مهمتر از بالا بودن لود سیستم، سفارشی کردن سیستم مطابق با نیازمندی های سناریو می باشد، و این سفارشی سازی در ادامه دقیق تر می شود. هیج سیستمی وجود ندارد که مناسب انجام سناریو های متفاوت(بسیار متفاوت) باشد. اگر یک سیستم برای اجرای سناریو های مختلف آداپته شده باشد، در زمان بالا بودن لود، سیستم تمام سناریوها را به صورت ضعیف handle می کند. - -## ویژگی های کلیدی یک سناریو OLAP {#wyjgy-hy-khlydy-ykh-snryw-olap} - -- اکثریت درخواست های برای خواندن می باشد. -- داده ها به صورت batch های بزرگ (\< 1000 رکورد) وارد می شوند، نه به صورت تکی؛ یا اینکه اصلا بروز نمی شوند. -- داده ها به دیتابیس اضافه می شوند و تغییر پیدا نمی کنند. -- برای خواندن، تعداد زیادی از رکورد ها از دیتابیس استخراج می شوند، اما فقط چند ستون از رکورد ها. -- جداول «wide» هستند، به این معنی تعداد زیادی ستون دارند. -- query ها نسبتا کم هستند (معمولا صدها query در ثانیه به ازای هر سرور یا کمتر) -- برای query های ساده، زمان تاخیر 50 میلی ثانیه مجاز باشد. -- مقادیر ستون ها کوچک باشد: اعداد و رشته های کوتاه (برای مثال 60 بایت به ازای هر url) -- نیازمند throughput بالا در هنگام اجرای یک query (بالای یک میلیارد رکورد در هر ثانیه به ازای هر سرور) -- تراکنش واجب نیست. -- نیازمندی کم برای consistency بودن داده ها. -- فقط یک جدول بزرگ به ازای هر query وجود دارد. تمام جداول کوچک هستند، به جز یکی. -- نتیجه query به طول قابل توجهی کوچکتر از source داده ها می باشد. به عبارتی دیگر در یک query، داده ها فیلتر یا تجمیع می شوند، پس نتایج در RAM یک سرور فیت می شوند. - -خوب خیلی ساده می توان دید که سناریو های OLAP خیلی متفاوت تر از دیگر سناریو های محبوب هستند (مثل OLTP یا Key-Value). پس اگر میخواهید performance مناسب داشته باشید، استفاده از دیتابیس های OLTP یا Key-Value برای اجرای query های OLAP معنی ندارد. برای مثال، اگر شما از دیتابیس MongoDB یا Redis برای آنالیز استفاده کنید، قطعا performance بسیار ضعیف تری نسبت به دیتابیس های OLAP خواهید داشت. - -## دلایل برتری دیتابیس های ستون گرا برای سناریو های OLAP {#dlyl-brtry-dytbys-hy-stwn-gr-bry-snryw-hy-olap} - -دیتابیس های ستون گرا مناسب سناریو های OLAP هستند -(حداقل 100 برابر در بیشتر query ها سرعت پردازش آنها بهتر است). دلایل این برتری در پایین شرح داده شده است، اما آسانترش این هست که به صورت visually این تفاوت را ببینیم: - -**ردیف گرا** - -![Row oriented](images/row-oriented.gif#) - -**ستون گرا** - -![Column oriented](images/column-oriented.gif#) - -تفاوت را دیدید؟ بیشتر بخوانید تا یاد بگیرید چرا این اتفاق رخ میدهد. - -### Input/output {#inputoutput} - -1. برای query های تحلیلی، تنها چند ستون از تمام ستون های جدول نیاز به خواندن دارد. در یک دیتابیس ستون گرا، شما فقط داده ی مورد نیاز را می خوانید. برای مثال، اگر شما نیاز به 5 ستون از 100 ستون را دارید، شما می توانید انتظار 20 برابر کاهش I/O را داشته باشید. -2. از آنجایی که داده در بسته ها خوانده می شوند، فشرده سازی ساده می باشد. همچنین داده های ستون ها برای فشرده سازی ساده می باشند. این باعث کاهش نرخ I/O در ادامه می شود. -3. با توجه به کاهش I/O، داده های بیشتری در system cache قرار می گیرند. - -برای مثال، query «تعداد رکوردها به ازای هر بستر نیازمندی» نیازمند خواندن ستون «آیدی بستر آگهی»، که 1 بایت بدون فشرده طول می کشد، خواهد بود. اگر بیشتر ترافیک مربوط به بستر های نیازمندی نبود، شما می توانید انتظار حداقل 10 برابر فشرده سازی این ستون را داشته باشید. زمانی که از الگوریتم فشرده سازی quick استفاده می کنید، عملیات decompression داده ها با سرعت حداقل چندین گیگابایت در ثانیه انجام می شود. به عبارت دیگر، این query توانایی پردازش تقریبا چندین میلیارد رکورد در ثانیه به ازای یک سرور را دارد. این سرعت در عمل واقعی و دست یافتنی است. - -### CPU {#cpu} - -از آنجایی که اجرای یک query نیازمند پردازش تعداد زیادی سطر می باشد، این کمک می کند تا تمام عملیات ها به جای ارسال به سطرهای جداگانه، برای کل بردار ارسال شود، یا برای ترکیب query engine به طوری که هیچ هزینه ی ارسالی وجود ندارد. اگر این کار رو نکنید، با هر half-decent disk subsystem، تفسیرگر query ناگزیر است که CPU را متوقف کند. این منطقی است که که در صورت امکان هر دو کار ذخیره سازی داده در ستون ها و پردازش ستون ها با هم انجام شود. - -دو راه برای انجام این کار وجود دارد: - -1. یک موتور بردار. تمام عملیات ها به جای مقادیر جداگانه، برای بردارها نوشته شوند. این به این معنیست که شما خیلی از مواقع نیازی به صدا کردن عملیات ها ندارید، و هزینه انتقال ناچیز است. کد عملیاتی شامل یک چرخه داخلی بهینه شده است. - -2. Code generation. کد تولید شده برای query دارای تمام تماس های غیرمستقیم در آن است. - -این در یک دیتابیس نرمال انجام نمی شود، چرا که برای اجرای query های ساده این کارها منطقی نیست. هرچند، استثناهاتی هم وجود دارد. برای مثال، MemSQL از code generation برای کاهش latency در هنگام پردازش query های SQL استفاده می کند. (برای مقایسه، مدیریت دیتابیس های آنالیزی نیازمند بهینه سازی توان عملیاتی (throughput) هستند نه latency.) - -توجه کنید که برای کارایی CPU، query language باید SQL یا MDX باشد، یا حداقل یک بردارد (J, K) باشد. query برای بهینه سازی باید فقط دارای حلقه های implicit باشد. - -
- -[مقاله اصلی](https://clickhouse.tech/docs/fa/) diff --git a/docs/fa/interfaces/cli.md b/docs/fa/interfaces/cli.md deleted file mode 100644 index f1b5dfa6d91..00000000000 --- a/docs/fa/interfaces/cli.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 17 -toc_title: "\u0645\u0634\u062A\u0631\u06CC \u062E\u0637 \u0641\u0631\u0645\u0627\u0646" ---- - -# مشتری خط فرمان {#command-line-client} - -تاتر یک مشتری خط فرمان بومی فراهم می کند: `clickhouse-client`. مشتری پشتیبانی از گزینه های خط فرمان و فایل های پیکربندی. برای کسب اطلاعات بیشتر, دیدن [پیکربندی](#interfaces_cli_configuration). - -[نصب](../getting-started/index.md) این از `clickhouse-client` بسته بندی و اجرا با فرمان `clickhouse-client`. - -``` bash -$ clickhouse-client -ClickHouse client version 19.17.1.1579 (official build). -Connecting to localhost:9000 as user default. -Connected to ClickHouse server version 19.17.1 revision 54428. - -:) -``` - -مشتری و سرور نسخه های مختلف سازگار با یکدیگر هستند, اما برخی از ویژگی های ممکن است در مشتریان قدیمی تر در دسترس باشد. ما توصیه می کنیم با استفاده از همان نسخه از مشتری به عنوان برنامه سرور. هنگامی که شما سعی می کنید به استفاده از یک مشتری از نسخه های قدیمی تر و سپس سرور, `clickhouse-client` پیام را نمایش میدهد: - - ClickHouse client version is older than ClickHouse server. It may lack support for new features. - -## استفاده {#cli_usage} - -مشتری را می توان در حالت تعاملی و غیر تعاملی (دسته ای) استفاده کرد. برای استفاده از حالت دسته ای ‘query’ پارامتر یا ارسال داده به ‘stdin’ (تایید می کند که ‘stdin’ یک ترمینال نیست), یا هر دو. هنگام استفاده از رابط اچ تی پی مشابه است ‘query’ پارامتر و ارسال داده به ‘stdin’, درخواست یک الحاق از است ‘query’ پارامتر, خوراک خط, و داده ها در ‘stdin’. این مناسب برای نمایش داده شد درج بزرگ است. - -نمونه ای از استفاده از مشتری برای وارد کردن داده ها: - -``` bash -$ echo -ne "1, 'some text', '2016-08-14 00:00:00'\n2, 'some more text', '2016-08-14 00:00:01'" | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; - -$ cat <<_EOF | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; -3, 'some text', '2016-08-14 00:00:00' -4, 'some more text', '2016-08-14 00:00:01' -_EOF - -$ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; -``` - -در حالت دسته ای, فرمت داده ها به طور پیش فرض جدول است. شما می توانید فرمت را در بند فرمت پرس و جو تنظیم کنید. - -به طور پیش فرض, شما فقط می توانید پردازش یک پرس و جو تنها در حالت دسته ای. برای ایجاد چندین نمایش داده شد از یک “script,” استفاده از `--multiquery` پارامتر. این برای همه نمایش داده شد به جز درج کار می کند. نتایج پرس و جو خروجی متوالی بدون جداکننده های اضافی می باشد. به طور مشابه, برای پردازش تعداد زیادی از نمایش داده شد, شما می توانید اجرا ‘clickhouse-client’ برای هر پرس و جو. توجه داشته باشید که ممکن است دهها میلی ثانیه برای راه اندازی ‘clickhouse-client’ برنامه - -در حالت تعاملی شما یک خط فرمان دریافت می کنید که می توانید نمایش داده شده را وارد کنید. - -اگر ‘multiline’ مشخص نشده است (به طور پیش فرض): برای اجرای پرس و جو را فشار دهید را وارد کنید. نقطه و ویرگول در پایان پرس و جو لازم نیست. برای ورود به پرس و جو چند خطی یک بک اسلش را وارد کنید `\` قبل از خط تغذیه. بعد از اینکه شما فشار وارد, از شما خواسته خواهد شد که برای ورود به خط بعدی از پرس و جو. - -اگر چند خطی مشخص شده است: برای اجرای یک پرس و جو, پایان با یک نقطه و ویرگول و مطبوعات را وارد کنید. اگر نقطه و ویرگول در پایان خط وارد شده حذف شد, از شما خواسته خواهد شد برای ورود به خط بعدی از پرس و جو. - -فقط یک پرس و جو تنها اجرا می شود, بنابراین همه چیز پس از نقطه و ویرگول نادیده گرفته شده است. - -شما می توانید مشخص کنید `\G` بجای یا بعد از نقطه و ویرگول. این نشان می دهد فرمت عمودی. در این قالب, هر مقدار بر روی یک خط جداگانه چاپ, مناسب است که برای جداول گسترده ای. این ویژگی غیر معمول برای سازگاری با خروجی زیر کلی اضافه شد. - -خط فرمان بر اساس ‘replxx’ (شبیه به ‘readline’). به عبارت دیگر از میانبرهای صفحهکلید اشنایی استفاده میکند و تاریخ را حفظ میکند. تاریخ به نوشته شده است `~/.clickhouse-client-history`. - -به طور پیش فرض, فرمت استفاده می شود قبل از شکست است. شما می توانید فرمت را در بند فرمت پرس و جو یا با مشخص کردن تغییر دهید `\G` در پایان پرس و جو, با استفاده از `--format` یا `--vertical` استدلال در خط فرمان, و یا با استفاده از فایل پیکربندی مشتری. - -برای خروج از مشتری, کنترل مطبوعات+د (یا کنترل+ج), و یا یکی از موارد زیر را وارد کنید به جای یک پرس و جو: “exit”, “quit”, “logout”, “exit;”, “quit;”, “logout;”, “q”, “Q”, “:q” - -هنگامی که پردازش یک پرس و جو مشتری نشان می دهد: - -1. پیشرفت, که به روز شده است بیش از 10 بار در ثانیه (به طور پیش فرض). برای نمایش داده شد سریع پیشرفت ممکن است زمان نمایش داده می شود. -2. پرس و جو فرمت شده پس از تجزیه, برای اشکال زدایی. -3. نتیجه در قالب مشخص شده است. -4. تعداد خطوط در نتیجه زمان گذشت و سرعت متوسط پردازش پرس و جو. - -شما می توانید یک پرس و جو طولانی با فشار دادن کنترل لغو+ج.با این حال, شما هنوز هم نیاز به کمی صبر کنید برای سرور به سقط درخواست. این ممکن است به لغو پرس و جو در مراحل خاص. اگر شما منتظر نیست و مطبوعات کنترل+ج بار دوم مشتری خروج خواهد شد. - -مشتری خط فرمان اجازه می دهد تا عبور داده های خارجی (جداول موقت خارجی) برای پرس و جو. برای کسب اطلاعات بیشتر به بخش مراجعه کنید “External data for query processing”. - -### نمایش داده شد با پارامترهای {#cli-queries-with-parameters} - -شما می توانید پرس و جو را با پارامترها ایجاد کنید و مقادیر را از برنامه مشتری منتقل کنید. این اجازه می دهد تا برای جلوگیری از قالب بندی پرس و جو با ارزش های پویا خاص در سمت سرویس گیرنده. به عنوان مثال: - -``` bash -$ clickhouse-client --param_parName="[1, 2]" -q "SELECT * FROM table WHERE a = {parName:Array(UInt16)}" -``` - -#### نحو پرس و جو {#cli-queries-with-parameters-syntax} - -یک پرس و جو را به طور معمول فرمت کنید و سپس مقادیر را که می خواهید از پارامترهای برنامه به پرس و جو در پرانتز در قالب زیر منتقل کنید قرار دهید: - -``` sql -{:} -``` - -- `name` — Placeholder identifier. In the console client it should be used in app parameters as `--param_ = value`. -- `data type` — [نوع داده](../sql-reference/data-types/index.md) از مقدار پارامتر برنامه. برای مثال یک ساختار داده مانند `(integer, ('string', integer))` می تواند داشته باشد `Tuple(UInt8, Tuple(String, UInt8))` نوع داده (شما همچنین می توانید از یکی دیگر استفاده کنید [عدد صحیح](../sql-reference/data-types/int-uint.md) انواع). - -#### مثال {#example} - -``` bash -$ clickhouse-client --param_tuple_in_tuple="(10, ('dt', 10))" -q "SELECT * FROM table WHERE val = {tuple_in_tuple:Tuple(UInt8, Tuple(String, UInt8))}" -``` - -## پیکربندی {#interfaces_cli_configuration} - -شما می توانید پارامترها را به `clickhouse-client` (همه پارامترها یک مقدار پیش فرض) با استفاده از: - -- از خط فرمان - - گزینه های خط فرمان نادیده گرفتن مقادیر پیش فرض و تنظیمات در فایل های پیکربندی. - -- فایل های پیکربندی. - - تنظیمات در فایل های پیکربندی نادیده گرفتن مقادیر پیش فرض. - -### گزینههای خط فرمان {#command-line-options} - -- `--host, -h` -– The server name, ‘localhost’ به طور پیش فرض. شما می توانید از نام یا نشانی اینترنتی4 یا ایپو6 استفاده کنید. -- `--port` – The port to connect to. Default value: 9000. Note that the HTTP interface and the native interface use different ports. -- `--user, -u` – The username. Default value: default. -- `--password` – The password. Default value: empty string. -- `--query, -q` – The query to process when using non-interactive mode. -- `--database, -d` – Select the current default database. Default value: the current database from the server settings (‘default’ به طور پیش فرض). -- `--multiline, -m` – If specified, allow multiline queries (do not send the query on Enter). -- `--multiquery, -n` – If specified, allow processing multiple queries separated by semicolons. -- `--format, -f` – Use the specified default format to output the result. -- `--vertical, -E` – If specified, use the Vertical format by default to output the result. This is the same as ‘–format=Vertical’. در این قالب, هر مقدار بر روی یک خط جداگانه چاپ, مفید است که در هنگام نمایش جداول گسترده. -- `--time, -t` – If specified, print the query execution time to ‘stderr’ در حالت غیر تعاملی. -- `--stacktrace` – If specified, also print the stack trace if an exception occurs. -- `--config-file` – The name of the configuration file. -- `--secure` – If specified, will connect to server over secure connection. -- `--param_` — Value for a [پرسوجو با پارامترها](#cli-queries-with-parameters). - -### پروندههای پیکربندی {#configuration_files} - -`clickhouse-client` با استفاده از اولین فایل موجود در زیر: - -- تعریف شده در `--config-file` پارامتر. -- `./clickhouse-client.xml` -- `~/.clickhouse-client/config.xml` -- `/etc/clickhouse-client/config.xml` - -نمونه ای از یک فایل پیکربندی: - -``` xml - - username - password - False - -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/interfaces/cli/) diff --git a/docs/fa/interfaces/cpp.md b/docs/fa/interfaces/cpp.md deleted file mode 100644 index db04e238caf..00000000000 --- a/docs/fa/interfaces/cpp.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 24 -toc_title: "\u062C++ \u06A9\u062A\u0627\u0628\u062E\u0627\u0646\u0647 \u0645\u0634\ - \u062A\u0631\u06CC" ---- - -# ج++ کتابخانه مشتری {#c-client-library} - -دیدن README در [صفحه اصلی](https://github.com/ClickHouse/clickhouse-cpp) مخزن. - -[مقاله اصلی](https://clickhouse.tech/docs/en/interfaces/cpp/) diff --git a/docs/fa/interfaces/formats.md b/docs/fa/interfaces/formats.md deleted file mode 100644 index b21328fe07f..00000000000 --- a/docs/fa/interfaces/formats.md +++ /dev/null @@ -1,1213 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 21 -toc_title: "\u0641\u0631\u0645\u062A \u0647\u0627\u06CC \u0648\u0631\u0648\u062F\u06CC\ - \ \u0648 \u062E\u0631\u0648\u062C\u06CC" ---- - -# فرمت برای داده های ورودی و خروجی {#formats} - -تاتر می توانید قبول و بازگشت داده ها در فرمت های مختلف. یک قالب پشتیبانی شده برای ورودی می تواند برای تجزیه داده های موجود در `INSERT`برای انجام `SELECT`بازدید کنندگان از یک جدول فایل حمایت مانند فایل, نشانی اینترنتی و یا اچ دی, و یا به خواندن یک فرهنگ لغت خارجی. فرمت پشتیبانی شده برای خروجی می تواند مورد استفاده قرار گیرد به ترتیب -نتایج یک `SELECT` و برای انجام `INSERT`ها را به یک جدول فایل حمایت. - -فرمت های پشتیبانی شده عبارتند از: - -| قالب | ورودی | خروجی | -|---------------------------------------------------------|-------|-------| -| [جدول دار](#tabseparated) | ✔ | ✔ | -| [اطلاعات دقیق](#tabseparatedraw) | ✗ | ✔ | -| [اطلاعات دقیق](#tabseparatedwithnames) | ✔ | ✔ | -| [اطلاعات دقیق](#tabseparatedwithnamesandtypes) | ✔ | ✔ | -| [قالب](#format-template) | ✔ | ✔ | -| [پاسخ تمپلیتینی](#templateignorespaces) | ✔ | ✗ | -| [CSV](#csv) | ✔ | ✔ | -| [اطلاعات دقیق](#csvwithnames) | ✔ | ✔ | -| [سفارشی](#format-customseparated) | ✔ | ✔ | -| [مقادیر](#data-format-values) | ✔ | ✔ | -| [عمودی](#vertical) | ✗ | ✔ | -| [عمودی](#verticalraw) | ✗ | ✔ | -| [JSON](#json) | ✗ | ✔ | -| [فوق العاده](#jsoncompact) | ✗ | ✔ | -| [جیسانچرو](#jsoneachrow) | ✔ | ✔ | -| [TSKV](#tskv) | ✔ | ✔ | -| [زیبا](#pretty) | ✗ | ✔ | -| [پیش تیمار](#prettycompact) | ✗ | ✔ | -| [بلوک پیش ساخته](#prettycompactmonoblock) | ✗ | ✔ | -| [کتاب های پیش بینی شده](#prettynoescapes) | ✗ | ✔ | -| [چوب نما](#prettyspace) | ✗ | ✔ | -| [Protobuf](#protobuf) | ✔ | ✔ | -| [اورو](#data-format-avro) | ✔ | ✔ | -| [هشدار داده می شود](#data-format-avro-confluent) | ✔ | ✗ | -| [پارکت](#data-format-parquet) | ✔ | ✔ | -| [ORC](#data-format-orc) | ✔ | ✗ | -| [مربوط به حوزه](#rowbinary) | ✔ | ✔ | -| [ارزشهای خبری عبارتند از:](#rowbinarywithnamesandtypes) | ✔ | ✔ | -| [بومی](#native) | ✔ | ✔ | -| [خالی](#null) | ✗ | ✔ | -| [XML](#xml) | ✗ | ✔ | -| [کاپپروتو](#capnproto) | ✔ | ✗ | - -شما می توانید برخی از پارامترهای پردازش فرمت با تنظیمات کلیک را کنترل کنید. برای اطلاعات بیشتر بخوانید [تنظیمات](../operations/settings/settings.md) بخش. - -## جدول دار {#tabseparated} - -در قالب جدولبندی داده ها توسط ردیف نوشته شده است. هر سطر شامل مقادیر جدا شده توسط زبانه. هر مقدار است که توسط یک تب به دنبال, به جز ارزش گذشته در ردیف, است که توسط یک خوراک خط به دنبال. به شدت یونیکس خط تغذیه در همه جا فرض شده است. ردیف گذشته نیز باید یک خوراک خط در پایان حاوی. ارزش ها در قالب متن نوشته شده, بدون گذاشتن علامت نقل قول, و با شخصیت های خاص فرار. - -این فرمت نیز تحت نام موجود است `TSV`. - -این `TabSeparated` فرمت مناسب برای پردازش داده ها با استفاده از برنامه های سفارشی و اسکریپت است. این است که به طور پیش فرض در رابط اچ تی پی و در حالت دسته ای مشتری خط فرمان استفاده می شود. این فرمت همچنین اجازه می دهد تا انتقال داده ها بین سرویس بهداشتی مختلف. مثلا, شما می توانید یک روگرفت از خروجی زیر و ارسال به کلیک, و یا بالعکس. - -این `TabSeparated` فرمت پشتیبانی خروجی ارزش کل (هنگام استفاده با بالغ) و ارزش های شدید (وقتی که ‘extremes’ به مجموعه 1). در این موارد, کل ارزش ها و افراط خروجی پس از داده های اصلی. نتیجه اصلی, کل ارزش, و افراط و از یکدیگر توسط یک خط خالی از هم جدا. مثال: - -``` sql -SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT TabSeparated`` -``` - -``` text -2014-03-17 1406958 -2014-03-18 1383658 -2014-03-19 1405797 -2014-03-20 1353623 -2014-03-21 1245779 -2014-03-22 1031592 -2014-03-23 1046491 - -1970-01-01 8873898 - -2014-03-17 1031592 -2014-03-23 1406958 -``` - -### قالببندی داده {#data-formatting} - -اعداد صحیح به شکل اعشاری نوشته شده است. اعداد می توانند حاوی اضافی باشند “+” شخصیت در ابتدا (هنگام تجزیه نادیده گرفته شد و هنگام قالب بندی ثبت نشده است). اعداد غیر منفی نمی توانند علامت منفی داشته باشند. هنگام خواندن, مجاز است به تجزیه یک رشته خالی به عنوان یک صفر, یا (برای انواع امضا شده) یک رشته متشکل از فقط یک علامت منفی به عنوان یک صفر. اعداد است که به نوع داده مربوطه مناسب نیست ممکن است به عنوان شماره های مختلف تجزیه, بدون پیغام خطا. - -اعداد ممیز شناور به شکل اعشاری نوشته شده است. نقطه به عنوان جداکننده اعشاری استفاده می شود. ورودی های نمایشی پشتیبانی می شوند ‘inf’, ‘+inf’, ‘-inf’ و ‘nan’. ورود اعداد ممیز شناور ممکن است شروع یا پایان با یک نقطه اعشار. -در قالب بندی, دقت ممکن است در اعداد ممیز شناور از دست داده. -در تجزیه, این است که به شدت مورد نیاز برای خواندن نزدیکترین شماره ماشین نمایندگی. - -تاریخ در فرمت یی-میلی متر-دی دی دی دی اس نوشته شده و تجزیه در قالب همان, اما با هر شخصیت به عنوان جدا. -تاریخ با زمان در قالب نوشته شده است `YYYY-MM-DD hh:mm:ss` و تجزیه در قالب همان, اما با هر شخصیت به عنوان جدا. -این همه در منطقه زمانی سیستم در زمان مشتری یا سرور شروع می شود (بسته به نوع فرمت داده ها) رخ می دهد. برای تاریخ با زمان, نور روز صرفه جویی در زمان مشخص نشده است. بنابراین اگر یک روگرفت است بار در طول نور روز صرفه جویی در زمان, روگرفت به صراحت مطابقت با داده ها نیست, و تجزیه یکی از دو بار را انتخاب کنید. -در طول یک عملیات به عنوان خوانده شده, تاریخ نادرست و تاریخ با زمان را می توان با سرریز طبیعی و یا تاریخ به عنوان تهی و بار تجزیه, بدون پیغام خطا. - -به عنوان یک استثنا, تجزیه تاریخ با زمان نیز در قالب برچسب زمان یونیکس پشتیبانی, اگر از دقیقا شامل 10 رقم اعشار. نتیجه این است زمان وابسته به منطقه نیست. فرمت های یی-ام-دی-دی-اچ: میلی متر: اس اس و نونن به طور خودکار متفاوت هستند. - -رشته ها خروجی با شخصیت های خاص بک اسلش فرار. توالی فرار زیر برای خروجی استفاده می شود: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\'`, `\\`. تجزیه همچنین از توالی ها پشتیبانی می کند `\a`, `\v` و `\xHH` (توالی فرار سحر و جادو) و هر `\c` دنباله هایی که `c` هر شخصیت (این توالی ها به تبدیل `c`). بدین ترتیب, خواندن داده ها پشتیبانی از فرمت های که یک خوراک خط را می توان به عنوان نوشته شده `\n` یا `\`, و یا به عنوان یک خوراک خط. مثلا, رشته `Hello world` با خوراک خط بین کلمات به جای فضا را می توان در هر یک از تغییرات زیر تجزیه شده است: - -``` text -Hello\nworld - -Hello\ -world -``` - -نوع دوم پشتیبانی می شود زیرا خروجی زیر هنگام نوشتن افسردگی های جدا شده از تب استفاده می کند. - -حداقل مجموعه ای از شخصیت های که شما نیاز به فرار در هنگام عبور داده ها در قالب جدول پخش: باریکه, خوراک خط (ال اف) و بک اسلش. - -فقط یک مجموعه کوچک از علامت فرار. شما به راحتی می توانید بر روی یک مقدار رشته تلو تلو خوردن که ترمینال خود را در خروجی خراب کردن. - -ارریس به عنوان یک لیست از ارزش کاما از هم جدا در براکت مربع نوشته شده است. موارد شماره در مجموعه به طور معمول فرمت می شوند. `Date` و `DateTime` انواع در نقل قول تک نوشته شده است. رشته ها در نقل قول های تک با قوانین فرار همان بالا نوشته شده است. - -[NULL](../sql-reference/syntax.md) به عنوان فرمت `\N`. - -هر عنصر [تو در تو](../sql-reference/data-types/nested-data-structures/nested.md) سازه ها به عنوان مجموعه ای نشان داده شده است. - -به عنوان مثال: - -``` sql -CREATE TABLE nestedt -( - `id` UInt8, - `aux` Nested( - a UInt8, - b String - ) -) -ENGINE = TinyLog -``` - -``` sql -INSERT INTO nestedt Values ( 1, [1], ['a']) -``` - -``` sql -SELECT * FROM nestedt FORMAT TSV -``` - -``` text -1 [1] ['a'] -``` - -## اطلاعات دقیق {#tabseparatedraw} - -متفاوت از `TabSeparated` فرمت که در ردیف بدون فرار نوشته شده است. -این فرمت فقط برای خروجی یک نتیجه پرس و جو مناسب است, اما نه برای تجزیه (بازیابی اطلاعات برای وارد کردن در یک جدول). - -این فرمت نیز تحت نام موجود است `TSVRaw`. - -## اطلاعات دقیق {#tabseparatedwithnames} - -متفاوت از `TabSeparated` فرمت در که نام ستون در سطر اول نوشته شده است. -در تجزیه, ردیف اول به طور کامل نادیده گرفته. شما می توانید نام ستون برای تعیین موقعیت خود و یا برای بررسی صحت خود استفاده کنید. -(پشتیبانی از تجزیه ردیف هدر ممکن است در اینده اضافه شده است.) - -این فرمت نیز تحت نام موجود است `TSVWithNames`. - -## اطلاعات دقیق {#tabseparatedwithnamesandtypes} - -متفاوت از `TabSeparated` فرمت در که نام ستون به سطر اول نوشته شده است, در حالی که انواع ستون در ردیف دوم هستند. -در تجزیه, ردیف اول و دوم به طور کامل نادیده گرفته. - -این فرمت نیز تحت نام موجود است `TSVWithNamesAndTypes`. - -## قالب {#format-template} - -این فرمت اجازه می دهد تا تعیین یک رشته فرمت سفارشی با متغیرهایی برای ارزش با یک قاعده فرار مشخص شده است. - -با استفاده از تنظیمات `format_template_resultset`, `format_template_row`, `format_template_rows_between_delimiter` and some settings of other formats (e.g. `output_format_json_quote_64bit_integers` هنگام استفاده از `JSON` فرار, مشاهده بیشتر) - -تنظیم `format_template_row` مشخص مسیر به فایل, که شامل رشته فرمت برای ردیف با نحو زیر: - -`delimiter_1${column_1:serializeAs_1}delimiter_2${column_2:serializeAs_2} ... delimiter_N`, - -کجا `delimiter_i` یک جداساز بین مقادیر است (`$` نماد را می توان به عنوان فرار `$$`), -`column_i` یک نام یا فهرست یک ستون است که مقادیر انتخاب شده یا درج شده است (اگر خالی باشد سپس ستون حذف خواهد شد), -`serializeAs_i` یک قانون فرار برای مقادیر ستون است. قوانین فرار زیر پشتیبانی می شوند: - -- `CSV`, `JSON`, `XML` (به طور مشابه به فرمت های نام های مشابه) -- `Escaped` (به طور مشابه به `TSV`) -- `Quoted` (به طور مشابه به `Values`) -- `Raw` (بدون فرار, به طور مشابه به `TSVRaw`) -- `None` (هیچ قانون فرار, مشاهده بیشتر) - -اگر یک قانون فرار حذف شده است, سپس `None` استفاده خواهد شد. `XML` و `Raw` فقط برای خروجی مناسب است. - -بنابراین, برای رشته فرمت زیر: - - `Search phrase: ${SearchPhrase:Quoted}, count: ${c:Escaped}, ad price: $$${price:JSON};` - -ارزش `SearchPhrase`, `c` و `price` ستون ها که به عنوان فرار `Quoted`, `Escaped` و `JSON` چاپ خواهد شد (برای انتخاب) و یا انتظار می رود (برای درج) میان `Search phrase:`, `, count:`, `, ad price: $` و `;` delimiters بود. به عنوان مثال: - -`Search phrase: 'bathroom interior design', count: 2166, ad price: $3;` - -این `format_template_rows_between_delimiter` تنظیم مشخص جداساز بین ردیف, که چاپ شده است (یا انتظار می رود) بعد از هر سطر به جز یکی از گذشته (`\n` به طور پیش فرض) - -تنظیم `format_template_resultset` مشخص کردن مسیر فایل که شامل یک رشته فرمت برای resultset. رشته فرمت برای حاصل است نحو همان رشته فرمت برای ردیف و اجازه می دهد تا برای مشخص کردن یک پیشوند, پسوند و یک راه برای چاپ برخی از اطلاعات اضافی. این شامل متغیرهایی زیر به جای نام ستون: - -- `data` ردیف با داده ها در `format_template_row` قالب, جدا شده توسط `format_template_rows_between_delimiter`. این حفره یا سوراخ باید اولین حفره یا سوراخ در رشته فرمت باشد. -- `totals` ردیف با کل ارزش ها در `format_template_row` قالب (هنگام استفاده با مجموع) -- `min` ردیف با حداقل مقدار در `format_template_row` فرمت (هنگامی که افراط و به مجموعه 1) -- `max` ردیف با حداکثر ارزش در است `format_template_row` فرمت (هنگامی که افراط و به مجموعه 1) -- `rows` تعداد کل ردیف خروجی است -- `rows_before_limit` است حداقل تعداد ردیف وجود دارد که بدون محدودیت بوده است. خروجی تنها در صورتی که پرس و جو شامل حد. اگر پرس و جو شامل گروه های, ردیف ها_افور_لیمیت_تلاست تعداد دقیق ردیف وجود دارد که بدون محدودیت بوده است. -- `time` زمان اجرای درخواست در ثانیه است -- `rows_read` است تعداد ردیف خوانده شده است -- `bytes_read` تعداد بایت (غیر فشرده) خوانده شده است - -متغیرهایی `data`, `totals`, `min` و `max` باید فرار حکومت مشخص نیست (یا `None` باید به صراحت مشخص شود). متغیرهایی باقی مانده ممکن است هر گونه قانون فرار مشخص شده اند. -اگر `format_template_resultset` تنظیم یک رشته خالی است, `${data}` به عنوان مقدار پیش فرض استفاده می شود. -برای قرار دادن نمایش داده شد فرمت اجازه می دهد تا پرش برخی از ستون ها و یا برخی از زمینه های اگر پیشوند یا پسوند (به عنوان مثال مراجعه کنید). - -انتخاب نمونه: - -``` sql -SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase ORDER BY c DESC LIMIT 5 FORMAT Template SETTINGS -format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format', format_template_rows_between_delimiter = '\n ' -``` - -`/some/path/resultset.format`: - -``` text - - Search phrases - - - - ${data} -
Search phrases
Search phrase Count
- - ${max} -
Max
- Processed ${rows_read:XML} rows in ${time:XML} sec - - -``` - -`/some/path/row.format`: - -``` text - ${0:XML} ${1:XML} -``` - -نتیجه: - -``` html - - Search phrases - - - - - - - - -
Search phrases
Search phrase Count
8267016
bathroom interior design 2166
yandex 1655
spring 2014 fashion 1549
freeform photos 1480
- - -
Max
8873898
- Processed 3095973 rows in 0.1569913 sec - - -``` - -درج مثال: - -``` text -Some header -Page views: 5, User id: 4324182021466249494, Useless field: hello, Duration: 146, Sign: -1 -Page views: 6, User id: 4324182021466249494, Useless field: world, Duration: 185, Sign: 1 -Total rows: 2 -``` - -``` sql -INSERT INTO UserActivity FORMAT Template SETTINGS -format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format' -``` - -`/some/path/resultset.format`: - -``` text -Some header\n${data}\nTotal rows: ${:CSV}\n -``` - -`/some/path/row.format`: - -``` text -Page views: ${PageViews:CSV}, User id: ${UserID:CSV}, Useless field: ${:CSV}, Duration: ${Duration:CSV}, Sign: ${Sign:CSV} -``` - -`PageViews`, `UserID`, `Duration` و `Sign` در داخل متغیرهایی نام ستون در جدول هستند. مقادیر پس از `Useless field` در ردیف و بعد از `\nTotal rows:` در پسوند نادیده گرفته خواهد شد. -همه delimiters در داده های ورودی باید به شدت در برابر delimiters در فرمت مشخص شده رشته. - -## پاسخ تمپلیتینی {#templateignorespaces} - -این فرمت فقط برای ورودی مناسب است. -مشابه به `Template`, اما پرش کاراکتر فضای سفید بین جداکننده ها و ارزش ها در جریان ورودی. با این حال, اگر رشته فرمت حاوی کاراکتر فضای سفید, این شخصیت خواهد شد در جریان ورودی انتظار می رود. همچنین اجازه می دهد برای مشخص متغیرهایی خالی (`${}` یا `${:None}`) به تقسیم برخی از جداساز به قطعات جداگانه به چشم پوشی از فضاهای بین. چنین متغیرهایی تنها برای پرش شخصیت فضای سفید استفاده می شود. -خواندن ممکن است `JSON` با استفاده از این فرمت, اگر ارزش ستون همان نظم در تمام ردیف. برای مثال درخواست زیر می تواند مورد استفاده قرار گیرد برای قرار دادن داده ها از خروجی نمونه ای از فرمت [JSON](#json): - -``` sql -INSERT INTO table_name FORMAT TemplateIgnoreSpaces SETTINGS -format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format', format_template_rows_between_delimiter = ',' -``` - -`/some/path/resultset.format`: - -``` text -{${}"meta"${}:${:JSON},${}"data"${}:${}[${data}]${},${}"totals"${}:${:JSON},${}"extremes"${}:${:JSON},${}"rows"${}:${:JSON},${}"rows_before_limit_at_least"${}:${:JSON}${}} -``` - -`/some/path/row.format`: - -``` text -{${}"SearchPhrase"${}:${}${phrase:JSON}${},${}"c"${}:${}${cnt:JSON}${}} -``` - -## TSKV {#tskv} - -شبیه به جدول, اما خروجی یک مقدار در نام=فرمت ارزش. نام ها به همان شیوه در قالب جدول فرار, و = نماد نیز فرار. - -``` text -SearchPhrase= count()=8267016 -SearchPhrase=bathroom interior design count()=2166 -SearchPhrase=yandex count()=1655 -SearchPhrase=2014 spring fashion count()=1549 -SearchPhrase=freeform photos count()=1480 -SearchPhrase=angelina jolie count()=1245 -SearchPhrase=omsk count()=1112 -SearchPhrase=photos of dog breeds count()=1091 -SearchPhrase=curtain designs count()=1064 -SearchPhrase=baku count()=1000 -``` - -[NULL](../sql-reference/syntax.md) به عنوان فرمت `\N`. - -``` sql -SELECT * FROM t_null FORMAT TSKV -``` - -``` text -x=1 y=\N -``` - -هنگامی که تعداد زیادی از ستون های کوچک وجود دارد, این فرمت بی اثر است, و به طور کلی هیچ دلیلی برای استفاده وجود دارد. با این وجود از لحاظ کارایی بدتر از جیسوناکرو نیست. - -Both data output and parsing are supported in this format. For parsing, any order is supported for the values of different columns. It is acceptable for some values to be omitted – they are treated as equal to their default values. In this case, zeros and blank rows are used as default values. Complex values that could be specified in the table are not supported as defaults. - -تجزیه اجازه می دهد تا حضور زمینه های اضافی `tskv` بدون علامت مساوی یا ارزش. این زمینه نادیده گرفته شده است. - -## CSV {#csv} - -با کاما از هم جدا فرمت ارزش ([RFC](https://tools.ietf.org/html/rfc4180)). - -هنگام قالب بندی, ردیف در دو نقل قول محصور. نقل قول دو در داخل یک رشته خروجی به عنوان دو نقل قول دو در یک ردیف است. هیچ قانون دیگری برای فرار از شخصیت وجود دارد. تاریخ و تاریخ زمان در دو نقل قول محصور شده است. اعداد خروجی بدون نقل قول. ارزش ها توسط یک شخصیت جداساز از هم جدا, که `,` به طور پیش فرض. شخصیت جداساز در تنظیمات تعریف شده است [_مخفی کردن _قابلیت _جدید](../operations/settings/settings.md#settings-format_csv_delimiter). ردیف ها با استفاده از خوراک خط یونیکس جدا می شوند. ارریس در سی سی اس وی به شرح زیر مرتب شده است: ابتدا مجموعه ای به یک رشته به عنوان در قالب تبسپار شده مرتب شده است و سپس رشته حاصل خروجی به سی سی اس وی در دو نقل قول است. دسته بندی ها در قالب سی اس وی به صورت ستون های جداگانه مرتب می شوند (به این معنا که لانه خود را در تاپل از دست داده است). - -``` bash -$ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FORMAT CSV" < data.csv -``` - -\* به طور پیش فرض, جداساز است `,`. دیدن [_مخفی کردن _قابلیت _جدید](../operations/settings/settings.md#settings-format_csv_delimiter) تنظیم برای اطلاعات بیشتر. - -هنگامی که تجزیه, تمام مقادیر را می توان یا با یا بدون نقل قول تجزیه. هر دو نقل قول دو و تک پشتیبانی می شوند. ردیف همچنین می توانید بدون نقل قول مرتب شود. در این مورد, به شخصیت جداساز و یا خوراک خط تجزیه (کروم و یا ال اف). در هنگام تجزیه ردیف بدون نقل قول فضاهای پیشرو و انتهایی و زبانه ها نادیده گرفته می شوند. برای اشتراک خط, یونیکس (کلیک کنید), پنجره ها (کروم ال اف) و سیستم عامل مک کلاسیک (کروم ال اف) انواع پشتیبانی می شوند. - -مقادیر ورودی بدون علامت خالی با مقادیر پیش فرض برای ستون های مربوطه جایگزین می شوند -[_پوشه های ورودی و خروجی](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields) -فعال است. - -`NULL` به عنوان فرمت `\N` یا `NULL` یا یک رشته بدون علامت خالی (تنظیمات را ببینید [_فرستادن به _کوچکنمایی](../operations/settings/settings.md#settings-input_format_csv_unquoted_null_literal_as_null) و [_پوشه های ورودی و خروجی](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields)). - -پشتیبانی از خروجی بالغ و افراط به همان شیوه به عنوان `TabSeparated`. - -## اطلاعات دقیق {#csvwithnames} - -همچنین چاپ ردیف هدر, شبیه به `TabSeparatedWithNames`. - -## سفارشی {#format-customseparated} - -مشابه به [قالب](#format-template) اما همه ستون ها را چاپ می کند یا می خواند و از قاعده فرار از تنظیم استفاده می کند `format_custom_escaping_rule` و جداکننده از تنظیمات `format_custom_field_delimiter`, `format_custom_row_before_delimiter`, `format_custom_row_after_delimiter`, `format_custom_row_between_delimiter`, `format_custom_result_before_delimiter` و `format_custom_result_after_delimiter`, نه از رشته فرمت. -همچنین وجود دارد `CustomSeparatedIgnoreSpaces` قالب که شبیه به `TemplateIgnoreSpaces`. - -## JSON {#json} - -خروجی داده ها در فرمت جانسون. علاوه بر جداول داده, همچنین خروجی نام ستون و انواع, همراه با برخی از اطلاعات اضافی: تعداد کل ردیف خروجی, و تعداد ردیف است که می تواند خروجی بوده است اگر یک محدودیت وجود ندارد. مثال: - -``` sql -SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTALS ORDER BY c DESC LIMIT 5 FORMAT JSON -``` - -``` json -{ - "meta": - [ - { - "name": "SearchPhrase", - "type": "String" - }, - { - "name": "c", - "type": "UInt64" - } - ], - - "data": - [ - { - "SearchPhrase": "", - "c": "8267016" - }, - { - "SearchPhrase": "bathroom interior design", - "c": "2166" - }, - { - "SearchPhrase": "yandex", - "c": "1655" - }, - { - "SearchPhrase": "spring 2014 fashion", - "c": "1549" - }, - { - "SearchPhrase": "freeform photos", - "c": "1480" - } - ], - - "totals": - { - "SearchPhrase": "", - "c": "8873898" - }, - - "extremes": - { - "min": - { - "SearchPhrase": "", - "c": "1480" - }, - "max": - { - "SearchPhrase": "", - "c": "8267016" - } - }, - - "rows": 5, - - "rows_before_limit_at_least": 141137 -} -``` - -جانسون سازگار با جاوا اسکریپت است. برای اطمینان از این, برخی از شخصیت ها علاوه بر فرار: بریده بریده `/` فرار به عنوان `\/`; معافیت خط جایگزین `U+2028` و `U+2029`, که شکستن برخی از مرورگرهای, به عنوان فرار `\uXXXX`. شخصیت های کنترل اسکی فرار: برگشت به عقب, خوراک فرم, خوراک خط, بازگشت حمل, و تب افقی با جایگزین `\b`, `\f`, `\n`, `\r`, `\t` , و همچنین بایت باقی مانده در محدوده 00-1ف با استفاده از `\uXXXX` sequences. Invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. For compatibility with JavaScript, Int64 and UInt64 integers are enclosed in double-quotes by default. To remove the quotes, you can set the configuration parameter [خروجی _فرمان_جسون_کوات_64بیت_تنظیمی](../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) به 0. - -`rows` – The total number of output rows. - -`rows_before_limit_at_least` حداقل تعداد ردیف وجود دارد که بدون محدودیت بوده است. خروجی تنها در صورتی که پرس و جو شامل حد. -اگر پرس و جو شامل گروه های, ردیف ها_افور_لیمیت_تلاست تعداد دقیق ردیف وجود دارد که بدون محدودیت بوده است. - -`totals` – Total values (when using WITH TOTALS). - -`extremes` – Extreme values (when extremes are set to 1). - -این فرمت فقط برای خروجی یک نتیجه پرس و جو مناسب است, اما نه برای تجزیه (بازیابی اطلاعات برای وارد کردن در یک جدول). - -پشتیبانی از کلیک [NULL](../sql-reference/syntax.md), است که به عنوان نمایش داده `null` در خروجی جانسون. - -همچنین نگاه کنید به [جیسانچرو](#jsoneachrow) قالب. - -## فوق العاده {#jsoncompact} - -متفاوت از جانسون تنها در ردیف داده ها خروجی در ارریس, نه در اشیا. - -مثال: - -``` json -{ - "meta": - [ - { - "name": "SearchPhrase", - "type": "String" - }, - { - "name": "c", - "type": "UInt64" - } - ], - - "data": - [ - ["", "8267016"], - ["bathroom interior design", "2166"], - ["yandex", "1655"], - ["fashion trends spring 2014", "1549"], - ["freeform photo", "1480"] - ], - - "totals": ["","8873898"], - - "extremes": - { - "min": ["","1480"], - "max": ["","8267016"] - }, - - "rows": 5, - - "rows_before_limit_at_least": 141137 -} -``` - -این فرمت فقط برای خروجی یک نتیجه پرس و جو مناسب است, اما نه برای تجزیه (بازیابی اطلاعات برای وارد کردن در یک جدول). -همچنین نگاه کنید به `JSONEachRow` قالب. - -## جیسانچرو {#jsoneachrow} - -هنگامی که با استفاده از این فرمت, تاتر خروجی ردیف به عنوان جدا, اجسام جسون-خط حد و مرز مشخصی, اما داده ها به عنوان یک کل است جسون معتبر نیست. - -``` json -{"SearchPhrase":"curtain designs","count()":"1064"} -{"SearchPhrase":"baku","count()":"1000"} -{"SearchPhrase":"","count()":"8267016"} -``` - -هنگام قرار دادن داده ها, شما باید یک شی جانسون جداگانه برای هر سطر فراهم. - -### درج داده {#inserting-data} - -``` sql -INSERT INTO UserActivity FORMAT JSONEachRow {"PageViews":5, "UserID":"4324182021466249494", "Duration":146,"Sign":-1} {"UserID":"4324182021466249494","PageViews":6,"Duration":185,"Sign":1} -``` - -کلیک اجازه می دهد تا: - -- هر منظور از جفت کلید ارزش در جسم. -- حذف برخی از ارزش ها. - -تاتر فضاهای بین عناصر و کاما را پس از اشیا نادیده می گیرد. شما می توانید تمام اشیا را در یک خط منتقل کنید. لازم نیست با شکستن خط اونارو جدا کنی - -**حذف پردازش مقادیر** - -را کلیک کنید جایگزین مقادیر حذف شده با مقادیر پیش فرض برای مربوطه [انواع داده ها](../sql-reference/data-types/index.md). - -اگر `DEFAULT expr` مشخص شده است, تاتر با استفاده از قوانین تعویض مختلف بسته به [_پوشه های ورودی و خروجی](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields) تنظیمات. - -جدول زیر را در نظر بگیرید: - -``` sql -CREATE TABLE IF NOT EXISTS example_table -( - x UInt32, - a DEFAULT x * 2 -) ENGINE = Memory; -``` - -- اگر `input_format_defaults_for_omitted_fields = 0` سپس مقدار پیش فرض برای `x` و `a` برابر `0` (به عنوان مقدار پیش فرض برای `UInt32` نوع داده). -- اگر `input_format_defaults_for_omitted_fields = 1` سپس مقدار پیش فرض برای `x` برابر `0` اما مقدار پیش فرض `a` برابر `x * 2`. - -!!! note "اخطار" - هنگام قرار دادن داده ها با `insert_sample_with_metadata = 1`, کلیکهاوس مصرف منابع محاسباتی بیشتر, در مقایسه با درج با `insert_sample_with_metadata = 0`. - -### انتخاب داده ها {#selecting-data} - -در نظر بگیرید که `UserActivity` جدول به عنوان مثال: - -``` text -┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ -│ 4324182021466249494 │ 5 │ 146 │ -1 │ -│ 4324182021466249494 │ 6 │ 185 │ 1 │ -└─────────────────────┴───────────┴──────────┴──────┘ -``` - -پرسوجو `SELECT * FROM UserActivity FORMAT JSONEachRow` بازگشت: - -``` text -{"UserID":"4324182021466249494","PageViews":5,"Duration":146,"Sign":-1} -{"UserID":"4324182021466249494","PageViews":6,"Duration":185,"Sign":1} -``` - -بر خلاف [JSON](#json) قالب, هیچ جایگزینی نامعتبر گفته-8 توالی وجود دارد. ارزش ها در همان راه برای فرار `JSON`. - -!!! note "یادداشت" - هر مجموعه ای از بایت می تواند خروجی در رشته ها. استفاده از `JSONEachRow` فرمت اگر شما اطمینان حاصل کنید که داده ها در جدول را می توان به عنوان جانسون بدون از دست دادن هر گونه اطلاعات فرمت شده است. - -### استفاده از ساختارهای تو در تو {#jsoneachrow-nested} - -اگر شما یک جدول با [تو در تو](../sql-reference/data-types/nested-data-structures/nested.md) ستون نوع داده, شما می توانید داده های جانسون با همان ساختار وارد. فعال کردن این ویژگی با [تغییر _کم_تر_تنظیم مجدد _جنسان](../operations/settings/settings.md#settings-input_format_import_nested_json) تنظیمات. - -برای مثال جدول زیر را در نظر بگیرید: - -``` sql -CREATE TABLE json_each_row_nested (n Nested (s String, i Int32) ) ENGINE = Memory -``` - -همانطور که شما می توانید در `Nested` شرح نوع داده, تاتر رفتار هر یک از اجزای ساختار تو در تو به عنوان یک ستون جداگانه (`n.s` و `n.i` برای جدول ما). شما می توانید داده ها را به روش زیر وارد کنید: - -``` sql -INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n.s": ["abc", "def"], "n.i": [1, 23]} -``` - -برای قرار دادن داده ها به عنوان یک شی جوسون سلسله مراتبی, تنظیم [وارد کردن _ترمپ_م_تر_اس_جسون ثبت شده=1](../operations/settings/settings.md#settings-input_format_import_nested_json). - -``` json -{ - "n": { - "s": ["abc", "def"], - "i": [1, 23] - } -} -``` - -بدون این تنظیم, محل کلیک می اندازد یک استثنا. - -``` sql -SELECT name, value FROM system.settings WHERE name = 'input_format_import_nested_json' -``` - -``` text -┌─name────────────────────────────┬─value─┐ -│ input_format_import_nested_json │ 0 │ -└─────────────────────────────────┴───────┘ -``` - -``` sql -INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n": {"s": ["abc", "def"], "i": [1, 23]}} -``` - -``` text -Code: 117. DB::Exception: Unknown field found while parsing JSONEachRow format: n: (at row 1) -``` - -``` sql -SET input_format_import_nested_json=1 -INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n": {"s": ["abc", "def"], "i": [1, 23]}} -SELECT * FROM json_each_row_nested -``` - -``` text -┌─n.s───────────┬─n.i────┐ -│ ['abc','def'] │ [1,23] │ -└───────────────┴────────┘ -``` - -## بومی {#native} - -فرمت موثر ترین. داده ها توسط بلوک ها در فرمت باینری نوشته شده و خوانده می شوند. برای هر بلوک, تعداد ردیف, تعداد ستون, نام ستون و انواع, و بخش هایی از ستون ها در این بلوک یکی پس از دیگری ثبت. به عبارت دیگر این قالب است “columnar” – it doesn't convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. - -شما می توانید این فرمت را به سرعت تولید افسردگی است که تنها می تواند توسط سندرم تونل کارپ به عنوان خوانده شده استفاده کنید. این حس برای کار با این فرمت خود را ندارد. - -## خالی {#null} - -هیچ چیز خروجی است. اما پرس و جو پردازش شده است و در هنگام استفاده از خط فرمان مشتری داده منتقل می شود به مشتری. این است که برای تست استفاده, از جمله تست عملکرد. -به طور مشخص, این فرمت فقط برای خروجی مناسب است, نه برای تجزیه. - -## زیبا {#pretty} - -خروجی داده ها به عنوان جداول یونیکد هنر, همچنین با استفاده از توالی انسی فرار برای تنظیم رنگ در ترمینال. -یک شبکه کامل از جدول کشیده شده است, و هر سطر را اشغال دو خط در ترمینال. -هر بلوک نتیجه خروجی به عنوان یک جدول جداگانه است. این لازم است به طوری که بلوک می تواند خروجی بدون نتیجه بافر (بافر می شود به منظور قبل از محاسبه عرض قابل مشاهده از تمام مقادیر لازم). - -[NULL](../sql-reference/syntax.md) خروجی به عنوان `ᴺᵁᴸᴸ`. - -مثال (نشان داده شده برای [پیش تیمار](#prettycompact) قالب): - -``` sql -SELECT * FROM t_null -``` - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -└───┴──────┘ -``` - -ردیف در زیبا \* فرمت فرار نیست. به عنوان مثال برای نشان داده شده است [پیش تیمار](#prettycompact) قالب: - -``` sql -SELECT 'String with \'quotes\' and \t character' AS Escaping_test -``` - -``` text -┌─Escaping_test────────────────────────┐ -│ String with 'quotes' and character │ -└──────────────────────────────────────┘ -``` - -برای جلوگیری از تخلیه اطلاعات بیش از حد به ترمینال تنها 10000 ردیف اول چاپ شده است. اگر تعداد ردیف بیشتر یا برابر با است 10,000, پیام “Showed first 10 000” چاپ شده است. -این فرمت فقط برای خروجی یک نتیجه پرس و جو مناسب است, اما نه برای تجزیه (بازیابی اطلاعات برای وارد کردن در یک جدول). - -فرمت زیبا پشتیبانی خروجی ارزش کل (هنگام استفاده با بالغ) و افراط و (وقتی که ‘extremes’ به مجموعه 1). در این موارد, کل ارزش ها و ارزش های شدید خروجی پس از داده های اصلی هستند, در جداول جداگانه. مثال (نشان داده شده برای [پیش تیمار](#prettycompact) قالب): - -``` sql -SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT PrettyCompact -``` - -``` text -┌──EventDate─┬───────c─┐ -│ 2014-03-17 │ 1406958 │ -│ 2014-03-18 │ 1383658 │ -│ 2014-03-19 │ 1405797 │ -│ 2014-03-20 │ 1353623 │ -│ 2014-03-21 │ 1245779 │ -│ 2014-03-22 │ 1031592 │ -│ 2014-03-23 │ 1046491 │ -└────────────┴─────────┘ - -Totals: -┌──EventDate─┬───────c─┐ -│ 1970-01-01 │ 8873898 │ -└────────────┴─────────┘ - -Extremes: -┌──EventDate─┬───────c─┐ -│ 2014-03-17 │ 1031592 │ -│ 2014-03-23 │ 1406958 │ -└────────────┴─────────┘ -``` - -## پیش تیمار {#prettycompact} - -متفاوت از [زیبا](#pretty) در که شبکه بین ردیف کشیده شده و نتیجه جمع و جور تر است. -این فرمت به طور پیش فرض در مشتری خط فرمان در حالت تعاملی استفاده می شود. - -## بلوک پیش ساخته {#prettycompactmonoblock} - -متفاوت از [پیش تیمار](#prettycompact) در که تا 10000 ردیف بافر و سپس خروجی به عنوان یک جدول واحد نه بلوک. - -## کتاب های پیش بینی شده {#prettynoescapes} - -متفاوت از زیبا که در توالی اس-فرار استفاده نمی شود. این برای نمایش این فرمت در یک مرورگر و همچنین برای استفاده ضروری است ‘watch’ ابزار خط فرمان. - -مثال: - -``` bash -$ watch -n1 "clickhouse-client --query='SELECT event, value FROM system.events FORMAT PrettyCompactNoEscapes'" -``` - -شما می توانید رابط قام برای نمایش در مرورگر استفاده کنید. - -### کتاب های پیش ساخته {#prettycompactnoescapes} - -همان تنظیمات قبلی. - -### کتاب های پیش بینی شده {#prettyspacenoescapes} - -همان تنظیمات قبلی. - -## چوب نما {#prettyspace} - -متفاوت از [پیش تیمار](#prettycompact) در که فضای سفید (شخصیت های فضایی) به جای شبکه استفاده می شود. - -## مربوط به حوزه {#rowbinary} - -فرمت ها و تجزیه داده ها توسط ردیف در فرمت باینری. سطر و ارزش ها به صورت متوالی ذکر شده, بدون جدا. -این فرمت کمتر از فرمت بومی است زیرا مبتنی بر ردیف است. - -اعداد صحیح استفاده از ثابت طول نمایندگی کوچک اندی. مثلا, اوینت64 با استفاده از 8 بایت. -تاریخ ساعت به عنوان اوینت32 حاوی برچسب زمان یونیکس به عنوان ارزش نشان داده است. -تاریخ به عنوان یک شی اوینت16 که شامل تعداد روز از 1970-01-01 به عنوان ارزش نشان داده شده است. -رشته به عنوان یک طول ورق (بدون علامت) نشان داده شده است [LEB128](https://en.wikipedia.org/wiki/LEB128)), پس از بایت رشته. -رشته ثابت است که به سادگی به عنوان یک دنباله از بایت نشان داده شده است. - -اری به عنوان یک طول ورینت (بدون علامت) نشان داده شده است [LEB128](https://en.wikipedia.org/wiki/LEB128)), پس از عناصر پی در پی از مجموعه. - -برای [NULL](../sql-reference/syntax.md#null-literal) حمایت کردن, یک بایت اضافی حاوی 1 یا 0 قبل از هر اضافه [Nullable](../sql-reference/data-types/nullable.md) ارزش. اگر 1, سپس ارزش است `NULL` و این بایت به عنوان یک مقدار جداگانه تفسیر. اگر 0, ارزش پس از بایت است `NULL`. - -## ارزشهای خبری عبارتند از: {#rowbinarywithnamesandtypes} - -مشابه به [مربوط به حوزه](#rowbinary) اما با هدر اضافه شده است: - -- [LEB128](https://en.wikipedia.org/wiki/LEB128)- کد گذاری تعداد ستون ها) -- N `String`مشخص کردن نامهای ستون -- N `String`بازدید کنندگان مشخص انواع ستون - -## مقادیر {#data-format-values} - -چاپ هر سطر در براکت. ردیف ها توسط کاما جدا می شوند. بعد از ردیف گذشته هیچ کاما وجود ندارد. مقادیر داخل براکت نیز با کاما از هم جدا هستند. اعداد خروجی در قالب اعشاری بدون نقل قول هستند. ارریس خروجی در براکت مربع است. رشته, تاریخ, و تاریخ با زمان خروجی در نقل قول. فرار قوانین و تجزیه شبیه به [جدول دار](#tabseparated) قالب. در قالب بندی فضاهای اضافی وارد نشده اند اما در طول تجزیه مجاز و نادیده گرفته می شوند (به جز فضاهای درون مقادیر مجموعه ای که مجاز نیستند). [NULL](../sql-reference/syntax.md) به عنوان نمایندگی `NULL`. - -The minimum set of characters that you need to escape when passing data in Values ​​format: single quotes and backslashes. - -این فرمت است که در استفاده می شود `INSERT INTO t VALUES ...` اما شما همچنین می توانید برای قالب بندی نتایج پرس و جو استفاده کنید. - -همچنین نگاه کنید به: [در حال خواندن:](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) و [در حال خواندن:](../operations/settings/settings.md#settings-input_format_values_deduce_templates_of_expressions) تنظیمات. - -## عمودی {#vertical} - -چاپ هر مقدار در یک خط جداگانه با نام ستون مشخص. این فرمت مناسب برای چاپ فقط یک یا چند ردیف است اگر هر سطر شامل تعداد زیادی از ستون. - -[NULL](../sql-reference/syntax.md) خروجی به عنوان `ᴺᵁᴸᴸ`. - -مثال: - -``` sql -SELECT * FROM t_null FORMAT Vertical -``` - -``` text -Row 1: -────── -x: 1 -y: ᴺᵁᴸᴸ -``` - -ردیف در فرمت عمودی فرار نیست: - -``` sql -SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT Vertical -``` - -``` text -Row 1: -────── -test: string with 'quotes' and with some special - characters -``` - -این فرمت فقط برای خروجی یک نتیجه پرس و جو مناسب است, اما نه برای تجزیه (بازیابی اطلاعات برای وارد کردن در یک جدول). - -## عمودی {#verticalraw} - -مشابه به [عمودی](#vertical), اما با فرار غیر فعال. این قالب فقط برای خروجی نتایج پرس و جو مناسب است, نه برای تجزیه (دریافت داده ها و قرار دادن در جدول). - -## XML {#xml} - -فرمت فقط برای خروجی مناسب است نه برای تجزیه. مثال: - -``` xml - - - - - - SearchPhrase - String - - - count() - UInt64 - - - - - - - 8267016 - - - bathroom interior design - 2166 - - - yandex - 1655 - - - 2014 spring fashion - 1549 - - - freeform photos - 1480 - - - angelina jolie - 1245 - - - omsk - 1112 - - - photos of dog breeds - 1091 - - - curtain designs - 1064 - - - baku - 1000 - - - 10 - 141137 - -``` - -اگر نام ستون یک فرمت قابل قبول ندارد, فقط ‘field’ به عنوان نام عنصر استفاده می شود. به طور کلی ساختار ایکس میل از ساختار جسون پیروی می کند. -Just as for JSON, invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. - -در مقادیر رشته, شخصیت `<` و `&` فرار به عنوان `<` و `&`. - -ارریس خروجی به عنوان `HelloWorld...`,و تاپل به عنوان `HelloWorld...`. - -## کاپپروتو {#capnproto} - -کپن پروتو فرمت پیام باینری شبیه به بافر پروتکل و صرفه جویی است, اما جسون یا مساگپک را دوست ندارد. - -پیام های کپ ان پروتو به شدت تایپ شده و نه خود توصیف, به این معنی که نیاز به یک شرح طرح خارجی. طرح در پرواز اعمال می شود و ذخیره سازی برای هر پرس و جو. - -``` bash -$ cat capnproto_messages.bin | clickhouse-client --query "INSERT INTO test.hits FORMAT CapnProto SETTINGS format_schema='schema:Message'" -``` - -کجا `schema.capnp` به نظر می رسد مثل این: - -``` capnp -struct Message { - SearchPhrase @0 :Text; - c @1 :Uint64; -} -``` - -Deserialization موثر است و معمولا نمی افزایش بار سیستم. - -همچنین نگاه کنید به [شمای فرمت](#formatschema). - -## Protobuf {#protobuf} - -Protobuf - یک [بافر پروتکل](https://developers.google.com/protocol-buffers/) قالب. - -این فرمت نیاز به یک طرح فرمت خارجی دارد. طرح بین نمایش داده شد ذخیره سازی. -تاتر از هر دو `proto2` و `proto3` syntaxes. تکرار / اختیاری / زمینه های مورد نیاز پشتیبانی می شوند. - -نمونه های استفاده: - -``` sql -SELECT * FROM test.table FORMAT Protobuf SETTINGS format_schema = 'schemafile:MessageType' -``` - -``` bash -cat protobuf_messages.bin | clickhouse-client --query "INSERT INTO test.table FORMAT Protobuf SETTINGS format_schema='schemafile:MessageType'" -``` - -جایی که فایل `schemafile.proto` به نظر می رسد مثل این: - -``` capnp -syntax = "proto3"; - -message MessageType { - string name = 1; - string surname = 2; - uint32 birthDate = 3; - repeated string phoneNumbers = 4; -}; -``` - -برای پیدا کردن مکاتبات بین ستون های جدول و زمینه های بافر پروتکل' نوع پیام تاتر نام خود را مقایسه می کند. -این مقایسه غیر حساس به حروف و شخصیت است `_` هشدار داده می شود `.` (نقطه) به عنوان برابر در نظر گرفته. -اگر نوع ستون و زمینه پیام بافر پروتکل متفاوت تبدیل لازم اعمال می شود. - -پیام های تو در تو پشتیبانی می شوند. برای مثال برای این زمینه است `z` در نوع پیام زیر - -``` capnp -message MessageType { - message XType { - message YType { - int32 z; - }; - repeated YType y; - }; - XType x; -}; -``` - -تاتر تلاش می کند برای پیدا کردن یک ستون به نام `x.y.z` (یا `x_y_z` یا `X.y_Z` و به همین ترتیب). -پیام های تو در تو مناسب برای ورودی یا خروجی هستند [ساختارهای داده تو در تو](../sql-reference/data-types/nested-data-structures/nested.md). - -مقادیر پیش فرض تعریف شده در یک طرح اولیه مانند این - -``` capnp -syntax = "proto2"; - -message MessageType { - optional int32 result_per_page = 3 [default = 10]; -} -``` - -اعمال نمی شود [پیشفرضهای جدول](../sql-reference/statements/create.md#create-default-values) به جای اونها استفاده میشه - -ClickHouse ورودی و خروجی protobuf پیام در `length-delimited` قالب. -این بدان معنی است قبل از هر پیام باید طول خود را به عنوان یک نوشته [ورینت](https://developers.google.com/protocol-buffers/docs/encoding#varints). -همچنین نگاه کنید به [چگونه به خواندن / نوشتن طول-حد و مرز مشخصی پیام های ورودی به زبان های محبوب](https://cwiki.apache.org/confluence/display/GEODE/Delimiting+Protobuf+Messages). - -## اورو {#data-format-avro} - -[هشدار داده می شود](http://avro.apache.org/) یک چارچوب سریال داده ردیف گرا توسعه یافته در درون پروژه هادوپ خود نمایی است. - -قالب کلیک اور پشتیبانی از خواندن و نوشتن [پروندههای داد و ستد](http://avro.apache.org/docs/current/spec.html#Object+Container+Files). - -### تطبیق انواع داده ها {#data_types-matching} - -جدول زیر انواع داده های پشتیبانی شده را نشان می دهد و چگونه با کلیک مطابقت دارند [انواع داده ها](../sql-reference/data-types/index.md) داخل `INSERT` و `SELECT` نمایش داده شد. - -| نوع داده اورو `INSERT` | نوع داده کلیک | نوع داده اورو `SELECT` | -|---------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|------------------------------| -| `boolean`, `int`, `long`, `float`, `double` | [اعضای هیات(8/16/32)](../sql-reference/data-types/int-uint.md), [اوینت (8/16/32)](../sql-reference/data-types/int-uint.md) | `int` | -| `boolean`, `int`, `long`, `float`, `double` | [Int64](../sql-reference/data-types/int-uint.md), [UInt64](../sql-reference/data-types/int-uint.md) | `long` | -| `boolean`, `int`, `long`, `float`, `double` | [Float32](../sql-reference/data-types/float.md) | `float` | -| `boolean`, `int`, `long`, `float`, `double` | [جسم شناور64](../sql-reference/data-types/float.md) | `double` | -| `bytes`, `string`, `fixed`, `enum` | [رشته](../sql-reference/data-types/string.md) | `bytes` | -| `bytes`, `string`, `fixed` | [رشته ثابت)](../sql-reference/data-types/fixedstring.md) | `fixed(N)` | -| `enum` | [شمارشی (8/16)](../sql-reference/data-types/enum.md) | `enum` | -| `array(T)` | [& توری)](../sql-reference/data-types/array.md) | `array(T)` | -| `union(null, T)`, `union(T, null)` | [Nullable(T)](../sql-reference/data-types/date.md) | `union(null, T)` | -| `null` | [Nullable(هیچ چیز)](../sql-reference/data-types/special-data-types/nothing.md) | `null` | -| `int (date)` \* | [تاریخ](../sql-reference/data-types/date.md) | `int (date)` \* | -| `long (timestamp-millis)` \* | [طول تاریخ 64 (3)](../sql-reference/data-types/datetime.md) | `long (timestamp-millis)` \* | -| `long (timestamp-micros)` \* | [طول تاریخ 64 (6)](../sql-reference/data-types/datetime.md) | `long (timestamp-micros)` \* | - -\* [انواع منطقی اورو](http://avro.apache.org/docs/current/spec.html#Logical+Types) - -انواع داده های ورودی پشتیبانی نشده: `record` (غیر ریشه), `map` - -انواع داده های منطقی پشتیبانی نشده: `uuid`, `time-millis`, `time-micros`, `duration` - -### درج داده {#inserting-data-1} - -برای وارد کردن داده ها از یک فایل اورو به جدول کلیک کنید: - -``` bash -$ cat file.avro | clickhouse-client --query="INSERT INTO {some_table} FORMAT Avro" -``` - -طرح ریشه فایل ورودی ورودی باید باشد `record` نوع. - -برای پیدا کردن مکاتبات بین ستون های جدول و زمینه های اور طرحواره کلیک نام خود را مقایسه می کند. این مقایسه حساس به حروف است. -زمینه های استفاده نشده قلم می شوند. - -انواع داده ها از ستون جدول کلیک می توانید از زمینه های مربوطه را از داده های اور قرار داده متفاوت است. در هنگام قرار دادن داده ها, تاتر تفسیر انواع داده ها با توجه به جدول بالا و سپس [کست](../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) داده ها به نوع ستون مربوطه. - -### انتخاب داده ها {#selecting-data-1} - -برای انتخاب داده ها از جدول کلیک به یک فایل پیشرو: - -``` bash -$ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Avro" > file.avro -``` - -نام ستون باید: - -- شروع با `[A-Za-z_]` -- متعاقبا تنها حاوی `[A-Za-z0-9_]` - -خروجی فشرده سازی فایل و فاصله همگام سازی را می توان با پیکربندی [_فرماندگی لبه بام](../operations/settings/settings.md#settings-output_format_avro_codec) و [_فرماندگی لبه چشم](../operations/settings/settings.md#settings-output_format_avro_sync_interval) به ترتیب. - -## هشدار داده می شود {#data-format-avro-confluent} - -ارتباط با پشتیبانی از رمز گشایی پیام های تک شی اور معمولا با استفاده می شود [کافکا](https://kafka.apache.org/) و [رجیستری طرح پساب](https://docs.confluent.io/current/schema-registry/index.html). - -هر پیام ایسترو جاسازی یک شناسه طرح است که می تواند به طرح واقعی با کمک رجیستری طرح حل و فصل شود. - -طرحواره ذخیره سازی یک بار حل شود. - -نشانی وب رجیستری طرحواره با پیکربندی [باز کردن _نمایش مجدد](../operations/settings/settings.md#settings-format_avro_schema_registry_url) - -### تطبیق انواع داده ها {#data_types-matching-1} - -مثل [اورو](#data-format-avro) - -### استفاده {#usage} - -به سرعت بررسی طرح قطعنامه شما می توانید استفاده کنید [کفککت](https://github.com/edenhill/kafkacat) با [کلیک-محلی](../operations/utilities/clickhouse-local.md): - -``` bash -$ kafkacat -b kafka-broker -C -t topic1 -o beginning -f '%s' -c 3 | clickhouse-local --input-format AvroConfluent --format_avro_schema_registry_url 'http://schema-registry' -S "field1 Int64, field2 String" -q 'select * from table' -1 a -2 b -3 c -``` - -برای استفاده `AvroConfluent` با [کافکا](../engines/table-engines/integrations/kafka.md): - -``` sql -CREATE TABLE topic1_stream -( - field1 String, - field2 String -) -ENGINE = Kafka() -SETTINGS -kafka_broker_list = 'kafka-broker', -kafka_topic_list = 'topic1', -kafka_group_name = 'group1', -kafka_format = 'AvroConfluent'; - -SET format_avro_schema_registry_url = 'http://schema-registry'; - -SELECT * FROM topic1_stream; -``` - -!!! note "اخطار" - تنظیم `format_avro_schema_registry_url` نیاز به پیکربندی در `users.xml` برای حفظ ارزش پس از راه اندازی مجدد. - -## پارکت {#data-format-parquet} - -[پارکت چوب کمر درد](http://parquet.apache.org/) فرمت ذخیره سازی ستونی گسترده در اکوسیستم هادوپ است. تاتر پشتیبانی خواندن و نوشتن عملیات برای این فرمت. - -### تطبیق انواع داده ها {#data_types-matching-2} - -جدول زیر انواع داده های پشتیبانی شده را نشان می دهد و چگونه با کلیک مطابقت دارند [انواع داده ها](../sql-reference/data-types/index.md) داخل `INSERT` و `SELECT` نمایش داده شد. - -| نوع داده پارکت (`INSERT`) | نوع داده کلیک | نوع داده پارکت (`SELECT`) | -|---------------------------|---------------------------------------------------------|---------------------------| -| `UINT8`, `BOOL` | [UInt8](../sql-reference/data-types/int-uint.md) | `UINT8` | -| `INT8` | [Int8](../sql-reference/data-types/int-uint.md) | `INT8` | -| `UINT16` | [UInt16](../sql-reference/data-types/int-uint.md) | `UINT16` | -| `INT16` | [Int16](../sql-reference/data-types/int-uint.md) | `INT16` | -| `UINT32` | [UInt32](../sql-reference/data-types/int-uint.md) | `UINT32` | -| `INT32` | [Int32](../sql-reference/data-types/int-uint.md) | `INT32` | -| `UINT64` | [UInt64](../sql-reference/data-types/int-uint.md) | `UINT64` | -| `INT64` | [Int64](../sql-reference/data-types/int-uint.md) | `INT64` | -| `FLOAT`, `HALF_FLOAT` | [Float32](../sql-reference/data-types/float.md) | `FLOAT` | -| `DOUBLE` | [جسم شناور64](../sql-reference/data-types/float.md) | `DOUBLE` | -| `DATE32` | [تاریخ](../sql-reference/data-types/date.md) | `UINT16` | -| `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | -| `STRING`, `BINARY` | [رشته](../sql-reference/data-types/string.md) | `STRING` | -| — | [رشته ثابت](../sql-reference/data-types/fixedstring.md) | `STRING` | -| `DECIMAL` | [دهدهی](../sql-reference/data-types/decimal.md) | `DECIMAL` | - -کلیک هاوس از دقت قابل تنظیم پشتیبانی می کند `Decimal` نوع. این `INSERT` پرس و جو رفتار پارکت `DECIMAL` نوع به عنوان محل کلیک `Decimal128` نوع. - -انواع داده های پارکت پشتیبانی نشده: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. - -انواع داده ها از ستون جدول کلیک خانه می تواند از زمینه های مربوطه را از داده پارکت قرار داده متفاوت است. در هنگام قرار دادن داده ها, تاتر تفسیر انواع داده ها با توجه به جدول بالا و سپس [بازیگران](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) داده ها به این نوع داده است که برای ستون جدول کلیک مجموعه. - -### درج و انتخاب داده ها {#inserting-and-selecting-data} - -شما می توانید داده های پارکت از یک فایل را به جدول کلیک توسط دستور زیر وارد کنید: - -``` bash -$ cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT Parquet" -``` - -شما می توانید داده ها را از یک جدول کلیک انتخاب کنید و با دستور زیر به برخی از فایل ها در قالب پارکت ذخیره کنید: - -``` bash -$ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_file.pq} -``` - -برای تبادل اطلاعات با هادوپ, شما می توانید استفاده کنید [موتور جدول اچ دی اف](../engines/table-engines/integrations/hdfs.md). - -## ORC {#data-format-orc} - -[آپاچی ORC](https://orc.apache.org/) فرمت ذخیره سازی ستونی گسترده در اکوسیستم هادوپ است. شما فقط می توانید داده ها را در این قالب به کلیک کنید. - -### تطبیق انواع داده ها {#data_types-matching-3} - -جدول زیر انواع داده های پشتیبانی شده را نشان می دهد و چگونه با کلیک مطابقت دارند [انواع داده ها](../sql-reference/data-types/index.md) داخل `INSERT` نمایش داده شد. - -| نوع داده اورک (`INSERT`) | نوع داده کلیک | -|--------------------------|-----------------------------------------------------| -| `UINT8`, `BOOL` | [UInt8](../sql-reference/data-types/int-uint.md) | -| `INT8` | [Int8](../sql-reference/data-types/int-uint.md) | -| `UINT16` | [UInt16](../sql-reference/data-types/int-uint.md) | -| `INT16` | [Int16](../sql-reference/data-types/int-uint.md) | -| `UINT32` | [UInt32](../sql-reference/data-types/int-uint.md) | -| `INT32` | [Int32](../sql-reference/data-types/int-uint.md) | -| `UINT64` | [UInt64](../sql-reference/data-types/int-uint.md) | -| `INT64` | [Int64](../sql-reference/data-types/int-uint.md) | -| `FLOAT`, `HALF_FLOAT` | [Float32](../sql-reference/data-types/float.md) | -| `DOUBLE` | [جسم شناور64](../sql-reference/data-types/float.md) | -| `DATE32` | [تاریخ](../sql-reference/data-types/date.md) | -| `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | -| `STRING`, `BINARY` | [رشته](../sql-reference/data-types/string.md) | -| `DECIMAL` | [دهدهی](../sql-reference/data-types/decimal.md) | - -تاتر از دقت قابل تنظیم از `Decimal` نوع. این `INSERT` پرس و جو رفتار اورک `DECIMAL` نوع به عنوان محل کلیک `Decimal128` نوع. - -انواع داده های پشتیبانی نشده: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. - -انواع داده ها از ستون جدول کلیک هاوس لازم نیست برای مطابقت با زمینه های داده اورک مربوطه. در هنگام قرار دادن داده ها, تاتر تفسیر انواع داده ها با توجه به جدول بالا و سپس [کست](../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) داده ها به نوع داده تعیین شده برای ستون جدول کلیک. - -### درج داده {#inserting-data-2} - -شما می توانید داده اورک از یک فایل به جدول کلیک توسط دستور زیر وارد کنید: - -``` bash -$ cat filename.orc | clickhouse-client --query="INSERT INTO some_table FORMAT ORC" -``` - -برای تبادل اطلاعات با هادوپ, شما می توانید استفاده کنید [موتور جدول اچ دی اف](../engines/table-engines/integrations/hdfs.md). - -## شمای فرمت {#formatschema} - -نام پرونده حاوی شمای قالب با تنظیم تنظیم می شود `format_schema`. -لازم است این تنظیم را هنگام استفاده از یکی از فرمت ها تنظیم کنید `Cap'n Proto` و `Protobuf`. -شمای فرمت ترکیبی از یک نام فایل و نام یک نوع پیام در این فایل است که توسط روده بزرگ جدا شده است, -e.g. `schemafile.proto:MessageType`. -اگر فایل دارای فرمت استاندارد برای فرمت (به عنوان مثال, `.proto` برای `Protobuf`), -این را می توان حذف و در این مورد طرح فرمت به نظر می رسد `schemafile:MessageType`. - -اگر داده های ورودی یا خروجی را از طریق [کارگیر](../interfaces/cli.md) در [حالت تعاملی](../interfaces/cli.md#cli_usage), نام پرونده مشخص شده در شمای فرمت -می تواند شامل یک مسیر مطلق و یا یک مسیر نسبت به دایرکتوری جاری در مشتری. -اگر شما با استفاده از مشتری در [حالت دسته ای](../interfaces/cli.md#cli_usage) مسیر طرح باید به دلایل امنیتی نسبی باشد. - -اگر داده های ورودی یا خروجی را از طریق [رابط قام](../interfaces/http.md) نام پرونده مشخص شده در شمای قالب -باید در دایرکتوری مشخص شده در واقع [قالب_شکلمات شیمی](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-format_schema_path) -در پیکربندی سرور. - -## پرش خطاها {#skippingerrors} - -برخی از فرمت های مانند `CSV`, `TabSeparated`, `TSKV`, `JSONEachRow`, `Template`, `CustomSeparated` و `Protobuf` می توانید ردیف شکسته جست و خیز اگر خطای تجزیه رخ داده است و ادامه تجزیه از ابتدای ردیف بعدی. ببینید [وارد کردن _فرست_مرزیابی _نمایش مجدد](../operations/settings/settings.md#settings-input_format_allow_errors_num) و -[ثبت نام](../operations/settings/settings.md#settings-input_format_allow_errors_ratio) تنظیمات. -محدودیت ها: -- در صورت خطای تجزیه `JSONEachRow` پرش تمام داده ها تا زمانی که خط جدید (یا بخش ویژه), بنابراین ردیف باید توسط حد و مرز مشخصی `\n` برای شمارش خطاها به درستی. -- `Template` و `CustomSeparated` استفاده از جداساز پس از ستون گذشته و جداساز بین ردیف برای پیدا کردن ابتدای سطر بعدی, بنابراین اشتباهات پرش کار می کند تنها اگر حداقل یکی از خالی نیست. - -[مقاله اصلی](https://clickhouse.tech/docs/en/interfaces/formats/) diff --git a/docs/fa/interfaces/http.md b/docs/fa/interfaces/http.md deleted file mode 100644 index 16d7aa390dc..00000000000 --- a/docs/fa/interfaces/http.md +++ /dev/null @@ -1,617 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 19 -toc_title: "\u0631\u0627\u0628\u0637 \u0642\u0627\u0645" ---- - -# رابط قام {#http-interface} - -رابط اچ تی پی به شما امکان استفاده از کلیک بر روی هر پلت فرم از هر زبان برنامه نویسی. ما برای کار از جاوا و پرل و همچنین اسکریپت های پوسته استفاده می کنیم. در بخش های دیگر, رابط قام است از پرل استفاده, پایتون, و رفتن. رابط قام محدود تر از رابط بومی است, اما سازگاری بهتر. - -به طور پیش فرض, کلیک سرور گوش برای اچ تی پی در بندر 8123 (این را می توان در پیکربندی تغییر). - -اگر شما یک دریافت / درخواست بدون پارامتر, باز می گردد 200 کد پاسخ و رشته که در تعریف [نقلقولهای جدید از این نویسنده](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-http_server_default_response) مقدار پیشفرض “Ok.” (با خوراک خط در پایان) - -``` bash -$ curl 'http://localhost:8123/' -Ok. -``` - -استفاده از دریافت / درخواست پینگ در بهداشت و درمان چک اسکریپت. این کنترل همیشه باز می گردد “Ok.” (با یک خوراک خط در پایان). موجود از نسخه 18.12.13. - -``` bash -$ curl 'http://localhost:8123/ping' -Ok. -``` - -ارسال درخواست به عنوان نشانی وب ‘query’ پارامتر, و یا به عنوان یک پست. و یا ارسال ابتدای پرس و جو در ‘query’ پارامتر, و بقیه در پست (بعدا توضیح خواهیم داد که چرا این لازم است). اندازه نشانی اینترنتی محدود به 16 کیلوبایت است بنابراین این را در نظر داشته باشید در هنگام ارسال نمایش داده شد بزرگ. - -اگر موفق, شما دریافت 200 کد پاسخ و در نتیجه در بدن پاسخ. -اگر یک خطا رخ می دهد, شما در دریافت 500 کد پاسخ و یک متن شرح خطا در بدن پاسخ. - -هنگام استفاده از روش دریافت, ‘readonly’ قرار است. به عبارت دیگر برای نمایش داده شد که تغییر داده ها شما فقط می توانید با استفاده از روش پست. شما می توانید پرس و جو خود را در قسمت پست یا در پارامتر نشانی وب ارسال کنید. - -مثالها: - -``` bash -$ curl 'http://localhost:8123/?query=SELECT%201' -1 - -$ wget -nv -O- 'http://localhost:8123/?query=SELECT 1' -1 - -$ echo -ne 'GET /?query=SELECT%201 HTTP/1.0\r\n\r\n' | nc localhost 8123 -HTTP/1.0 200 OK -Date: Wed, 27 Nov 2019 10:30:18 GMT -Connection: Close -Content-Type: text/tab-separated-values; charset=UTF-8 -X-ClickHouse-Server-Display-Name: clickhouse.ru-central1.internal -X-ClickHouse-Query-Id: 5abe861c-239c-467f-b955-8a201abb8b7f -X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} - -1 -``` - -همانطور که می بینید, حلقه تا حدودی ناخوشایند است که در فضاهای باید نشانی اینترنتی فرار. -اگر چه سازمان تجارت جهانی از همه چیز خود فرار می کند ما توصیه نمی کنیم از این استفاده کنیم زیرا هنگام استفاده از زنده ماندن و انتقال رمزگذاری به خوبی کار نمی کند 1.1. - -``` bash -$ echo 'SELECT 1' | curl 'http://localhost:8123/' --data-binary @- -1 - -$ echo 'SELECT 1' | curl 'http://localhost:8123/?query=' --data-binary @- -1 - -$ echo '1' | curl 'http://localhost:8123/?query=SELECT' --data-binary @- -1 -``` - -اگر بخشی از پرس و جو در پارامتر ارسال, و بخشی در پست, خوراک خط بین این دو بخش داده قرار داده. -مثال (این کار نخواهد کرد): - -``` bash -$ echo 'ECT 1' | curl 'http://localhost:8123/?query=SEL' --data-binary @- -Code: 59, e.displayText() = DB::Exception: Syntax error: failed at position 0: SEL -ECT 1 -, expected One of: SHOW TABLES, SHOW DATABASES, SELECT, INSERT, CREATE, ATTACH, RENAME, DROP, DETACH, USE, SET, OPTIMIZE., e.what() = DB::Exception -``` - -به طور پیش فرض, داده ها در قالب جدولبندی بازگشت (برای اطلاعات بیشتر, دیدن “Formats” بخش). -شما با استفاده از بند فرمت پرس و جو به درخواست هر فرمت دیگر. - -``` bash -$ echo 'SELECT 1 FORMAT Pretty' | curl 'http://localhost:8123/?' --data-binary @- -┏━━━┓ -┃ 1 ┃ -┡━━━┩ -│ 1 │ -└───┘ -``` - -روش پست انتقال داده ها برای درج نمایش داده شد لازم است. در این مورد می توانید ابتدا پرس و جو را در پارامتر نشانی وب بنویسید و از پست برای انتقال داده ها برای وارد کردن استفاده کنید. داده ها برای وارد کردن می تواند, مثلا, تخلیه تب جدا از خروجی زیر. در این راه وارد کردن پرس و جو جایگزین بارگذاری داده های محلی INFILE از MySQL. - -نمونه: ایجاد یک جدول: - -``` bash -$ echo 'CREATE TABLE t (a UInt8) ENGINE = Memory' | curl 'http://localhost:8123/' --data-binary @- -``` - -با استفاده از قرار دادن پرس و جو برای درج داده ها: - -``` bash -$ echo 'INSERT INTO t VALUES (1),(2),(3)' | curl 'http://localhost:8123/' --data-binary @- -``` - -داده ها را می توان به طور جداگانه از پرس و جو ارسال می شود: - -``` bash -$ echo '(4),(5),(6)' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20VALUES' --data-binary @- -``` - -شما می توانید هر فرمت داده را مشخص کنید. این ‘Values’ فرمت همان چیزی است که هنگام نوشتن به مقادیر تی استفاده می شود: - -``` bash -$ echo '(7),(8),(9)' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20FORMAT%20Values' --data-binary @- -``` - -برای وارد کردن داده ها از تخلیه زبانه جدا شده فرمت مربوطه را مشخص کنید: - -``` bash -$ echo -ne '10\n11\n12\n' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20FORMAT%20TabSeparated' --data-binary @- -``` - -خواندن محتویات جدول. داده ها خروجی به صورت تصادفی به دلیل پردازش پرس و جو موازی است: - -``` bash -$ curl 'http://localhost:8123/?query=SELECT%20a%20FROM%20t' -7 -8 -9 -10 -11 -12 -1 -2 -3 -4 -5 -6 -``` - -حذف جدول. - -``` bash -$ echo 'DROP TABLE t' | curl 'http://localhost:8123/' --data-binary @- -``` - -برای درخواست موفق است که یک جدول داده ها بازگشت نیست, بدن پاسخ خالی بازگشته است. - -شما می توانید فرمت فشرده سازی کلیک داخلی در هنگام انتقال داده ها استفاده کنید. داده های فشرده دارای فرمت غیر استاندارد است و شما باید از ویژه استفاده کنید `clickhouse-compressor` برنامه ای برای کار با ان (با ان نصب شده است `clickhouse-client` بسته). برای افزایش بهره وری از درج داده, شما می توانید سرور سمت تایید کنترلی با استفاده از غیر فعال کردن [تغییر در حسابهای کاربری دستگاه](../operations/settings/settings.md#settings-http_native_compression_disable_checksumming_on_decompress) تنظیمات. - -اگر شما مشخص `compress=1` در نشانی وب سرور دادههای ارسالی شما را فشرده میکند. -اگر شما مشخص `decompress=1` در نشانی اینترنتی کارگزار دادههای مشابهی را که در `POST` روش. - -شما همچنین می توانید استفاده کنید را انتخاب کنید [فشردهسازی قام](https://en.wikipedia.org/wiki/HTTP_compression). برای ارسال یک فشرده `POST` درخواست, اضافه هدر درخواست `Content-Encoding: compression_method`. به منظور کلیک برای فشرده سازی پاسخ, شما باید اضافه `Accept-Encoding: compression_method`. پشتیبانی از کلیک `gzip`, `br` و `deflate` [روش های فشرده سازی](https://en.wikipedia.org/wiki/HTTP_compression#Content-Encoding_tokens). برای فعال کردن فشرده سازی قام, شما باید از خانه کلیک استفاده [نصب و راه اندازی](../operations/settings/settings.md#settings-enable_http_compression) تنظیمات. شما می توانید سطح فشرده سازی داده ها در پیکربندی [_تنظیم مجدد به حالت اولیه](#settings-http_zlib_compression_level) تنظیم برای تمام روش های فشرده سازی. - -شما می توانید این برای کاهش ترافیک شبکه در هنگام انتقال مقدار زیادی از داده ها و یا برای ایجاد افسردگی است که بلافاصله فشرده استفاده کنید. - -نمونه هایی از ارسال داده ها با فشرده سازی: - -``` bash -#Sending data to the server: -$ curl -vsS "http://localhost:8123/?enable_http_compression=1" -d 'SELECT number FROM system.numbers LIMIT 10' -H 'Accept-Encoding: gzip' - -#Sending data to the client: -$ echo "SELECT 1" | gzip -c | curl -sS --data-binary @- -H 'Content-Encoding: gzip' 'http://localhost:8123/' -``` - -!!! note "یادداشت" - برخی از مشتریان اچ تی پی ممکن است داده ها را از حالت فشرده خارج از سرور به طور پیش فرض (با `gzip` و `deflate`) و شما ممکن است داده ها از حالت فشرده خارج حتی اگر شما با استفاده از تنظیمات فشرده سازی به درستی. - -شما می توانید از ‘database’ پارامتر نشانی وب برای مشخص کردن پایگاه داده به طور پیش فرض. - -``` bash -$ echo 'SELECT number FROM numbers LIMIT 10' | curl 'http://localhost:8123/?database=system' --data-binary @- -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -``` - -به طور پیش فرض, پایگاه داده است که در تنظیمات سرور ثبت نام به عنوان پایگاه داده به طور پیش فرض استفاده. به طور پیش فرض, این پایگاه داده به نام است ‘default’. متناوبا, شما همیشه می توانید مشخص کردن پایگاه داده با استفاده از یک نقطه قبل از نام جدول. - -نام کاربری و رمز عبور را می توان در یکی از سه راه نشان داد: - -1. با استفاده از احراز هویت اولیه. مثال: - - - -``` bash -$ echo 'SELECT 1' | curl 'http://user:password@localhost:8123/' -d @- -``` - -1. در ‘user’ و ‘password’ پارامترهای نشانی وب. مثال: - - - -``` bash -$ echo 'SELECT 1' | curl 'http://localhost:8123/?user=user&password=password' -d @- -``` - -1. با استفاده از ‘X-ClickHouse-User’ و ‘X-ClickHouse-Key’ سرصفحهها. مثال: - - - -``` bash -$ echo 'SELECT 1' | curl -H 'X-ClickHouse-User: user' -H 'X-ClickHouse-Key: password' 'http://localhost:8123/' -d @- -``` - -اگر نام کاربر مشخص نشده است `default` نام استفاده شده است. اگر رمز عبور مشخص نشده است, رمز عبور خالی استفاده شده است. -شما همچنین می توانید از پارامترهای نشانی وب برای مشخص کردن هر گونه تنظیمات برای پردازش یک پرس و جو یا کل پروفایل های تنظیمات استفاده کنید. هشدار داده می شودمشخصات=وب و حداکثر_نظیم = 1000000000 & پرس و جو = انتخاب+1 - -برای کسب اطلاعات بیشتر, دیدن [تنظیمات](../operations/settings/index.md) بخش. - -``` bash -$ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:8123/?' --data-binary @- -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -``` - -برای اطلاعات در مورد پارامترهای دیگر, بخش را ببینید “SET”. - -به طور مشابه, شما می توانید جلسات کلیک در پروتکل قام استفاده. برای انجام این کار, شما نیاز به اضافه کردن `session_id` دریافت پارامتر به درخواست. شما می توانید هر رشته به عنوان شناسه جلسه استفاده کنید. به طور پیش فرض جلسه پس از 60 ثانیه عدم فعالیت خاتمه می یابد. برای تغییر این فاصله, تغییر `default_session_timeout` تنظیم در پیکربندی سرور یا اضافه کردن `session_timeout` دریافت پارامتر به درخواست. برای بررسی وضعیت جلسه از `session_check=1` پارامتر. فقط یک پرس و جو در یک زمان می تواند در یک جلسه اجرا شود. - -شما می توانید اطلاعات در مورد پیشرفت یک پرس و جو در دریافت `X-ClickHouse-Progress` هدر پاسخ. برای انجام این کار, فعال کردن [نمایش سایت](../operations/settings/settings.md#settings-send_progress_in_http_headers). مثال توالی هدر: - -``` text -X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128"} -X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128"} -X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128"} -``` - -زمینه های سربرگ احتمالی: - -- `read_rows` — Number of rows read. -- `read_bytes` — Volume of data read in bytes. -- `total_rows_to_read` — Total number of rows to be read. -- `written_rows` — Number of rows written. -- `written_bytes` — Volume of data written in bytes. - -درخواست های در حال اجرا به طور خودکار متوقف نمی شود اگر اتصال قام از دست داده است. تجزیه و قالب بندی داده ها در سمت سرور انجام, و با استفاده از شبکه ممکن است بی اثر. -اختیاری ‘query_id’ پارامتر را می توان به عنوان شناسه پرس و جو (هر رشته) منتقل می شود. برای کسب اطلاعات بیشتر به بخش مراجعه کنید “Settings, replace_running_query”. - -اختیاری ‘quota_key’ پارامتر را می توان به عنوان کلید سهمیه (هر رشته) منتقل می شود. برای کسب اطلاعات بیشتر به بخش مراجعه کنید “Quotas”. - -رابط اچ تی پی اجازه می دهد تا عبور داده های خارجی (جداول موقت خارجی) برای پرس و جو. برای کسب اطلاعات بیشتر به بخش مراجعه کنید “External data for query processing”. - -## بافر پاسخ {#response-buffering} - -شما می توانید پاسخ بافر در سمت سرور را فعال کنید. این `buffer_size` و `wait_end_of_query` پارامترهای نشانی وب برای این منظور فراهم شده است. - -`buffer_size` تعیین تعداد بایت در نتیجه به بافر در حافظه سرور. اگر یک بدن نتیجه بزرگتر از این حد است, بافر به کانال قام نوشته شده, و داده های باقی مانده به طور مستقیم به کانال قام ارسال. - -برای اطمینان از اینکه کل پاسخ بافر شده است, تنظیم `wait_end_of_query=1`. در این مورد داده هایی که در حافظه ذخیره نمی شوند در یک فایل سرور موقت بافر می شوند. - -مثال: - -``` bash -$ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wait_end_of_query=1' -d 'SELECT toUInt8(number) FROM system.numbers LIMIT 9000000 FORMAT RowBinary' -``` - -استفاده از بافر برای جلوگیری از شرایطی که یک خطای پردازش پرس و جو رخ داده است پس از کد پاسخ و هدر قام به مشتری ارسال شد. در این وضعیت یک پیغام خطا نوشته شده است در پایان پاسخ بدن و در سمت سرویس گیرنده خطا تنها می تواند تشخیص داده شده در مرحله تجزیه. - -### نمایش داده شد با پارامترهای {#cli-queries-with-parameters} - -شما می توانید پرس و جو را با پارامترها ایجاد کنید و مقادیر را از پارامترهای درخواست مربوط به اچ تی پی منتقل کنید. برای کسب اطلاعات بیشتر, دیدن [نمایش داده شد با پارامترهای کلی](cli.md#cli-queries-with-parameters). - -### مثال {#example} - -``` bash -$ curl -sS "
?param_id=2¶m_phrase=test" -d "SELECT * FROM table WHERE int_column = {id:UInt8} and string_column = {phrase:String}" -``` - -## حذف میانبر در صفحه خانه {#predefined_http_interface} - -تاتر پشتیبانی از نمایش داده شد خاص از طریق رابط قام. مثلا, شما می توانید داده ها را به یک جدول به شرح زیر ارسال: - -``` bash -$ echo '(4),(5),(6)' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20VALUES' --data-binary @- -``` - -کلیک هاوس همچنین از رابط اچ تی پی از پیش تعریف شده پشتیبانی می کند که می تواند به شما در ادغام راحت تر با ابزارهای شخص ثالث مانند کمک کند [پرومته صادر کننده](https://github.com/percona-lab/clickhouse_exporter). - -مثال: - -- اول از همه, اضافه کردن این بخش به فایل پیکربندی سرور: - - - -``` xml - - - /predefined_query - POST,GET - - predefined_query_handler - SELECT * FROM system.metrics LIMIT 5 FORMAT Template SETTINGS format_template_resultset = 'prometheus_template_output_format_resultset', format_template_row = 'prometheus_template_output_format_row', format_template_rows_between_delimiter = '\n' - - - ... - ... - -``` - -- شما هم اکنون می توانید لینک را مستقیما برای داده ها در قالب پرومته درخواست کنید: - - - -``` bash -$ curl -v 'http://localhost:8123/predefined_query' -* Trying ::1... -* Connected to localhost (::1) port 8123 (#0) -> GET /predefined_query HTTP/1.1 -> Host: localhost:8123 -> User-Agent: curl/7.47.0 -> Accept: */* -> -< HTTP/1.1 200 OK -< Date: Tue, 28 Apr 2020 08:52:56 GMT -< Connection: Keep-Alive -< Content-Type: text/plain; charset=UTF-8 -< X-ClickHouse-Server-Display-Name: i-mloy5trc -< Transfer-Encoding: chunked -< X-ClickHouse-Query-Id: 96fe0052-01e6-43ce-b12a-6b7370de6e8a -< X-ClickHouse-Format: Template -< X-ClickHouse-Timezone: Asia/Shanghai -< Keep-Alive: timeout=3 -< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< -# HELP "Query" "Number of executing queries" -# TYPE "Query" counter -"Query" 1 - -# HELP "Merge" "Number of executing background merges" -# TYPE "Merge" counter -"Merge" 0 - -# HELP "PartMutation" "Number of mutations (ALTER DELETE/UPDATE)" -# TYPE "PartMutation" counter -"PartMutation" 0 - -# HELP "ReplicatedFetch" "Number of data parts being fetched from replica" -# TYPE "ReplicatedFetch" counter -"ReplicatedFetch" 0 - -# HELP "ReplicatedSend" "Number of data parts being sent to replicas" -# TYPE "ReplicatedSend" counter -"ReplicatedSend" 0 - -* Connection #0 to host localhost left intact - - -* Connection #0 to host localhost left intact -``` - -همانطور که شما می توانید از مثال ببینید, اگر `` در پیکربندی پیکربندی پیکربندی شده است.فایل و `` می تواند شامل بسیاری از `s`. کلیک هاوس خواهد درخواست قام قام دریافت به نوع از پیش تعریف شده در مطابقت `` و اولین همسان اجرا می شود کنترل. سپس خانه را کلیک کنید پرس و جو از پیش تعریف شده مربوطه اجرا اگر بازی موفق است. - -> حالا `` می توانید پیکربندی کنید ``, ``, ``,``: -> `` برای تطبیق روش بخشی از درخواست قام است. `` به طور کامل مطابق با تعریف [روش](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) در پروتکل قام. این پیکربندی اختیاری است. اگر در فایل پیکربندی تعریف نشده, این کار بخش روش درخواست قام مطابقت ندارد. -> -> `` وظیفه تطبیق بخشی نشانی وب از درخواست قام است. این سازگار با است [RE2](https://github.com/google/re2)عبارات منظم است. این پیکربندی اختیاری است. اگر در فایل پیکربندی تعریف نشده باشد با بخش نشانی وب درخواست قام مطابقت ندارد. -> -> `` برای تطبیق بخش هدر درخواست قام است. این است که سازگار با عبارات منظم را دوباره2 است. این پیکربندی اختیاری است. اگر در فایل پیکربندی تعریف نشده است, این کار بخش هدر درخواست قام مطابقت ندارد. -> -> `` شامل بخش پردازش اصلی. حالا `` می توانید پیکربندی کنید ``, ``, ``, ``, ``, ``. -> \> `` در حال حاضر پشتیبانی از سه نوع: **باز تعریف**, **هشدار داده می شود**, **ایستا**. -> \> -> \> `` - استفاده از با نوع بازتعریف_کرکی_ هندلر, اجرا پرس و جو زمانی که کنترل نامیده می شود. -> \> -> \> `` - استفاده با نوع داینامیک_کرکی_خندلر عصارهها و اجرا مقدار مربوط به `` ارزش در پارامترهای درخواست قام. -> \> -> \> `` - استفاده با نوع استاتیک, پاسخ کد وضعیت. -> \> -> \> `` - استفاده با نوع استاتیک پاسخ [نوع محتوا](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type). -> \> -> \> `` - استفاده با نوع استاتیک, محتوای پاسخ ارسال شده به مشتری, هنگام استفاده از پیشوند ‘file://’ یا ‘config://’, پیدا کردن محتوا از فایل و یا پیکربندی ارسال به مشتری. - -بعد روش پیکربندی برای متفاوت هستند ``. - -## باز تعریف {#predefined_query_handler} - -`` پشتیبانی از تنظیمات و مقادیر قوری_پرم. شما می توانید پیکربندی کنید `` در نوع ``. - -`` مقدار پرس و جو از پیش تعریف شده است ``, است که توسط کلیکهاوس اجرا زمانی که یک درخواست قام همسان است و در نتیجه از پرس و جو بازگشته است. این پیکربندی باید است. - -مثال زیر مقادیر را تعریف می کند `max_threads` و `max_alter_threads` تنظیمات, سپس نمایش داده شد جدول سیستم برای بررسی اینکه این تنظیمات با موفقیت تعیین شد. - -مثال: - -``` xml - - - [^/]+)(/(?P[^/]+))?]]> - GET - - TEST_HEADER_VALUE - [^/]+)(/(?P[^/]+))?]]> - - - predefined_query_handler - SELECT value FROM system.settings WHERE name = {name_1:String} - SELECT name, value FROM system.settings WHERE name = {name_2:String} - - - -``` - -``` bash -$ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost:8123/query_param_with_url/1/max_threads/max_alter_threads?max_threads=1&max_alter_threads=2' -1 -max_alter_threads 2 -``` - -!!! note "احتیاط" - در یک `` تنها پشتیبانی از یک `` از یک نوع درج. - -## هشدار داده می شود {#dynamic_query_handler} - -داخل ``, پرس و جو در قالب پرام از درخواست قام نوشته شده است. تفاوت این است که در ``, پرس و جو در فایل پیکربندی نوشت. شما می توانید پیکربندی کنید `` داخل ``. - -عصاره کلیک و اجرا ارزش مربوط به `` مقدار در نشانی وب درخواست قام. مقدار پیش فرض `` هست `/query` . این پیکربندی اختیاری است. در صورتی که هیچ تعریف در فایل پیکربندی وجود دارد, پرم در تصویب نشده است. - -برای آزمایش این قابلیت به عنوان مثال تعریف ارزش از max_threads و max_alter_threads و پرس و جو که آیا تنظیمات راه اندازی شد با موفقیت. - -مثال: - -``` xml - - - - TEST_HEADER_VALUE_DYNAMIC - - dynamic_query_handler - query_param - - - -``` - -``` bash -$ curl -H 'XXX:TEST_HEADER_VALUE_DYNAMIC' 'http://localhost:8123/own?max_threads=1&max_alter_threads=2¶m_name_1=max_threads¶m_name_2=max_alter_threads&query_param=SELECT%20name,value%20FROM%20system.settings%20where%20name%20=%20%7Bname_1:String%7D%20OR%20name%20=%20%7Bname_2:String%7D' -max_threads 1 -max_alter_threads 2 -``` - -## ایستا {#static} - -`` می توانید بازگشت [_نوع تماس](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type), [وضعیت](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) و پاسخ دهنده. پاسخ _حرکتکننده می تواند محتوای مشخص شده را بازگرداند - -مثال: - -بازگشت یک پیام. - -``` xml - - - GET - xxx - /hi - - static - 402 - text/html; charset=UTF-8 - Say Hi! - - - -``` - -``` bash -$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/hi' -* Trying ::1... -* Connected to localhost (::1) port 8123 (#0) -> GET /hi HTTP/1.1 -> Host: localhost:8123 -> User-Agent: curl/7.47.0 -> Accept: */* -> XXX:xxx -> -< HTTP/1.1 402 Payment Required -< Date: Wed, 29 Apr 2020 03:51:26 GMT -< Connection: Keep-Alive -< Content-Type: text/html; charset=UTF-8 -< Transfer-Encoding: chunked -< Keep-Alive: timeout=3 -< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< -* Connection #0 to host localhost left intact -Say Hi!% -``` - -پیدا کردن محتوا از پیکربندی ارسال به مشتری. - -``` xml -
]]> - - - - GET - xxx - /get_config_static_handler - - static - config://get_config_static_handler - - - -``` - -``` bash -$ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler' -* Trying ::1... -* Connected to localhost (::1) port 8123 (#0) -> GET /get_config_static_handler HTTP/1.1 -> Host: localhost:8123 -> User-Agent: curl/7.47.0 -> Accept: */* -> XXX:xxx -> -< HTTP/1.1 200 OK -< Date: Wed, 29 Apr 2020 04:01:24 GMT -< Connection: Keep-Alive -< Content-Type: text/plain; charset=UTF-8 -< Transfer-Encoding: chunked -< Keep-Alive: timeout=3 -< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< -* Connection #0 to host localhost left intact -
% -``` - -پیدا کردن محتوا از فایل ارسال به مشتری. - -``` xml - - - GET - xxx - /get_absolute_path_static_handler - - static - text/html; charset=UTF-8 - file:///absolute_path_file.html - - - - GET - xxx - /get_relative_path_static_handler - - static - text/html; charset=UTF-8 - file://./relative_path_file.html - - - -``` - -``` bash -$ user_files_path='/var/lib/clickhouse/user_files' -$ sudo echo "Relative Path File" > $user_files_path/relative_path_file.html -$ sudo echo "Absolute Path File" > $user_files_path/absolute_path_file.html -$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_absolute_path_static_handler' -* Trying ::1... -* Connected to localhost (::1) port 8123 (#0) -> GET /get_absolute_path_static_handler HTTP/1.1 -> Host: localhost:8123 -> User-Agent: curl/7.47.0 -> Accept: */* -> XXX:xxx -> -< HTTP/1.1 200 OK -< Date: Wed, 29 Apr 2020 04:18:16 GMT -< Connection: Keep-Alive -< Content-Type: text/html; charset=UTF-8 -< Transfer-Encoding: chunked -< Keep-Alive: timeout=3 -< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< -Absolute Path File -* Connection #0 to host localhost left intact -$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler' -* Trying ::1... -* Connected to localhost (::1) port 8123 (#0) -> GET /get_relative_path_static_handler HTTP/1.1 -> Host: localhost:8123 -> User-Agent: curl/7.47.0 -> Accept: */* -> XXX:xxx -> -< HTTP/1.1 200 OK -< Date: Wed, 29 Apr 2020 04:18:31 GMT -< Connection: Keep-Alive -< Content-Type: text/html; charset=UTF-8 -< Transfer-Encoding: chunked -< Keep-Alive: timeout=3 -< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< -Relative Path File -* Connection #0 to host localhost left intact -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/interfaces/http_interface/) diff --git a/docs/fa/interfaces/index.md b/docs/fa/interfaces/index.md deleted file mode 100644 index cf64a63b471..00000000000 --- a/docs/fa/interfaces/index.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0648\u0627\u0633\u0637" -toc_priority: 14 -toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" ---- - -# واسط {#interfaces} - -تاتر فراهم می کند دو رابط شبکه (هر دو می تواند به صورت اختیاری در ال اس برای امنیت بیشتر پیچیده): - -- [HTTP](http.md), که مستند شده است و به راحتی استفاده به طور مستقیم. -- [بومی TCP](tcp.md), که دارای سربار کمتر. - -در اغلب موارد توصیه می شود به استفاده از ابزار مناسب و یا کتابخانه به جای تعامل با کسانی که به طور مستقیم. به طور رسمی توسط یاندکس پشتیبانی به شرح زیر است: - -- [مشتری خط فرمان](cli.md) -- [JDBC driver](jdbc.md) -- [ODBC driver](odbc.md) -- [ج++ کتابخانه مشتری](cpp.md) - -همچنین طیف گسترده ای از کتابخانه های شخص ثالث برای کار با کلیک وجود دارد: - -- [کتابخانه های مشتری](third-party/client-libraries.md) -- [یکپارچگی](third-party/integrations.md) -- [رابط های بصری](third-party/gui.md) - -[مقاله اصلی](https://clickhouse.tech/docs/en/interfaces/) diff --git a/docs/fa/interfaces/jdbc.md b/docs/fa/interfaces/jdbc.md deleted file mode 100644 index 8af1efdaa89..00000000000 --- a/docs/fa/interfaces/jdbc.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 22 -toc_title: JDBC Driver ---- - -# JDBC Driver {#jdbc-driver} - -- **[راننده رسمی](https://github.com/ClickHouse/clickhouse-jdbc)** -- رانندگان شخص ثالث: - - [جستجو](https://github.com/housepower/ClickHouse-Native-JDBC) - - [کلیک کنیدهوش4ج](https://github.com/blynkkk/clickhouse4j) - -[مقاله اصلی](https://clickhouse.tech/docs/en/interfaces/jdbc/) diff --git a/docs/fa/interfaces/mysql.md b/docs/fa/interfaces/mysql.md deleted file mode 100644 index cb0e478d1a4..00000000000 --- a/docs/fa/interfaces/mysql.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 20 -toc_title: "\u0631\u0627\u0628\u0637 MySQL" ---- - -# رابط MySQL {#mysql-interface} - -کلیک پروتکل سیم خروجی زیر را پشتیبانی می کند. این را می توان با فعال [_وارد کردن](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-mysql_port) تنظیم در پرونده پیکربندی: - -``` xml -9004 -``` - -مثال اتصال با استفاده از ابزار خط فرمان `mysql`: - -``` bash -$ mysql --protocol tcp -u default -P 9004 -``` - -خروجی اگر اتصال موفق شد: - -``` text -Welcome to the MySQL monitor. Commands end with ; or \g. -Your MySQL connection id is 4 -Server version: 20.2.1.1-ClickHouse - -Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. - -Oracle is a registered trademark of Oracle Corporation and/or its -affiliates. Other names may be trademarks of their respective -owners. - -Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. - -mysql> -``` - -برای سازگاری با تمام مشتریان خروجی زیر, توصیه می شود برای مشخص کردن رمز عبور کاربر با [دو شی1](../operations/settings/settings-users.md#password_double_sha1_hex) در فایل پیکربندی. -اگر رمز عبور کاربر مشخص شده است با استفاده از [SHA256](../operations/settings/settings-users.md#password_sha256_hex) برخی از مشتریان قادر نخواهد بود به تصدیق (رمز و نسخه های قدیمی خط فرمان ابزار خروجی زیر). - -محدودیت ها: - -- نمایش داده شد تهیه پشتیبانی نمی شوند - -- برخی از انواع داده ها به عنوان رشته ارسال می شود - -[مقاله اصلی](https://clickhouse.tech/docs/en/interfaces/mysql/) diff --git a/docs/fa/interfaces/odbc.md b/docs/fa/interfaces/odbc.md deleted file mode 100644 index 80dd17842c5..00000000000 --- a/docs/fa/interfaces/odbc.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 23 -toc_title: ODBC Driver ---- - -# ODBC Driver {#odbc-driver} - -- [راننده رسمی](https://github.com/ClickHouse/clickhouse-odbc). - -[مقاله اصلی](https://clickhouse.tech/docs/en/interfaces/odbc/) diff --git a/docs/fa/interfaces/tcp.md b/docs/fa/interfaces/tcp.md deleted file mode 100644 index 165270978e0..00000000000 --- a/docs/fa/interfaces/tcp.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 18 -toc_title: "\u0631\u0627\u0628\u0637 \u06A9\u0627\u0631\u0628\u0631\u06CC \u0628\u0648\ - \u0645\u06CC)" ---- - -# رابط کاربری بومی) {#native-interface-tcp} - -پروتکل بومی در [مشتری خط فرمان](cli.md), برای ارتباط بین سرور در طول پردازش پرس و جو توزیع, و همچنین در دیگر ج++ برنامه. متاسفانه بومی ClickHouse پروتکل ندارد رسمی مشخصات رتبهدهی نشده است, اما می توان آن را مهندسی معکوس از ClickHouse کد منبع (شروع [اینجا](https://github.com/ClickHouse/ClickHouse/tree/master/src/Client)) و / یا با متوقف کردن و تجزیه و تحلیل ترافیک تی پی. - -[مقاله اصلی](https://clickhouse.tech/docs/en/interfaces/tcp/) diff --git a/docs/fa/interfaces/third-party/client-libraries.md b/docs/fa/interfaces/third-party/client-libraries.md deleted file mode 100644 index ad2d3e86aae..00000000000 --- a/docs/fa/interfaces/third-party/client-libraries.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 26 -toc_title: "\u06A9\u062A\u0627\u0628\u062E\u0627\u0646\u0647 \u0647\u0627\u06CC \u0645\ - \u0634\u062A\u0631\u06CC" ---- - -# کتابخانه های مشتری از توسعه دهندگان شخص ثالث {#client-libraries-from-third-party-developers} - -!!! warning "تکذیبنامهها" - یاندکس می کند **نه** حفظ کتابخانه های ذکر شده در زیر و هر تست گسترده برای اطمینان از کیفیت خود را انجام نداده اند. - -- پایتون - - [اطالعات.کلیک _شورم](https://github.com/Infinidat/infi.clickhouse_orm) - - [کلیک راننده](https://github.com/mymarilyn/clickhouse-driver) - - [کلیک مشتری](https://github.com/yurial/clickhouse-client) - - [اطلاعات دقیق](https://github.com/maximdanilchenko/aiochclient) -- PHP - - [اسمی2 / تلفن هوشمند](https://packagist.org/packages/smi2/phpClickHouse) - - [8bitov/clickhouse-php-مشتری](https://packagist.org/packages/8bitov/clickhouse-php-client) - - [برزرکینس / ناحیه کاربری](https://packagist.org/packages/bozerkins/clickhouse-client) - - [ساده / کلیک-کارفرما](https://packagist.org/packages/simpod/clickhouse-client) - - [سوا کد / پی اچ پی-کلیک-خانه مشتری](https://packagist.org/packages/seva-code/php-click-house-client) - - [دریازده ج++ مشتری](https://github.com/SeasX/SeasClick) -- برو - - [فاحشه خانه](https://github.com/kshvakov/clickhouse/) - - [برو کلیک هاوس](https://github.com/roistat/go-clickhouse) - - [میلروگو-کلیک هاوس](https://github.com/mailru/go-clickhouse) - - [گلنگ-خانه کلیک](https://github.com/leprosus/golang-clickhouse) -- نوکس - - [فاحشه خانه)](https://github.com/TimonKK/clickhouse) - - [گره کلیک هاوس](https://github.com/apla/node-clickhouse) -- پرل - - [perl-DBD-ClickHouse](https://github.com/elcamlost/perl-DBD-ClickHouse) - - [صفحه اصلی](https://metacpan.org/release/HTTP-ClickHouse) - - [هرفنت-کلیکهاوس](https://metacpan.org/release/AnyEvent-ClickHouse) -- روبی - - [تاتر (روبی)](https://github.com/shlima/click_house) - - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) -- R - - [کلیک تحقیق](https://github.com/hannesmuehleisen/clickhouse-r) - - [خانه روستایی](https://github.com/IMSMWU/RClickHouse) -- جاوا - - [فاحشه خانه-مشتری-جاوا](https://github.com/VirtusAI/clickhouse-client-java) - - [کلیک مشتری](https://github.com/Ecwid/clickhouse-client) -- اسکالا - - [تاتر-اسکالا-کارفرما](https://github.com/crobox/clickhouse-scala-client) -- کوتلین - - [AORM](https://github.com/TanVD/AORM) -- C# - - [Octonica.ClickHouseClient](https://github.com/Octonica/ClickHouseClient) - - [فاحشه خانه.ادو](https://github.com/killwort/ClickHouse-Net) - - [فاحشه خانه.کارگیر](https://github.com/DarkWanderer/ClickHouse.Client) - - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) -- اکسیر - - [کلیک](https://github.com/appodeal/clickhousex/) - - [ستون](https://github.com/sofakingworld/pillar) -- نیم - - [نیم کلیک خانه](https://github.com/leonardoce/nim-clickhouse) - -[مقاله اصلی](https://clickhouse.tech/docs/en/interfaces/third-party/client_libraries/) diff --git a/docs/fa/interfaces/third-party/gui.md b/docs/fa/interfaces/third-party/gui.md deleted file mode 100644 index 8f9a3ae9bc6..00000000000 --- a/docs/fa/interfaces/third-party/gui.md +++ /dev/null @@ -1,156 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 28 -toc_title: "\u0631\u0627\u0628\u0637 \u0647\u0627\u06CC \u0628\u0635\u0631\u06CC" ---- - -# رابط های بصری از توسعه دهندگان شخص ثالث {#visual-interfaces-from-third-party-developers} - -## متن باز {#open-source} - -### ترکیب {#tabix} - -رابط وب برای کلیک در [ترکیب](https://github.com/tabixio/tabix) پروژه. - -ویژگی ها: - -- با کلیک هاوس به طور مستقیم از مرورگر کار می کند, بدون نیاز به نصب نرم افزار اضافی. -- ویرایشگر پرس و جو با نحو برجسته. -- تکمیل خودکار دستورات. -- ابزار برای تجزیه و تحلیل گرافیکی از اجرای پرس و جو. -- گزینه های طرح رنگ. - -[مستندات زبانه](https://tabix.io/doc/). - -### خانه {#houseops} - -[خانه](https://github.com/HouseOps/HouseOps) برای اطلاعات بیشتر کلیک کنید - -ویژگی ها: - -- سازنده پرس و جو با نحو برجسته. مشاهده پاسخ در یک جدول و یا نمایش جسون. -- صادرات نتایج پرس و جو به عنوان فیلم و یا جانسون. -- فهرست فرایندها با توصیف. نوشتن حالت. توانایی متوقف کردن (`KILL`) یک فرایند . -- نمودار پایگاه داده. نشان می دهد تمام جداول و ستون خود را با اطلاعات اضافی. -- نمایش سریع از اندازه ستون. -- پیکربندی سرور. - -ویژگی های زیر برای توسعه برنامه ریزی شده است: - -- مدیریت پایگاه داده. -- مدیریت کاربر. -- تجزیه و تحلیل داده های زمان واقعی. -- نظارت خوشه. -- مدیریت خوشه. -- نظارت بر تکرار و کافکا جداول. - -### فانوس دریایی {#lighthouse} - -[فانوس دریایی](https://github.com/VKCOM/lighthouse) یک رابط وب بسیار سبک وزن و برای کلیک است. - -ویژگی ها: - -- فهرست جدول با فیلتر کردن و ابرداده. -- پیش نمایش جدول با فیلتر کردن و مرتب سازی. -- فقط خواندنی نمایش داده شد اعدام. - -### قرمز {#redash} - -[قرمز](https://github.com/getredash/redash) یک پلت فرم برای تجسم داده ها است. - -پشتیبانی از منابع چندگانه داده ها از جمله ClickHouse, Redash می توانید پیوستن به نتایج حاصل از پرس و جو از داده های مختلف منابع را به یکی از نهایی مجموعه. - -ویژگی ها: - -- ویرایشگر قدرتمند نمایش داده شد. -- پایگاه اکسپلورر. -- ابزار تجسم, که اجازه می دهد شما را به نمایندگی از داده ها در اشکال مختلف. - -### DBeaver {#dbeaver} - -[DBeaver](https://dbeaver.io/) - مشتری جهانی پایگاه داده دسکتاپ با پشتیبانی از کلیک. - -ویژگی ها: - -- توسعه پرس و جو با نحو برجسته و تکمیل خودکار. -- فهرست جدول با فیلتر و ابرداده جستجو. -- جدول پیش نمایش داده ها. -- جستجوی کامل متن. - -### کلیک کل {#clickhouse-cli} - -[کلیک کل](https://github.com/hatarist/clickhouse-cli) یک مشتری خط فرمان جایگزین برای فاحشه خانه است که در پایتون 3 نوشته شده است. - -ویژگی ها: - -- تکمیل خودکار. -- نحو برجسته برای نمایش داده شد و خروجی داده ها. -- پشتیبانی پیجر برای خروجی داده ها. -- دستورات مانند: - -### کلیک-شق {#clickhouse-flamegraph} - -[کلیک-شق](https://github.com/Slach/clickhouse-flamegraph) یک ابزار تخصصی برای تجسم است `system.trace_log` به عنوان [شق](http://www.brendangregg.com/flamegraphs.html). - -### کارخانه کلیک {#clickhouse-plantuml} - -[cickhouse-plantuml](https://pypi.org/project/clickhouse-plantuml/) یک اسکریپت برای تولید است [PlantUML](https://plantuml.com/) نمودار طرح جداول. - -## بازرگانی {#commercial} - -### نوار داده {#datagrip} - -[نوار داده](https://www.jetbrains.com/datagrip/) یک پایگاه داده محیط برنامه نویسی از جت با پشتیبانی اختصاصی برای کلیک. آن را نیز تعبیه شده در دیگر IntelliJ مبتنی بر ابزار: PyCharm, IntelliJ IDEA, GoLand, PhpStorm و دیگران. - -ویژگی ها: - -- تکمیل کد بسیار سریع است. -- برجسته نحو تاتر. -- پشتیبانی از ویژگی های خاص به تاتر, مثلا, ستون تو در تو, موتورهای جدول. -- ویرایشگر داده. -- Refactorings. -- جستجو و ناوبری. - -### داده های یاندکس {#yandex-datalens} - -[داده های یاندکس](https://cloud.yandex.ru/services/datalens) یک سرویس تجسم داده ها و تجزیه و تحلیل. - -ویژگی ها: - -- طیف گسترده ای از تصویری در دسترس, از نمودار نوار ساده به داشبورد پیچیده. -- داشبورد می تواند در دسترس عموم ساخته شده است. -- پشتیبانی از منابع داده های متعدد از جمله خانه عروسکی. -- ذخیره سازی برای داده های محقق بر اساس کلیک. - -اطلاعات دقیق [به صورت رایگان در دسترس است](https://cloud.yandex.com/docs/datalens/pricing) برای پروژه های کم بار, حتی برای استفاده تجاری. - -- [اسناد داده](https://cloud.yandex.com/docs/datalens/). -- [اموزش](https://cloud.yandex.com/docs/solutions/datalens/data-from-ch-visualization) در تجسم داده ها از یک پایگاه داده کلیک. - -### Holistics نرم افزار {#holistics-software} - -[Holistics](https://www.holistics.io/) یک پلت فرم داده کامل پشته و ابزار هوش کسب و کار است. - -ویژگی ها: - -- خودکار ایمیل, شل و ورق گوگل برنامه از گزارش. -- ویرایشگر گذاشتن با تصویری, کنترل نسخه, تکمیل خودکار, اجزای پرس و جو قابل استفاده مجدد و فیلتر پویا. -- تجزیه و تحلیل ترافیک جاسازی شده از گزارش ها و داشبورد از طریق ای فریم. -- تهیه داده ها و قابلیت های ال. -- گذاشتن پشتیبانی مدل سازی داده ها برای نقشه برداری رابطه ای از داده ها. - -### تماشاچی {#looker} - -[تماشاچی](https://looker.com) یک پلت فرم داده ها و ابزار هوش کسب و کار با پشتیبانی از 50 گویش پایگاه داده از جمله کلیک است. تماشاچی به عنوان یک پلت فرم تولید و خود میزبانی در دسترس است. کاربران می توانند تماشاچی از طریق مرورگر برای کشف داده ها استفاده کنید, ساخت تصویری و داشبورد, گزارش برنامه, و به اشتراک گذاری بینش خود را با همکاران. تماشاچی فراهم می کند مجموعه ای غنی از ابزار برای جاسازی این ویژگی در برنامه های کاربردی دیگر و رابط برنامه کاربردی -به ادغام داده ها با برنامه های کاربردی دیگر. - -ویژگی ها: - -- توسعه ساده و چابک با استفاده از نگاه میلی لیتر, یک زبان است که پشتیبانی از سرپرستی - [مدل سازی داده ها](https://looker.com/platform/data-modeling) برای حمایت از نویسندگان گزارش و کاربران نهایی. -- ادغام گردش کار قدرتمند از طریق تماشاچی [کنشهای داده](https://looker.com/platform/actions). - -[چگونه برای پیکربندی تاتر در تماشاچی.](https://docs.looker.com/setup-and-management/database-config/clickhouse) - -[مقاله اصلی](https://clickhouse.tech/docs/en/interfaces/third-party/gui/) diff --git a/docs/fa/interfaces/third-party/index.md b/docs/fa/interfaces/third-party/index.md deleted file mode 100644 index bc0a2d3f49d..00000000000 --- a/docs/fa/interfaces/third-party/index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0634\u062E\u0635 \u062B\u0627\u0644\u062B" -toc_priority: 24 ---- - - diff --git a/docs/fa/interfaces/third-party/integrations.md b/docs/fa/interfaces/third-party/integrations.md deleted file mode 100644 index 80cf94c53e0..00000000000 --- a/docs/fa/interfaces/third-party/integrations.md +++ /dev/null @@ -1,110 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 27 -toc_title: "\u06CC\u06A9\u067E\u0627\u0631\u0686\u06AF\u06CC" ---- - -# کتابخانه ادغام از توسعه دهندگان شخص ثالث {#integration-libraries-from-third-party-developers} - -!!! warning "تکذیبنامهها" - یاندکس می کند **نه** حفظ ابزار و کتابخانه های ذکر شده در زیر و هر تست گسترده برای اطمینان از کیفیت خود را انجام نداده اند. - -## محصولات زیربنایی {#infrastructure-products} - -- سیستم های مدیریت پایگاه داده رابطه ای - - [MySQL](https://www.mysql.com) - - [mysql2ch](https://github.com/long2ice/mysql2ch) - - [در حال بارگذاری](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - - [تاتر-خروجی زیر-داده خوان](https://github.com/Altinity/clickhouse-mysql-data-reader) - - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator) - - [PostgreSQL](https://www.postgresql.org) - - [_پاک کردن تصویر](https://github.com/Percona-Lab/clickhousedb_fdw) - - [اطالعات._پاک کردن](https://github.com/Infinidat/infi.clickhouse_fdw) (استفاده [اطالعات.کلیک _شورم](https://github.com/Infinidat/infi.clickhouse_orm)) - - [پی جی 2چ](https://github.com/mkabilov/pg2ch) - - [_پاک کردن](https://github.com/adjust/clickhouse_fdw) - - [MSSQL](https://en.wikipedia.org/wiki/Microsoft_SQL_Server) - - [کلیک کنیدهاجر](https://github.com/zlzforever/ClickHouseMigrator) -- صف پیام - - [کافکا](https://kafka.apache.org) - - [در حال بارگذاری](https://github.com/housepower/clickhouse_sinker) (استفاده [برو کارگیر](https://github.com/ClickHouse/clickhouse-go/)) - - [stream-loader-clickhouse](https://github.com/adform/stream-loader) -- پردازش جریان - - [لرزش](https://flink.apache.org) - - [سینک فلینک-کلیک](https://github.com/ivi-ru/flink-clickhouse-sink) -- ذخیره سازی شی - - [S3](https://en.wikipedia.org/wiki/Amazon_S3) - - [کلیک-پشتیبان گیری](https://github.com/AlexAkulov/clickhouse-backup) -- ارکستراسیون کانتینر - - [کوبرنتس](https://kubernetes.io) - - [کلیک اپراتور](https://github.com/Altinity/clickhouse-operator) -- مدیریت پیکربندی - - [عروسک](https://puppet.com) - - [نام و نام خانوادگی / خانه کلیک](https://forge.puppet.com/innogames/clickhouse) - - [مفدوتوف / خانه کلیک](https://forge.puppet.com/mfedotov/clickhouse) -- نظارت - - [گرافیت](https://graphiteapp.org) - - [نمودار](https://github.com/yandex/graphouse) - - [کربن کلیک](https://github.com/lomik/carbon-clickhouse) + - - [گرافیت-تاتر](https://github.com/lomik/graphite-clickhouse) - - [گرافیت-چ بهینه ساز](https://github.com/innogames/graphite-ch-optimizer) - بهینه سازی پارتیشن های ریخته شده در [اطلاعات دقیق](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) اگر قوانین از [پیکربندی رولپ](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) می تواند اعمال شود - - [گرافانا](https://grafana.com/) - - [فاحشه خانه-گرافانا](https://github.com/Vertamedia/clickhouse-grafana) - - [پرومتیوس](https://prometheus.io/) - - [کلیک _گزارشسپر](https://github.com/f1yegor/clickhouse_exporter) - - [مجلس رقص رسمی دبیرستان](https://github.com/Percona-Lab/PromHouse) - - [کلیک _گزارشسپر](https://github.com/hot-wifi/clickhouse_exporter) (استفاده [برو کارگیر](https://github.com/kshvakov/clickhouse/)) - - [Nagios](https://www.nagios.org/) - - [_تخچه نشانزنی](https://github.com/exogroup/check_clickhouse/) - - [check_clickhouse.py](https://github.com/innogames/igmonplugins/blob/master/src/check_clickhouse.py) - - [زاببیکس](https://www.zabbix.com) - - [هوکاوس-زاببیکس-قالب](https://github.com/Altinity/clickhouse-zabbix-template) - - [کلیپ برد چند منظوره](https://sematext.com/) - - [ادغام کلیک](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse) -- ثبت - - [rsyslog](https://www.rsyslog.com/) - - [املتخانه](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) - - [روان کننده](https://www.fluentd.org) - - [خانه ثبت](https://github.com/flant/loghouse) (برای [کوبرنتس](https://kubernetes.io)) - - [ثبت](https://www.sematext.com/logagent) - - [خروجی ورود-پلاگین-خانه کلیک](https://sematext.com/docs/logagent/output-plugin-clickhouse/) -- نق - - [نیروی برتر](https://dev.maxmind.com/geoip/) - - [خانه هوشمند](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip) - -## اکوسیستم های زبان برنامه نویسی {#programming-language-ecosystems} - -- پایتون - - [SQLAlchemy](https://www.sqlalchemy.org) - - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (استفاده [اطالعات.کلیک _شورم](https://github.com/Infinidat/infi.clickhouse_orm)) - - [پانداها](https://pandas.pydata.org) - - [پانداهاوس](https://github.com/kszucs/pandahouse) -- PHP - - [دکترین](https://www.doctrine-project.org/) - - [خانه هوشمند](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) -- R - - [هواپیمای دوباله](https://db.rstudio.com/dplyr/) - - [خانه روستایی](https://github.com/IMSMWU/RClickHouse) (استفاده [صفحه اصلی](https://github.com/artpaul/clickhouse-cpp)) -- جاوا - - [هادوپ](http://hadoop.apache.org) - - [سرویس پرداخت درونبرنامهای پلی](https://github.com/jaykelin/clickhouse-hdfs-loader) (استفاده [JDBC](../../sql-reference/table-functions/jdbc.md)) -- اسکالا - - [اککا](https://akka.io) - - [تاتر-اسکالا-کارفرما](https://github.com/crobox/clickhouse-scala-client) -- C# - - [ADO.NET](https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/ado-net-overview) - - [فاحشه خانه.ادو](https://github.com/killwort/ClickHouse-Net) - - [فاحشه خانه.کارگیر](https://github.com/DarkWanderer/ClickHouse.Client) - - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) - - [مهاجرت.](https://github.com/ilyabreev/ClickHouse.Net.Migrations) -- اکسیر - - [Ecto](https://github.com/elixir-ecto/ecto) - - [حذف جستجو](https://github.com/appodeal/clickhouse_ecto) -- Ruby - - [Ruby on Rails](https://rubyonrails.org/) - - [activecube](https://github.com/bitquery/activecube) - - [ActiveRecord](https://github.com/PNixx/clickhouse-activerecord) - - [GraphQL](https://github.com/graphql) - - [activecube-graphql](https://github.com/bitquery/activecube-graphql) - -[مقاله اصلی](https://clickhouse.tech/docs/en/interfaces/third-party/integrations/) diff --git a/docs/fa/interfaces/third-party/proxy.md b/docs/fa/interfaces/third-party/proxy.md deleted file mode 100644 index a63ee6d86de..00000000000 --- a/docs/fa/interfaces/third-party/proxy.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 29 -toc_title: "\u067E\u0631\u0627\u06A9\u0633\u06CC \u0647\u0627" ---- - -# سرور های پروکسی از توسعه دهندگان شخص ثالث {#proxy-servers-from-third-party-developers} - -## لوسیون صورت {#chproxy} - -[لوسیون صورت](https://github.com/Vertamedia/chproxy), یک پروکسی قام است و متعادل کننده بار برای پایگاه داده کلیک. - -ویژگی ها: - -- هر کاربر مسیریابی و ذخیره پاسخ. -- محدودیت های انعطاف پذیر. -- تمدید گواهی اس اس ال به صورت خودکار. - -اجرا در بروید. - -## خانه کوچک {#kittenhouse} - -[خانه کوچک](https://github.com/VKCOM/kittenhouse) طراحی شده است که یک پروکسی محلی بین کلاینت و سرور نرم افزار در صورتی که غیر ممکن است و یا ناخوشایند به بافر قرار دادن داده ها در سمت نرم افزار خود را. - -ویژگی ها: - -- در حافظه و بر روی دیسک بافر داده. -- در هر جدول مسیریابی. -- متعادل کننده بار و چک کردن سلامت. - -اجرا در بروید. - -## کلیک-فله {#clickhouse-bulk} - -[کلیک-فله](https://github.com/nikepan/clickhouse-bulk) یک جمع درج کلیک ساده است. - -ویژگی ها: - -- درخواست گروه و ارسال شده توسط حد و یا فاصله. -- چندین سرور از راه دور. -- احراز هویت عمومی. - -اجرا در بروید. - -[مقاله اصلی](https://clickhouse.tech/docs/en/interfaces/third-party/proxy/) diff --git a/docs/fa/introduction/adopters.md b/docs/fa/introduction/adopters.md deleted file mode 100644 index 654f3a24736..00000000000 --- a/docs/fa/introduction/adopters.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 8 -toc_title: "\u067E\u0630\u06CC\u0631\u0627" ---- - -# پذیرندگان خانه کلیک {#clickhouse-adopters} - -!!! warning "تکذیبنامهها" - لیست زیر از شرکت ها با استفاده از کلیک و داستان های موفقیت خود را از منابع عمومی مونتاژ, در نتیجه ممکن است از واقعیت فعلی متفاوت. اگر داستان اتخاذ خانه کلیک در شرکت خود را به اشتراک بگذارید قدردانی می کنیم [افزودن به فهرست](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/introduction/adopters.md) اما لطفا اطمینان حاصل کنید که با انجام این کار هیچ مشکلی ندارید. فراهم کردن به روز رسانی با انتشارات از شرکت های دیگر نیز مفید است. - -| شرکت | صنعت | حذف جستجو | اندازه خوشه | (سازمان ملل متحد) حجم داده های فشرده\* | مرجع | -|------------------------------------------------------------------------------------------------|----------------------------------------|-------------------------|------------------------------------------|------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 2گیس | مپز | نظارت | — | — | [بحث در روسیه, جولای 2019](https://youtu.be/58sPkXfq6nw) | -| Aloha Browser | نرم افزار تلفن همراه | باطن مرورگر | — | — | [اسلاید در روسیه, بیشتر 2019](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | -| Amadeus | سفر | تجزیه و تحلیل | — | — | [اطلاعیه مطبوعاتی مارس 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | -| اپسفیر | تجزیه و تحلیل ترافیک تلفن همراه | محصول اصلی | — | — | [بحث در روسیه, جولای 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | -| دادههای رایانه | پلت فرم داده | محصول اصلی | — | — | [اسلاید در روسیه, دسامبر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | -| بدو | دوستیابی | زمانهای | — | — | [اسلاید در روسیه, دسامبر 2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | -| بنکس | تله متری شبکه و تجزیه و تحلیل | محصول اصلی | — | — | [اسلاید به زبان انگلیسی, اکتبر 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | -| بلومبرگ | امور مالی, رسانه ها | نظارت | 102 سرور | — | [اسلاید مه 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | -| نفخ | بلاکچین | تجزیه و تحلیل | — | — | [اسلاید در روسیه, اوت 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | -| تمرد برای مخابرات چین | مخابرات | تجزیه و تحلیل | — | — | [اسلاید در چین, ژانویه 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | -| CARTO | اطلاعات کسب و کار | تجزیه و تحلیل جغرافیایی | — | — | [پردازش جغرافیایی با کلیک](https://carto.com/blog/geospatial-processing-with-clickhouse/) | -| CERN | پژوهش و فناوری | تجربه | — | — | [اطلاعیه مطبوعاتی مارس 2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | -| سیسکو | شبکه | تجزیه و تحلیل ترافیک | — | — | [بحث رعد و برق, اکتبر 2019](https://youtu.be/-hI1vDR2oPY?t=5057) | -| Citadel Securities | امور مالی | — | — | — | [مشارکت, مارس 2019](https://github.com/ClickHouse/ClickHouse/pull/4774) | -| سیتیموبیل | تاکسی | تجزیه و تحلیل | — | — | [پست وبلاگ در روسیه مارس 2020](https://habr.com/en/company/citymobil/blog/490660/) | -| تماس با ما | تجزیه و تحلیل ترافیک وب | محصول اصلی | — | — | [پست وبلاگ در فرانسه, نوامبر 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | -| ابر پرچم | CDN | تجزیه و تحلیل ترافیک | 36 سرور | — | [پست وبلاگ, بیشتر 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [پست وبلاگ, مارس 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | -| کوروت | تجزیه و تحلیل | محصول اصلی | — | — | [اسلاید به زبان انگلیسی, مارس 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | -| CraiditX 氪信 | Finance AI | تجزیه و تحلیل | — | — | [اسلاید به زبان انگلیسی, نوامبر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | -| Criteo | خرده فروشی | محصول اصلی | — | — | [اسلاید به زبان انگلیسی, اکتبر 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | -| Deutsche Bank | امور مالی | بی تجزیه و تحلیل | — | — | [اسلاید به زبان انگلیسی, اکتبر 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | -| سردستهزنان خواننده اپرا | مشاوره دیجیتال | محصول اصلی | — | — | [اسلاید به زبان انگلیسی, سپتامبر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | -| اعمال | بازرگانی | معیارهای ورود به سیستم | — | — | [بحث در روسیه, بیشتر 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | -| ژنی | شبکه تبلیغاتی | محصول اصلی | — | — | [پست وبلاگ در ژاپن, جولای 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | -| HUYA | جریان ویدیو | تجزیه و تحلیل | — | — | [اسلاید در چین, اکتبر 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| Idealista | املاک و مستغلات | تجزیه و تحلیل | — | — | [پست وبلاگ به زبان انگلیسی, مارس 2019](https://clickhouse.tech/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | -| اینفویستا | شبکه ها | تجزیه و تحلیل | — | — | [اسلاید به زبان انگلیسی, اکتبر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | -| نام | بازی ها | معیارهای ورود به سیستم | — | — | [اسلاید در روسیه, سپتامبر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | -| پوششی | بستر های نرم افزاری برای خدمات تصویری | تجزیه و تحلیل | — | — | [اسلاید در روسیه, بیشتر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | -| دادههای کدیاک | ابرها | محصول اصلی | — | — | [اسلاید در انگلیسی, مارس 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | -| Kontur | توسعه نرم افزار | متریک | — | — | [بحث در روسیه, نوامبر 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | -| اشک | شبکه تبلیغاتی | محصول اصلی | 75 سرور (3 کپی) | 5.27 پوند | [پست وبلاگ در روسیه, فوریه 2017](https://habr.com/en/post/322620/) | -| Mail.ru راه حل های ابر | خدمات ابری | محصول اصلی | — | — | [مقاله در روسیه](https://mcs.mail.ru/help/db-create/clickhouse#) | -| مسگبرد | مخابرات | امار | — | — | [اسلاید به زبان انگلیسی, نوامبر 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | -| MGID | شبکه تبلیغاتی | تجزیه و تحلیل وب سایت | — | — | [پست وبلاگ در روسیه, مارس 2020](http://gs-studio.com/news-about-it/32777----clickhouse---c) | -| بعد از ظهر | مانیتورینگ و تجزیه و تحلیل داده ها | محصول اصلی | — | — | [اسلاید در چین, اکتبر 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | -| Pragma Innovation | تله متری و تجزیه و تحلیل داده های بزرگ | محصول اصلی | — | — | [اسلاید به زبان انگلیسی, اکتبر 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | -| QINGCLOUD | خدمات ابری | محصول اصلی | — | — | [اسلاید در چین, اکتبر 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | -| فرمانده | DDoS protection | محصول اصلی | — | — | [وبلاگ پست, مارس 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | -| Percent 百分点 | تجزیه و تحلیل | محصول اصلی | — | — | [اسلاید در چین, خرداد 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | -| رامبلر | خدمات اینترنت | تجزیه و تحلیل | — | — | [بحث در روسیه, مارس 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | -| شمارش | پیامرسانی | ثبت | — | — | [بحث در چین, نوامبر 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | -| Traffic Stars | شبکه تبلیغاتی | — | — | — | [اسلاید در روسیه, بیشتر 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | -| S7 Airlines | خطوط هوایی | معیارهای ورود به سیستم | — | — | [بحث در روسیه, مارس 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | -| SEMrush | بازاریابی | محصول اصلی | — | — | [اسلاید در روسیه, اوت 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | -| scireum GmbH | تجارت الکترونیک | محصول اصلی | — | — | [بحث در, فوریه 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | -| نگهبانی | توسعه دهنده نرم افزار | باطن برای محصول | — | — | [پست وبلاگ به زبان انگلیسی, بیشتر 2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | -| SGK | دولت امنیت اجتماعی | تجزیه و تحلیل | — | — | [اسلاید به زبان انگلیسی, نوامبر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) | -| seo.do | تجزیه و تحلیل | محصول اصلی | — | — | [اسلاید به زبان انگلیسی, نوامبر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) | -| سینا | اخبار | — | — | — | [اسلاید در چین, اکتبر 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | -| SMI2 | اخبار | تجزیه و تحلیل | — | — | [پست وبلاگ در روسیه, نوامبر 2017](https://habr.com/ru/company/smi2/blog/314558/) | -| اسپانک | تجزیه و تحلیل کسب و کار | محصول اصلی | — | — | [اسلاید به زبان انگلیسی, ژانویه 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | -| رفع ابهام | موسیقی | تجربه | — | — | [اسلاید, جولای 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | -| شمارش | داده های بزرگ | پردازش داده ها | — | — | [اسلاید در چین, اکتبر 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | -| بارگذاری | تاکسی | ثبت | — | — | [اسلاید فوریه 2020](https://presentations.clickhouse.tech/meetup40/uber.pdf) | -| اطلاعات دقیق | شبکه اجتماعی | ارقام, ورود به سیستم | — | — | [اسلاید در روسیه, اوت 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | -| خردمندهها | راه حل های فناوری اطلاعات | تجزیه و تحلیل | — | — | [اسلاید در روسیه, بیشتر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | -| Xiaoxin Tech | تحصیلات | هدف مشترک | — | — | [اسلاید به زبان انگلیسی, نوامبر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | -| خیمیایا | به اشتراک گذاری صوتی | OLAP | — | — | [اسلاید به زبان انگلیسی, نوامبر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | -| Yandex Cloud | ابر عمومی | محصول اصلی | — | — | [بحث در روسیه, دسامبر 2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | -| Yandex DataLens | اطلاعات کسب و کار | محصول اصلی | — | — | [اسلاید در روسیه, دسامبر 2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | -| Yandex Market | تجارت الکترونیک | معیارهای ورود به سیستم | — | — | [بحث در روسیه, ژانویه 2019](https://youtu.be/_l1qP0DyBcA?t=478) | -| Yandex Metrica | تجزیه و تحلیل ترافیک وب | محصول اصلی | 360 سرور در یک خوشه, 1862 سرور در یک بخش | 66.41 PiB / 5.68 PiB | [اسلاید فوریه 2020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | -| ЦВТ | توسعه نرم افزار | معیارهای ورود به سیستم | — | — | [پست وبلاگ, مارس 2019, در روسیه](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | -| МКБ | بانک | نظارت بر وب سیستم | — | — | [اسلاید در روسیه, سپتامبر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | -| Jinshuju 金数据 | بی تجزیه و تحلیل | محصول اصلی | — | — | [اسلاید در چین, اکتبر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | -| پشت پا | پلتفرم | محصول اصلی | — | — | [توییتر پست](https://twitter.com/mieldonkers/status/1248884119158882304) | -| بازیهای جنگی | بازی ها | | — | — | [مصاحبه](https://habr.com/en/post/496954/) | -| دیوانه | بازی ها | | — | — | جلسه زنده در ناهار کلیک | -| عمل کننده | بازی ها | | — | — | [مقاله](https://www.altinity.com/blog/migrating-from-redshift-to-clickhouse) | - -[مقاله اصلی](https://clickhouse.tech/docs/en/introduction/adopters/) diff --git a/docs/fa/introduction/distinctive-features.md b/docs/fa/introduction/distinctive-features.md deleted file mode 100644 index 94ee2ee76fa..00000000000 --- a/docs/fa/introduction/distinctive-features.md +++ /dev/null @@ -1,73 +0,0 @@ -
- -# ویژگی های برجسته ClickHouse {#wyjgy-hy-brjsth-clickhouse} - -## مدیریت دیتابیس ستون گرای واقعی {#mdyryt-dytbys-stwn-gry-wq-y} - -در یک مدیریت دیتابیس ستون گرای واقعی، هیچ مقداری فضای اضافی برای ذخیره سازی ندارد. برای مثال، این به این معنیست که برای مقادیر، constant-length باید پشتیبانی شوند تا از ذخیره سازی طول مقدار به عنوان یه عدد integer کنار مقدار جلوگیری شود. در این مورد، یک میلیارد مقدار Uint8 باید در واقع در حالت غیرفشرده 1 گیگابایت فضا اشغال کند، در غیراین صورت به شدت بر عملکرد CPU تاثیر میگذارد. این خیلی مهم هست که داده ها به صورت compact ذخیره سازی شوند حتی زمانی که uncompressed هستند، از آنجا که سرعت سرعت decompress (CPU Usage) عمدتا به حجم داده های uncompress بستگی دارد. - -این بسیار قابل توجه است چون سیستم هایی وجود دارند که توانایی ذخیره سازی مقادیر ستون ها را به صورت جداگانه دارند، اما به دلیل بهینه سازی آنها برای دیگر سناریو ها، نمیتوانند به طور موثر پردازش های تحیلی انجام دهند. برای مثال HBase، BigTable، Cassandra و HyperTable. در این سیستم ها، شما توان عملیاتی حدود صدها هزار سطر در ثانیه را دارید، اما نه صدها میلیون سطر در ثانیه. - -همچنین توجه داشته باشید که ClickHouse یک مدیریت دیتابیس است نه فقط یک دیتابیس. ClickHouse اجازه میدهد که در زمان اجرا اقدام به ساخت جدول، دیتابیس کنید، داده load کنید و query های خود را بدون restart و یا reconfigure مجدد سرور، اجرا کنید - -## فشرده سازی داده ها {#fshrdh-szy-ddh-h} - -بعضی دیتابیس های ستون گرا (InfiniDB CE یا MonetDB) از فشرده سازی داده ها استفاده نمی کنند. با این حال، فشرده سازی داده ها برای رسیدن به عملکرد عالی ضروری است. - -## Disk Storage of Data {#disk-storage-of-data} - -خیلی از مدیریت دیتابیس های ستون گرا (مثل SAP HANA و Google PowerDrill) فقط داخل RAM کار می کنند. این رویکرد منجر به تخصیص بودجه سخت افزاری بالا برای تحقق آنالیز های real-time می شود. ClickHouse برای کار بر روی هارد دیسک های معمولی طراحی شده است، که هزینه ی برای هر گیگابایت داده را تضمین می کند، اما اگر SSD و RAM موجود باشد به طور کامل مورد استفاده قرار می گیرد. - -## پردازش موازی روی چندین هسته {#prdzsh-mwzy-rwy-chndyn-hsth} - -query های بزرگ به طور طبیعی با استفاده از تمام منابع موجود در روی سرور فعلی، موازی سازی می شوند. - -## پردازش توزیع شده بر روی چندین سرور {#prdzsh-twzy-shdh-br-rwy-chndyn-srwr} - -تقریبا هیچ کدام از DBMS هایی که بالاتر ذکر شد، از اجرای query ها به صورت توزیع شده پشتیبانی نمی کنند. در ClickHouse، داده ها می توانن در shard های مختلف مستقر شوند. هر shard میتوامند گروهی از replica ها برای بالا بردن تحمل پذیری در برابر خطا (fault tolerance) را داشته باشد. یک query به صورت موازی بر روی تمامی shard های اجرا می شود. این برای کاربر شفاف است. - -## پشتیبانی SQL {#pshtybny-sql} - -اگر شما با SQL آشنا باشید، ما واقعا نمیتونیم در مورد پشتیبانی از SQL صحبت کنیم. تمام توابع اسم های مختلفی دارند. با این حال، این یک زبان بر پایه SQL هست که نمیتواند در بسیاری از موارد با SQL متفاوت باشد. JOIN ها پشتیبانی می شود. subquery ها در FROM، IN و JOIN پشتیبانی می شود. Dependent subquery ها پشتیبانی نمی شود. - -ClickHouse زبان بر پایه SQL است که در بسیاری از موارد از استاندارد SQL پیروی می کند. GROUP BY، ORDER BY، scalar subquery ها در FROM، IN و JOIN پشتیبانی می شود. subquery های مرتبط و window function ها پشتیبانی نمی شود. - -## موتور بردارد {#mwtwr-brdrd} - -داده ها نه فقط براساس ستون ها ذخیره می شوند، بلکه با استفاده از برداردها (بخشی از ستون ها) پردازش می شوند. این قابلیت باعث رسیدن به بهره وری بالای CPU می شود. - -## بروزرسانی داده ها به صورت Real-Time {#brwzrsny-ddh-h-bh-swrt-real-time} - -ClickHouse از جداول دارای Primary Key پشتیبانی می کند. به منظور اجرای query های range بر روی Primary Key، داده ها به صورت افزایشی (مرتب شده) با استفاده از merge tree ذخیره سازی می شوند. با توجه به این، داده ها می توانند به طور پیوسته به جدول اضافه شوند. هیچ نوع lock در هنگام مصرف داده ها وجود ندارد. - -## Index {#index} - -داشتن داده ها به صورت فیزیکی و مرتب شده براساس Primary Key این قابلیت را می دهد که استخراج کردن داده برای مقدار خاص و یا مقادیر range با کمترین latencey، یعنی کمتر از چند هزار میلی ثانیه ممکن شود. - -## مناسب برای query های آنلاین {#mnsb-bry-query-hy-anlyn} - -latency پایین به این معنی است که query ها بتونن بدون delay و بدون تلاش برای رسیدن به پیش پاسخ(از قبل محاسبه شده) دقیقا در همان لحظه که کاربر در حال load صفحه است پردازش شوند. به عبارتی دیگر، آنلاین - -## پشتیبانی از محاسبات تقریبی {#pshtybny-z-mhsbt-tqryby} - -ClickHouse روش های مختلفی برای کسب دقیق performance ارائه می دهد: - -1. توابع Aggregate برای محاسبات تقریبی تعداد مقادیر متمایز (distinct)، median و quantity ها -2. اجرای یک query بر پایه بخشی از داده ها (داده ی sample) و دریافت خروجی تقریبی. در این مورد داده ی نسبتا کمتری از دیسک بازیابی می شود. -3. اجرای یک Aggregation برای تعداد محدودی از کلید های تصافی، به جای تمام کلید ها. در شرایط خاص برای توزیع کلید در داده ها، این روش کمک می کند به نتایج منطقی برسیم با استفاده از منابع کمتر. - -## Replication داده ها و Integrity {#replication-ddh-h-w-integrity} - -ClickHouse از روش asynchronous multimaster replication استفاده می کند. بعد از نوشتن داده در یکی از replica های موجود، داده به صورت توزیع شده به بقیه replica ها منتقل می شود. این سیستم داده های مشابه را در replica های مختلف نگه داری می کند. در اکثر موارد که سیستم fail می شوند، داده ها به صورت اتوماتیک restore می شوند و یا در موارد پیچیده به صورت نیمه اتوماتیک restore می شوند. - -برای اطلاعات بیشتر، به بخش [replication داده ها](../engines/table-engines/mergetree-family/replication.md) مراجعه کنید. - -## ویژگی های از ClickHouse که می تواند معایبی باشد {#wyjgy-hy-z-clickhouse-khh-my-twnd-m-yby-bshd} - -1. بدون پشتیبانی کامل از تراکنش -2. عدم توانایی برای تغییر و یا حذف داده های در حال حاضر وارد شده با سرعت بالا و تاخیر کم. برای پاک کردن و یا اصلاح داده ها، به عنوان مثال برای پیروی از [GDPR](https://gdpr-info.eu)، دسته ای پاک و به روزرسانی وجود دارد.حال توسعه می باشد. -3. Sparse index باعث می شود ClickHouse چندان مناسب اجرای پرسمان های point query برای دریافت یک ردیف از داده ها با استفاده از کلید آنها نباشد. - -
- -[مقاله اصلی](https://clickhouse.tech/docs/fa/introduction/distinctive_features/) diff --git a/docs/fa/introduction/history.md b/docs/fa/introduction/history.md deleted file mode 100644 index 214100ab544..00000000000 --- a/docs/fa/introduction/history.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 7 -toc_title: "\u062A\u0627\u0631\u06CC\u062E\u0686\u0647" ---- - -# تاریخچه کلیک {#clickhouse-history} - -تاتر در ابتدا به قدرت توسعه داده شده است [یاندکسمتریکا](https://metrica.yandex.com/), [دومین بزرگترین پلت فرم تجزیه و تحلیل ترافیک وب در جهان](http://w3techs.com/technologies/overview/traffic_analysis/all) و همچنان بخش اصلی این سیستم است. با بیش از 13 تریلیون رکورد در پایگاه داده و بیش از 20 میلیارد حوادث روزانه, خانه رعیتی اجازه می دهد تا تولید گزارش های سفارشی در پرواز به طور مستقیم از داده های غیر جمع. این مقاله به طور خلاصه اهداف کلیک در مراحل اولیه توسعه خود را پوشش می دهد. - -یاندکسمتریکا گزارش های سفارشی در پرواز را بر اساس بازدید ها و جلسات با بخش های دلخواه تعریف شده توسط کاربر ایجاد می کند. انجام این کار اغلب نیاز به ساختمان مجموعه های پیچیده مانند تعداد کاربران منحصر به فرد. اطلاعات جدید برای ساخت یک گزارش می رسد در زمان واقعی است. - -همانطور که از مارس 2014, یاندکس.متریکا روزانه حدود 12 میلیارد رویداد (نمایش صفحه و کلیک) را ردیابی کرد. همه این وقایع باید ذخیره شود برای ساخت گزارش های سفارشی. پرس و جو تنها ممکن است نیاز به اسکن میلیون ها ردیف در عرض چند صد میلی ثانیه و یا صدها میلیون ردیف فقط در چند ثانیه. - -## استفاده در یاندکس.متریکا و سایر خدمات یاندکس {#usage-in-yandex-metrica-and-other-yandex-services} - -خانه عروسکی در خدمت اهداف متعدد در یاندکس.متریکا -وظیفه اصلی این است برای ساخت گزارش در حالت اینترنتی با استفاده از داده های غیر جمع. با استفاده از یک خوشه 374 سرور, که ذخیره بیش از 20.3 تریلیون ردیف در پایگاه داده. حجم داده های فشرده است در مورد 2 سرب, بدون حسابداری برای تکراری و کپی. حجم داده های غیر فشرده (در فرمت تسو) حدود 17 پوند خواهد بود. - -کلیک هاوس همچنین نقش کلیدی در فرایندهای زیر ایفا می کند: - -- ذخیره سازی داده ها برای پخش جلسه از یاندکس.متریکا -- پردازش اطلاعات متوسط. -- ساختمان گزارش های جهانی با تجزیه و تحلیل ترافیک. -- در حال اجرا نمایش داده شد برای اشکال زدایی یاندکس.موتور متریکا. -- تجزیه و تحلیل سیاهههای مربوط از رابط کاربر. - -امروزه چند ده ClickHouse تاسیسات در دیگر Yandex خدمات و ادارات: جستجوی عمودی, e-commerce, تبلیغات, کسب و کار, تجزیه و تحلیل ترافیک تلفن همراه و توسعه و خدمات شخصی ، - -## داده های جمع شده و غیر جمع شده {#aggregated-and-non-aggregated-data} - -یک نظر گسترده است که برای محاسبه ارقام به طور موثر وجود دارد, شما باید داده ها جمع از این حجم داده ها را کاهش می دهد. - -اما تجمع داده ها با محدودیت های زیادی همراه است: - -- شما باید یک لیست از پیش تعریف شده از گزارش های مورد نیاز داشته باشد. -- کاربر می تواند گزارش های سفارشی را ندارد. -- هنگامی که جمع بیش از تعداد زیادی از کلید های متمایز, حجم داده ها به سختی کاهش می یابد, بنابراین تجمع بی فایده است. -- برای تعداد زیادی از گزارش, بیش از حد بسیاری از تغییرات تجمع وجود دارد (انفجار ترکیبی). -- هنگامی که جمع کلید با کارتنیت بالا (مانند نشانیهای وب), حجم داده ها توسط بسیار کاهش می یابد (کمتر از دو قسم). -- به همین دلیل حجم داده ها با تجمع ممکن است به جای کوچک شدن رشد می کنند. -- کاربران تمام گزارش هایی را که برای ما تولید می کنیم مشاهده نمی کنند. بخش بزرگی از این محاسبات بی فایده است. -- یکپارچگی منطقی داده ها ممکن است برای تجمع های مختلف نقض شده است. - -اگر ما هر چیزی جمع نیست و کار با داده های غیر جمع, این ممکن است حجم محاسبات را کاهش می دهد. - -با این حال, با تجمع, بخش قابل توجهی از کار گرفته شده است و نسبتا کلمی تکمیل. در مقابل محاسبات اینترنتی نیاز به محاسبه به همان سرعتی که ممکن است از کاربر در حال انتظار برای نتیجه. - -یاندکسمتریکا دارای یک سیستم تخصصی برای جمع بندی داده ها به نام متراژ, که برای اکثر گزارش مورد استفاده قرار گرفت. -شروع در 2009, یاندکس.Metrica همچنین با استفاده از تخصصی OLAP پایگاه داده برای عدم جمع آوری داده ها به نام OLAPServer که قبلا با استفاده از گزارش ساز. -برای دادههای غیر تجمیع خوب کار میکرد اما محدودیتهای زیادی داشت که اجازه نداد برای تمامی گزارشها مورد نظر استفاده قرار گیرد. این شامل عدم پشتیبانی از انواع داده ها (فقط اعداد) و ناتوانی در به روز رسانی تدریجی داده ها در زمان واقعی (تنها می تواند با بازنویسی داده ها روزانه انجام می شود). اولسپرز یک دسی بل نیست, اما یک دسی بل تخصصی. - -هدف اولیه برای ClickHouse بود برای حذف محدودیت های OLAPServer و حل مشکل از کار کردن با غیر جمع آوری داده ها برای همه, گزارش, اما در طول سال رشد کرده است به طور کلی هدف مدیریت پایگاه داده سیستم مناسب برای طیف گسترده ای از وظایف تحلیلی. - -[مقاله اصلی](https://clickhouse.tech/docs/en/introduction/history/) diff --git a/docs/fa/introduction/index.md b/docs/fa/introduction/index.md deleted file mode 100644 index 3e2861957a5..00000000000 --- a/docs/fa/introduction/index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" -toc_priority: 1 ---- - - diff --git a/docs/fa/introduction/performance.md b/docs/fa/introduction/performance.md deleted file mode 100644 index 497f84e1958..00000000000 --- a/docs/fa/introduction/performance.md +++ /dev/null @@ -1,29 +0,0 @@ -
- -# Performance {#performance} - -با توجه به نتایج تست های Yandex، ClickHouse بهترین عملکرد را برای سناریوهای عملیاتی قابل مقایسه با دیگر سیستم های در کلاس خود را از خود نشان داد. این تست ها شامل بالاترین توان عملیاتی برای query های طولانی، و کمترین latency برای query های کوتاه بود. نتایج این تست های در [صفحه ی جدا](https://clickhouse.tech/benchmark/dbms/) موجود است. - -benchmark های زیادی وجود دارند که این نتایج را تایید می کنند. میتوانید این نتایج را جستجو کنید و یا [این لینک های benchmark](https://clickhouse.tech/#independent-benchmarks). مستقل را ببینید. - -## توان عملیاتی برای یک query بزرگ {#twn-mlyty-bry-ykh-query-bzrg} - -توان عملیاتی می تواند به صورت تعداد سطر در ثانیه و یا تعداد مگابایت در ثانیه اندازه گیری شود. اگر داده ها در page cache قرار داشته باشند، یک query برای اجرا شدن بر روی سخت افزارهای مدرن چندان پیچیده نخواهد بود و با سرعت تقریبا 2 تا 10 گیگابایت در ثانیه برای داده های غیرفشرده و در یک سرور پردازش خواهد شد (برای ساده ترین موارد، سرعت ممکن است به 30 گیگابایت در ثانیه برسد). اگر داده ها در page cache قرار نداشته باشند، سرعت محدود به دیسک و همچنین چگونگی فشرده سازی داده ها بر روی دیسک می باشد. برای مثال اگر یک دیسک اجازه ی خواندن داده ها با سرعت 400 مگابایت در ثانیه را بدهد، و داده ها با نرخ 3 فشرده سازی شده باشند، سرعت در حدود 1.2 گیگابایت در ثانیه خواهد بود. برای گرفتن تعداد رکورد در ثانیه، سرعت بایت در ثانیه را تقسیم بر کل سایز ستون ها مورد استفاده در query می کنیم. برای مثال اگر 10 بایت از ستوه ها استخراج می شود، سرعت در حدود 100 تا 200 میلیون سطر در ثانیه می باشد. - -سرعت پردازش در سرویس توزیع شده تقریبا به صورت خطی افزایش پیدا می کند، اما فقط وقتی که نتایج سطرهای به دست آمده از aggeration یا مرتب سازی زیاد بزرگ نباشند. - -## Latency در زمان پردازش Query های کوتاه {#latency-dr-zmn-prdzsh-query-hy-khwth} - -اگر یک query از Primary Key استفاده کند و تعداد زیادی از سطر ها را برای پردازش select نکند (صدها هزار)، و از تعداد زیادی ستون استفاده نکند،اگر داده ها در page cache قرار داشته باشند، ما میتوانیم انتظار latency کمتر از 50 میلی ثانیه را داشته باشیم. در غیر این صورت محاسبه زمان براساس تعداد seek ها انجام خواهد گرفت. اگر شما از هارد های دیسکی استفاده می کنید، برای سیستمی که overload ندارد، محاسبه تقریبی latency با استفاده از این فرمول ممکن است: زمان seek (10 ms) \* تعداد ستون های مورد نیاز در query \* تعداد قطعات داده - -## توان عملیاتی در هنگام پردازش تعداد زیادی از query های کوتاه {#twn-mlyty-dr-hngm-prdzsh-t-dd-zydy-z-query-hy-khwth} - -تحت شرایط مشابه، ClickHouse توانایی رسیدگی به چند صد query در ثانیه به ازای یک سرور را دارد ( بالای چند هزار در ثانیه در بهترین مورد). از آنجایی که این سناریو در مدیریت دیتابیس های آنالیزی معمول نیست، بهتر است نهایتا انتظار چند صد query در ثانیه را داشته باشید. - -## Performance در هنگام درج داده ها {#performance-dr-hngm-drj-ddh-h} - -پیشنهاد می کنیم درج داده ها را به صورت دسته ای و حداقل 100 سطر در هر دسته انجام دهید و یا بیش از یک درخواست insert در ثانیه را نداشته باشید. در هنگام درج داده در جدول MergeTree از یک dump جدا شده با tab، سرعت درج داده از 50 تا 200 مگابایت در ثانیه می باشد. اگر سطر های درج شده حدود 1 کیلوبایت باشند، سرعت حدود 50 هزار تا 200 هزار سطر در ثانیه می باشد. اگر سطر ها کوچک باشند بازدهی بالایی در تعداد سطر در ثانیه خواهیم داشت. در Banner System Data -`>` 500 هزار سطر در ثانیه، در Graphite data -`>` 1 میلیون سطر در ثانیه). برای بهبود کارایی، شما می توانید چندین insert را به صورت موازی اجرا کنید، که در این حالت کارایی سیستم به صورت خطی افزایش می یابد. - -
- -[مقاله اصلی](https://clickhouse.tech/docs/fa/introduction/performance/) diff --git a/docs/fa/operations/access-rights.md b/docs/fa/operations/access-rights.md deleted file mode 100644 index 6746a661424..00000000000 --- a/docs/fa/operations/access-rights.md +++ /dev/null @@ -1,144 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 48 -toc_title: "\u06A9\u0646\u062A\u0631\u0644 \u062F\u0633\u062A\u0631\u0633\u06CC \u0648\ - \ \u0645\u062F\u06CC\u0631\u06CC\u062A \u062D\u0633\u0627\u0628" ---- - -# کنترل دسترسی و مدیریت حساب {#access-control} - -تاتر از مدیریت کنترل دسترسی بر اساس [RBAC](https://en.wikipedia.org/wiki/Role-based_access_control) نزدیک شو - -نهادهای دسترسی کلیک کنید: -- [حساب کاربری](#user-account-management) -- [نقش](#role-management) -- [سیاست سطر](#row-policy-management) -- [تنظیمات](#settings-profiles-management) -- [سهمیه](#quotas-management) - -شما می توانید اشخاص دسترسی با استفاده از پیکربندی کنید: - -- گردش کار گذاشتن رانده. - - شما نیاز به [فعالسازی](#enabling-access-control) این قابلیت. - -- کارگزار [پروندههای پیکربندی](configuration-files.md) `users.xml` و `config.xml`. - -ما توصیه می کنیم با استفاده از گردش کار گذاشتن محور. هر دو روش پیکربندی به طور همزمان کار, بنابراین اگر شما با استفاده از فایل های پیکربندی سرور برای مدیریت حساب و حقوق دسترسی, شما به نرمی می توانید به گردش کار گذاشتن محور حرکت. - -!!! note "اخطار" - شما می توانید نهاد دسترسی مشابه توسط هر دو روش پیکربندی به طور همزمان مدیریت نیست. - -## استفاده {#access-control-usage} - -به طور پیش فرض سرور کلیک حساب کاربر را فراهم می کند `default` که مجاز نیست با استفاده از کنترل دسترسی گذاشتن محور و مدیریت حساب اما تمام حقوق و مجوز. این `default` حساب کاربری است که در هر مورد استفاده می شود زمانی که نام کاربری تعریف نشده است, مثلا, در ورود از مشتری و یا در نمایش داده شد توزیع. در پرس و جو توزیع پردازش یک حساب کاربری پیش فرض استفاده شده است, اگر پیکربندی سرور یا خوشه مشخص نیست [کاربر و رمز عبور](../engines/table-engines/special/distributed.md) خواص. - -اگر شما فقط شروع به استفاده از تاتر, شما می توانید سناریوی زیر استفاده کنید: - -1. [فعالسازی](#enabling-access-control) کنترل دسترسی مبتنی بر مربع و مدیریت حساب برای `default` کاربر. -2. ورود زیر `default` حساب کاربری و ایجاد تمام کاربران مورد نیاز است. فراموش نکنید که برای ایجاد یک حساب کاربری مدیر (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`). -3. [محدود کردن مجوزها](settings/permissions-for-queries.md#permissions_for_queries) برای `default` کاربر و غیر فعال کردن کنترل دسترسی مبتنی بر مربع و مدیریت حساب. - -### خواص راه حل فعلی {#access-control-properties} - -- شما می توانید مجوز برای پایگاه داده ها و جداول اعطای حتی در صورتی که وجود ندارد. -- اگر یک جدول حذف شد, تمام امتیازات که به این جدول مطابقت لغو نمی. بنابراین, اگر یک جدول جدید بعد با همین نام ایجاد شده است تمام امتیازات تبدیل دوباره واقعی. برای لغو امتیازات مربوط به جدول حذف شده, شما نیاز به انجام, مثلا, `REVOKE ALL PRIVILEGES ON db.table FROM ALL` پرس و جو. -- هیچ تنظیمات طول عمر برای امتیازات وجود دارد. - -## حساب کاربری {#user-account-management} - -یک حساب کاربری یک نهاد دسترسی است که اجازه می دهد تا به اجازه کسی در خانه کلیک است. یک حساب کاربری شامل: - -- اطلاعات شناسایی. -- [امتیازات](../sql-reference/statements/grant.md#grant-privileges) که تعریف دامنه نمایش داده شد کاربر می تواند انجام دهد. -- میزبان که از اتصال به سرور کلیک مجاز است. -- نقش اعطا شده و به طور پیش فرض. -- تنظیمات با محدودیت های خود را که به طور پیش فرض در ورود کاربر اعمال می شود. -- اختصاص داده پروفایل تنظیمات. - -امتیازات به یک حساب کاربری را می توان با اعطا [GRANT](../sql-reference/statements/grant.md) پرس و جو و یا با اختصاص [نقش ها](#role-management). برای لغو امتیازات از یک کاربر, تاتر فراهم می کند [REVOKE](../sql-reference/statements/revoke.md) پرس و جو. به لیست امتیازات برای یک کاربر, استفاده از - [SHOW GRANTS](../sql-reference/statements/show.md#show-grants-statement) بیانیه. - -نمایش داده شد مدیریت: - -- [CREATE USER](../sql-reference/statements/create.md#create-user-statement) -- [ALTER USER](../sql-reference/statements/alter.md#alter-user-statement) -- [DROP USER](../sql-reference/statements/misc.md#drop-user-statement) -- [SHOW CREATE USER](../sql-reference/statements/show.md#show-create-user-statement) - -### تنظیمات استفاده {#access-control-settings-applying} - -تنظیمات را می توان با روش های مختلف تنظیم: برای یک حساب کاربری, در نقش اعطا و تنظیمات پروفایل خود را. در ورود کاربر, اگر یک محیط در اشخاص دسترسی های مختلف مجموعه, ارزش و محدودیتهای این تنظیم توسط اولویت های زیر اعمال می شود (از بالاتر به پایین تر): - -1. تنظیمات حساب کاربری. -2. تنظیمات نقش های پیش فرض حساب کاربری. اگر یک محیط در برخی از نقش ها تنظیم شده است, سپس سفارش از تنظیم استفاده تعریف نشده است. -3. تنظیمات در پروفایل تنظیمات اختصاص داده شده به یک کاربر و یا به نقش پیش فرض خود را. اگر یک محیط در برخی از پروفیل های مجموعه, سپس منظور از تنظیم استفاده از تعریف نشده است. -4. تنظیمات به طور پیش فرض به تمام سرور و یا از اعمال [نمایه پیشفرض](server-configuration-parameters/settings.md#default-profile). - -## نقش {#role-management} - -نقش یک ظرف برای اشخاص دسترسی است که می تواند به یک حساب کاربری اعطا شده است. - -نقش شامل: - -- [امتیازات](../sql-reference/statements/grant.md#grant-privileges) -- تنظیمات و محدودیت ها -- فهرست نقش های اعطا شده - -نمایش داده شد مدیریت: - -- [CREATE ROLE](../sql-reference/statements/create.md#create-role-statement) -- [ALTER ROLE](../sql-reference/statements/alter.md#alter-role-statement) -- [DROP ROLE](../sql-reference/statements/misc.md#drop-role-statement) -- [SET ROLE](../sql-reference/statements/misc.md#set-role-statement) -- [SET DEFAULT ROLE](../sql-reference/statements/misc.md#set-default-role-statement) -- [SHOW CREATE ROLE](../sql-reference/statements/show.md#show-create-role-statement) - -امتیازات به نقش را می توان با اعطا [GRANT](../sql-reference/statements/grant.md) پرس و جو. برای لغو امتیازات از یک فاحشه خانه نقش فراهم می کند [REVOKE](../sql-reference/statements/revoke.md) پرس و جو. - -## سیاست سطر {#row-policy-management} - -سیاست ردیف یک فیلتر است که تعریف می کند که یا ردیف برای یک کاربر و یا برای نقش در دسترس است. سیاست ردیف شامل فیلتر برای یک جدول خاص و لیستی از نقش ها و/یا کاربران که باید این سیاست ردیف استفاده کنید. - -نمایش داده شد مدیریت: - -- [CREATE ROW POLICY](../sql-reference/statements/create.md#create-row-policy-statement) -- [ALTER ROW POLICY](../sql-reference/statements/alter.md#alter-row-policy-statement) -- [DROP ROW POLICY](../sql-reference/statements/misc.md#drop-row-policy-statement) -- [SHOW CREATE ROW POLICY](../sql-reference/statements/show.md#show-create-row-policy-statement) - -## تنظیمات {#settings-profiles-management} - -مشخصات تنظیمات مجموعه ای از [تنظیمات](settings/index.md). مشخصات تنظیمات شامل تنظیمات و محدودیت, و لیستی از نقش ها و/یا کاربران که این سهمیه اعمال می شود. - -نمایش داده شد مدیریت: - -- [CREATE SETTINGS PROFILE](../sql-reference/statements/create.md#create-settings-profile-statement) -- [ALTER SETTINGS PROFILE](../sql-reference/statements/alter.md#alter-settings-profile-statement) -- [DROP SETTINGS PROFILE](../sql-reference/statements/misc.md#drop-settings-profile-statement) -- [SHOW CREATE SETTINGS PROFILE](../sql-reference/statements/show.md#show-create-settings-profile-statement) - -## سهمیه {#quotas-management} - -سهمیه محدودیت استفاده از منابع. ببینید [سهمیه](quotas.md). - -سهمیه شامل مجموعه ای از محدودیت برای برخی از مدت زمان, و لیستی از نقش ها و/و یا کاربران که باید این سهمیه استفاده. - -نمایش داده شد مدیریت: - -- [CREATE QUOTA](../sql-reference/statements/create.md#create-quota-statement) -- [ALTER QUOTA](../sql-reference/statements/alter.md#alter-quota-statement) -- [DROP QUOTA](../sql-reference/statements/misc.md#drop-quota-statement) -- [SHOW CREATE QUOTA](../sql-reference/statements/show.md#show-create-quota-statement) - -## فعال کردن کنترل دسترسی مبتنی بر مربع و مدیریت حساب {#enabling-access-control} - -- راه اندازی یک دایرکتوری برای ذخیره سازی تنظیمات. - - فروشگاه های کلیک دسترسی به تنظیمات نهاد در مجموعه پوشه در [_پوشه دستیابی](server-configuration-parameters/settings.md#access_control_path) پارامتر پیکربندی سرور. - -- فعال کردن گذاشتن محور کنترل دسترسی و مدیریت حساب برای حداقل یک حساب کاربری. - - به طور پیش فرض کنترل دسترسی مبتنی بر مربع و مدیریت حساب برای همه کاربران تبدیل شده است. شما نیاز به پیکربندی حداقل یک کاربر در `users.xml` فایل پیکربندی و اختصاص 1 به [مدیریت دسترسی](settings/settings-users.md#access_management-user-setting) تنظیمات. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/access_rights/) diff --git a/docs/fa/operations/backup.md b/docs/fa/operations/backup.md deleted file mode 100644 index 1dbe4d7c493..00000000000 --- a/docs/fa/operations/backup.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 49 -toc_title: "\u067E\u0634\u062A\u06CC\u0628\u0627\u0646 \u06AF\u06CC\u0631\u06CC \u062F\ - \u0627\u062F\u0647 \u0647\u0627" ---- - -# پشتیبان گیری داده ها {#data-backup} - -در حالی که [تکرار](../engines/table-engines/mergetree-family/replication.md) provides protection from hardware failures, it does not protect against human errors: accidental deletion of data, deletion of the wrong table or a table on the wrong cluster, and software bugs that result in incorrect data processing or data corruption. In many cases mistakes like these will affect all replicas. ClickHouse has built-in safeguards to prevent some types of mistakes — for example, by default [شما نمی توانید فقط جداول را با یک موتور ادغام مانند حاوی بیش از 50 گیگابایت داده رها کنید](https://github.com/ClickHouse/ClickHouse/blob/v18.14.18-stable/programs/server/config.xml#L322-L330). با این حال, این پادمان تمام موارد ممکن را پوشش نمی دهد و می تواند دور. - -به منظور به طور موثر کاهش خطاهای انسانی ممکن است, شما باید با دقت تهیه یک استراتژی برای پشتیبان گیری و بازیابی اطلاعات خود را **در پیش**. - -هر شرکت دارای منابع مختلف در دسترس و کسب و کار مورد نیاز, بنابراین هیچ راه حل جهانی برای پشتیبان گیری تاتر و بازیابی است که هر وضعیت مناسب وجود دارد. چه کار می کند برای یک گیگابایت از داده ها به احتمال زیاد برای ده ها پتابایت کار نمی کند. انواع روش های ممکن با جوانب مثبت و منفی خود را که در زیر مورد بحث وجود دارد. این یک ایده خوب برای استفاده از روش های مختلف به جای فقط یک به منظور جبران کاستی های مختلف خود است. - -!!! note "یادداشت" - به خاطر داشته باشید که اگر شما چیزی حمایت کردن و هرگز سعی در بازگرداندن, شانس هستند که بازگرداندن به درستی کار نمی کند زمانی که شما در واقع نیاز (یا حداقل طول خواهد کشید از کسب و کار می تواند تحمل). بنابراین هر روش پشتیبان گیری شما را انتخاب کنید, اطمینان حاصل کنید که به طور خودکار روند بازگرداندن و همچنین, و تمرین در یک خوشه محل انتخابی یدکی به طور منظم. - -## تکثیر اطلاعات منبع در جایی دیگر {#duplicating-source-data-somewhere-else} - -اغلب داده هایی که به خانه کلیک مصرف از طریق نوعی از صف مداوم تحویل داده, مانند [نمایی کافکا](https://kafka.apache.org). در این مورد ممکن است یک مجموعه اضافی از مشترکین را پیکربندی کنید که جریان داده های مشابه را می خواند در حالی که نوشته شده است کلیک کنید و جایی در ذخیره سازی سرد ذخیره کنید. اکثر شرکت ها در حال حاضر برخی از پیش فرض توصیه می شود ذخیره سازی سرد, که می تواند یک فروشگاه شی و یا یک فایل سیستم توزیع مانند [HDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html). - -## گزارشهای ویژه سیستم پرونده {#filesystem-snapshots} - -برخی از سیستم های فایل های محلی قابلیت عکس فوری (به عنوان مثال, [ZFS](https://en.wikipedia.org/wiki/ZFS)), اما ممکن است بهترین انتخاب برای خدمت نمایش داده شد زندگی می کنند. یک راه حل ممکن است برای ایجاد کپی های اضافی با این نوع از سیستم فایل و حذف از [توزیع شده](../engines/table-engines/special/distributed.md) جداول که برای استفاده `SELECT` نمایش داده شد. عکس های فوری در چنین کپی خواهد شد در دسترس از هر گونه نمایش داده شد که تغییر داده ها باشد. به عنوان یک جایزه, این کپی ممکن است تنظیمات سخت افزار خاص با دیسک های بیشتر متصل در هر سرور, خواهد بود که مقرون به صرفه. - -## تاتر-کپی {#clickhouse-copier} - -[تاتر-کپی](utilities/clickhouse-copier.md) یک ابزار همه کاره است که در ابتدا به جداول پتابایت به اندازه دوباره سفال ساخته شده است. همچنین می تواند برای تهیه پشتیبان و بازیابی اهداف استفاده شود زیرا به طور قابل اعتماد داده ها را بین جداول کلیک و خوشه ها کپی می کند. - -برای حجم کمتری از داده ها, ساده `INSERT INTO ... SELECT ...` به جداول از راه دور نیز ممکن است کار کند. - -## دستکاری با قطعات {#manipulations-with-parts} - -کلیک اجازه می دهد تا با استفاده از `ALTER TABLE ... FREEZE PARTITION ...` پرس و جو برای ایجاد یک کپی محلی از پارتیشن های جدول. این اجرا با استفاده از hardlinks به `/var/lib/clickhouse/shadow/` پوشه, بنابراین معمولا فضای دیسک اضافی برای داده های قدیمی مصرف نمی. نسخه های ایجاد شده از فایل ها توسط سرور کلیک هاوس انجام نمی شود, بنابراین شما فقط می توانید ترک وجود دارد: شما یک نسخه پشتیبان تهیه ساده است که هیچ سیستم خارجی اضافی نیاز ندارد, اما هنوز هم مستعد ابتلا به مشکلات سخت افزاری خواهد بود. به همین دلیل بهتر است از راه دور به مکان دیگری کپی کنید و سپس نسخه های محلی را حذف کنید. توزیع فایل سیستم ها و فروشگاه های شی هنوز هم یک گزینه خوب برای این, اما عادی فایل های پیوست شده سرور با ظرفیت به اندازه کافی بزرگ ممکن است کار و همچنین (در این مورد انتقال از طریق فایل سیستم شبکه و یا شاید رخ می دهد [درباره ما](https://en.wikipedia.org/wiki/Rsync)). - -برای کسب اطلاعات بیشتر در مورد نمایش داده شد مربوط به دستکاری پارتیشن, دیدن [تغییر مستندات](../sql-reference/statements/alter.md#alter_manipulations-with-partitions). - -یک ابزار شخص ثالث در دسترس است به طور خودکار این روش: [کلیک-پشتیبان گیری](https://github.com/AlexAkulov/clickhouse-backup). - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/backup/) diff --git a/docs/fa/operations/configuration-files.md b/docs/fa/operations/configuration-files.md deleted file mode 100644 index c1416100c6d..00000000000 --- a/docs/fa/operations/configuration-files.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 50 -toc_title: "\u067E\u0631\u0648\u0646\u062F\u0647\u0647\u0627\u06CC \u067E\u06CC\u06A9\ - \u0631\u0628\u0646\u062F\u06CC" ---- - -# پروندههای پیکربندی {#configuration_files} - -تاتر پشتیبانی از مدیریت پیکربندی چند فایل. فایل پیکربندی سرور اصلی است `/etc/clickhouse-server/config.xml`. فایل های دیگر باید در `/etc/clickhouse-server/config.d` فهرست راهنما. - -!!! note "یادداشت" - تمام فایل های پیکربندی باید در فرمت میلی لیتر باشد. همچنین معمولا باید یک عنصر ریشه داشته باشند ``. - -برخی از تنظیمات مشخص شده در فایل پیکربندی اصلی را می توان در دیگر فایل های پیکربندی باطل. این `replace` یا `remove` صفات را می توان برای عناصر این فایل های پیکربندی مشخص شده است. - -اگر نه مشخص شده است, ترکیبی از محتویات عناصر به صورت بازگشتی, جایگزینی مقادیر کودکان تکراری. - -اگر `replace` مشخص شده است, جایگزین کل عنصر با یک مشخص. - -اگر `remove` مشخص شده است, حذف عنصر. - -پیکربندی همچنین می توانید تعریف “substitutions”. اگر یک عنصر است `incl` ویژگی, جایگزینی مربوطه را از فایل خواهد شد به عنوان ارزش استفاده. به طور پیش فرض, مسیر به فایل با تعویض است `/etc/metrika.xml`. این را می توان در تغییر [شامل _فروم](server-configuration-parameters/settings.md#server_configuration_parameters-include_from) عنصر در پیکربندی سرور. مقادیر جایگزینی در مشخص `/yandex/substitution_name` عناصر در این فایل. اگر جایگزینی مشخص شده در `incl` وجود ندارد, این است که در ورود به سیستم ثبت. برای جلوگیری از جایگزینی ورود به سیستم کلیک کنید `optional="true"` ویژگی (مثلا, تنظیمات برای [& کلاندارها](server-configuration-parameters/settings.md)). - -تعویض همچنین می توانید از باغ وحش انجام شود. برای انجام این کار ویژگی را مشخص کنید `from_zk = "/path/to/node"`. مقدار عنصر با محتویات گره در جایگزین `/path/to/node` در باغ وحش. شما همچنین می توانید یک زیر درخت کل در گره باغ وحش قرار داده و به طور کامل به عنصر منبع وارد می شود. - -این `config.xml` فایل می تواند یک پیکربندی جداگانه با تنظیمات کاربر مشخص, پروفایل, و سهمیه. مسیر نسبی به این پیکربندی در مجموعه `users_config` عنصر. به طور پیش فرض است `users.xml`. اگر `users_config` حذف شده است, تنظیمات کاربر, پروفایل, و سهمیه به طور مستقیم در مشخص `config.xml`. - -پیکربندی کاربران را می توان به فایل های جداگانه شبیه به تقسیم `config.xml` و `config.d/`. -نام فهرست راهنما به نام `users_config` تنظیم بدون `.xml` پس از مخلوط با `.d`. -فهرست راهنما `users.d` به طور پیش فرض استفاده می شود, مانند `users_config` پیشفرضها به `users.xml`. -مثلا, شما می توانید فایل پیکربندی جداگانه برای هر کاربر مثل این دارند: - -``` bash -$ cat /etc/clickhouse-server/users.d/alice.xml -``` - -``` xml - - - - analytics - - ::/0 - - ... - analytics - - - -``` - -برای هر فایل پیکربندی سرور نیز تولید می کند `file-preprocessed.xml` فایل در هنگام شروع. این فایل ها شامل تمام تعویض های تکمیل شده و لغو می شوند و برای استفاده اطلاعاتی در نظر گرفته می شوند. اگر تعویض باغ وحش در فایل های پیکربندی مورد استفاده قرار گرفت اما باغ وحش در دسترس بر روی شروع سرور نیست, سرور بارهای پیکربندی از فایل پیش پردازش. - -مسیر سرور تغییر در فایل های پیکربندی, و همچنین فایل ها و گره باغ وحش که در هنگام انجام تعویض و لغو مورد استفاده قرار گرفت, و بارگذاری مجدد تنظیمات برای کاربران و خوشه در پرواز. این به این معنی است که شما می توانید خوشه تغییر, کاربران, و تنظیمات خود را بدون راه اندازی مجدد سرور. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/configuration_files/) diff --git a/docs/fa/operations/index.md b/docs/fa/operations/index.md deleted file mode 100644 index 22fcfb0fa38..00000000000 --- a/docs/fa/operations/index.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0639\u0645\u0644\u06CC\u0627\u062A" -toc_priority: 41 -toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" ---- - -# عملیات {#operations} - -کتابچه راهنمای عملیات کلیک متشکل از بخش های اصلی زیر است: - -- [الزامات](requirements.md) -- [نظارت](monitoring.md) -- [عیب یابی](troubleshooting.md) -- [توصیه های استفاده](tips.md) -- [روش به روز رسانی](update.md) -- [حقوق دسترسی](access-rights.md) -- [پشتیبان گیری داده ها](backup.md) -- [پروندههای پیکربندی](configuration-files.md) -- [سهمیه](quotas.md) -- [جداول سیستم](system-tables.md) -- [پارامترهای پیکربندی سرور](server-configuration-parameters/index.md) -- [چگونه برای تست سخت افزار خود را با کلیک](performance-test.md) -- [تنظیمات](settings/index.md) -- [تاسیسات](utilities/index.md) - -{## [مقاله اصلی](https://clickhouse.tech/docs/en/operations/) ##} diff --git a/docs/fa/operations/monitoring.md b/docs/fa/operations/monitoring.md deleted file mode 100644 index 48baa568952..00000000000 --- a/docs/fa/operations/monitoring.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 45 -toc_title: "\u0646\u0638\u0627\u0631\u062A" ---- - -# نظارت {#monitoring} - -شما می توانید نظارت: - -- استفاده از منابع سخت افزاری. -- معیارهای سرور کلیک. - -## استفاده از منابع {#resource-utilization} - -کلیک می کند دولت از منابع سخت افزاری به خودی خود نظارت نیست. - -این است که به شدت توصیه می شود به راه اندازی نظارت برای: - -- بار و درجه حرارت در پردازنده. - - شما می توانید استفاده کنید [راهنمایی و رانندگی](https://en.wikipedia.org/wiki/Dmesg), [توربوستات](https://www.linux.org/docs/man8/turbostat.html) و یا ابزار های دیگر. - -- استفاده از سیستم ذخیره سازی, رم و شبکه. - -## معیارهای سرور کلیک {#clickhouse-server-metrics} - -سرور کلیک ابزار برای نظارت خود دولت تعبیه شده است. - -برای پیگیری رویدادهای سرور استفاده از سیاهههای مربوط به سرور. دیدن [چوبگر](server-configuration-parameters/settings.md#server_configuration_parameters-logger) بخش از فایل پیکربندی. - -جمعهای کلیک: - -- معیارهای مختلف چگونه سرور با استفاده از منابع محاسباتی. -- ارقام مشترک در پردازش پرس و جو. - -شما می توانید معیارهای موجود در [سیستم.متریک](../operations/system-tables.md#system_tables-metrics), [سیستم.رویدادها](../operations/system-tables.md#system_tables-events) و [سیستم._نامهنویسی ناهمزمان](../operations/system-tables.md#system_tables-asynchronous_metrics) میز - -شما می توانید کلیک کنید هاوس به صادرات معیارهای به پیکربندی کنید [گرافیت](https://github.com/graphite-project). دیدن [بخش گرافیت](server-configuration-parameters/settings.md#server_configuration_parameters-graphite) در فایل پیکربندی سرور کلیک. قبل از پیکربندی صادرات معیارهای, شما باید راه اندازی گرافیت با پیروی از رسمی خود را [راهنما](https://graphite.readthedocs.io/en/latest/install.html). - -شما می توانید کلیک کنید هاوس به صادرات معیارهای به پیکربندی کنید [پرومتیوس](https://prometheus.io). دیدن [بخش پرومته](server-configuration-parameters/settings.md#server_configuration_parameters-prometheus) در فایل پیکربندی سرور کلیک. قبل از پیکربندی صادرات معیارهای, شما باید راه اندازی پرومته با پیروی از رسمی خود [راهنما](https://prometheus.io/docs/prometheus/latest/installation/). - -علاوه بر این, شما می توانید در دسترس بودن سرور از طریق صفحه اصلی نظارت. ارسال `HTTP GET` درخواست برای `/ping`. اگر سرور در دسترس است, با پاسخ `200 OK`. - -برای نظارت بر سرور در یک پیکربندی خوشه, شما باید مجموعه ای از [_شروع مجدد _شروع مجدد _شروع مجدد _کاربری](settings/settings.md#settings-max_replica_delay_for_distributed_queries) پارامتر و استفاده از منبع قام `/replicas_status`. یک درخواست برای `/replicas_status` بازگشت `200 OK` اگر ماکت در دسترس است و در پشت کپی دیگر به تعویق افتاد. اگر یک ماکت به تاخیر افتاد, باز می گردد `503 HTTP_SERVICE_UNAVAILABLE` با اطلاعات در مورد شکاف. diff --git a/docs/fa/operations/optimizing-performance/index.md b/docs/fa/operations/optimizing-performance/index.md deleted file mode 100644 index 92223aee0e4..00000000000 --- a/docs/fa/operations/optimizing-performance/index.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0628\u0647\u06CC\u0646\u0647 \u0633\u0627\u0632\u06CC \u0639\u0645\ - \u0644\u06A9\u0631\u062F" -toc_priority: 52 ---- - - diff --git a/docs/fa/operations/optimizing-performance/sampling-query-profiler.md b/docs/fa/operations/optimizing-performance/sampling-query-profiler.md deleted file mode 100644 index 3573c5e0806..00000000000 --- a/docs/fa/operations/optimizing-performance/sampling-query-profiler.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 54 -toc_title: "\u067E\u0631\u0648\u0641\u0627\u06CC\u0644 \u067E\u0631\u0633 \u0648 \u062C\ - \u0648" ---- - -# پروفایل پرس و جو نمونه برداری {#sampling-query-profiler} - -فاحشه خانه اجرا می شود نمونه برداری پیشفیلتر که اجازه می دهد تجزیه و تحلیل اجرای پرس و جو. با استفاده از نیمرخ شما می توانید روال کد منبع که اغلب در طول اجرای پرس و جو استفاده پیدا. شما می توانید زمان پردازنده و دیوار ساعت زمان صرف شده از جمله زمان بیکار ردیابی. - -برای استفاده از پروفیل: - -- برپایی [_قطع](../server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) بخش پیکربندی سرور. - - در این بخش پیکربندی [_قطع](../../operations/system-tables.md#system_tables-trace_log) جدول سیستم حاوی نتایج حاصل از عملکرد پیشفیلتر. این است که به طور پیش فرض پیکربندی شده است. به یاد داشته باشید که داده ها در این جدول تنها برای یک سرور در حال اجرا معتبر است. پس از راه اندازی مجدد سرور, تاتر تمیز نمی کند تا جدول و تمام نشانی حافظه مجازی ذخیره شده ممکن است نامعتبر. - -- برپایی [ایران در تهران](../settings/settings.md#query_profiler_cpu_time_period_ns) یا [جستجو](../settings/settings.md#query_profiler_real_time_period_ns) تنظیمات. هر دو تنظیمات را می توان به طور همزمان استفاده کرد. - - این تنظیمات به شما اجازه پیکربندی تایمر پیشفیلتر. همانطور که این تنظیمات جلسه هستند, شما می توانید فرکانس نمونه برداری های مختلف برای کل سرور از, کاربران فردی و یا پروفایل های کاربر, برای جلسه تعاملی خود را, و برای هر پرس و جو فردی. - -فرکانس نمونه گیری به طور پیش فرض یک نمونه در هر ثانیه است و هر دو پردازنده و تایمر واقعی را فعال کنید. این فرکانس اجازه می دهد تا اطلاعات کافی در مورد خوشه کلیک کنید. همزمان, کار با این فرکانس, پیشفیلتر می کند عملکرد سرور کلیک را تحت تاثیر قرار نمی. اگر شما نیاز به مشخصات هر پرس و جو فردی سعی کنید به استفاده از فرکانس نمونه برداری بالاتر است. - -برای تجزیه و تحلیل `trace_log` جدول سیستم: - -- نصب `clickhouse-common-static-dbg` بسته ببینید [نصب از بسته های دب](../../getting-started/install.md#install-from-deb-packages). - -- اجازه توابع درون گرایی توسط [اجازه دادن به _فعال کردن اختلال در عملکرد](../settings/settings.md#settings-allow_introspection_functions) تنظیمات. - - به دلایل امنیتی, توابع درون گرایی به طور پیش فرض غیر فعال. - -- استفاده از `addressToLine`, `addressToSymbol` و `demangle` [توابع درون گرایی](../../sql-reference/functions/introspection.md) برای گرفتن نام تابع و موقعیت خود را در کد کلیک کنید. برای دریافت یک پروفایل برای برخی از پرس و جو, شما نیاز به جمع داده ها از `trace_log` جدول شما می توانید داده ها را با توابع فردی یا کل ردیابی پشته جمع کنید. - -اگر شما نیاز به تجسم `trace_log` اطلاعات را امتحان کنید [شق](../../interfaces/third-party/gui/#clickhouse-flamegraph) و [سرعت سنج](https://github.com/laplab/clickhouse-speedscope). - -## مثال {#example} - -در این مثال ما: - -- پالایش `trace_log` داده ها توسط یک شناسه پرس و جو و تاریخ جاری. - -- جمع توسط ردیابی پشته. - -- با استفاده از توابع درون گرایی, ما یک گزارش از دریافت: - - - نام نمادها و توابع کد منبع مربوطه. - - محل کد منبع از این توابع. - - - -``` sql -SELECT - count(), - arrayStringConcat(arrayMap(x -> concat(demangle(addressToSymbol(x)), '\n ', addressToLine(x)), trace), '\n') AS sym -FROM system.trace_log -WHERE (query_id = 'ebca3574-ad0a-400a-9cbc-dca382f5998c') AND (event_date = today()) -GROUP BY trace -ORDER BY count() DESC -LIMIT 10 -``` - -``` text -{% include "examples/sampling_query_profiler_result.txt" %} -``` diff --git a/docs/fa/operations/performance-test.md b/docs/fa/operations/performance-test.md deleted file mode 100644 index 4bd5cc2c15f..00000000000 --- a/docs/fa/operations/performance-test.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 54 -toc_title: "\u0633\u062E\u062A \u0627\u0641\u0632\u0627\u0631 \u062A\u0633\u062A" ---- - -# چگونه برای تست سخت افزار خود را با کلیک {#how-to-test-your-hardware-with-clickhouse} - -با این آموزش شما می توانید اجرا پایه ClickHouse آزمون عملکرد بر روی هر سرور بدون نصب و راه اندازی ClickHouse بسته است. - -1. برو به “commits” صفحه: https://github.com/ClickHouse/ClickHouse/commits/master - -2. با کلیک بر روی اولین علامت چک سبز یا صلیب قرمز با سبز “ClickHouse Build Check” و با کلیک بر روی “Details” لینک نزدیک “ClickHouse Build Check”. چنین لینک در برخی از مرتکب وجود دارد, برای مثال مرتکب با اسناد و مدارک. در این مورد, را انتخاب کنید نزدیکترین ارتکاب داشتن این لینک. - -3. رونوشت از پیوند به “clickhouse” دودویی برای amd64 یا aarch64. - -4. به سرور بروید و با استفاده از ابزار دانلود کنید: - - - - # For amd64: - wget https://clickhouse-builds.s3.yandex.net/0/00ba767f5d2a929394ea3be193b1f79074a1c4bc/1578163263_binary/clickhouse - # For aarch64: - wget https://clickhouse-builds.s3.yandex.net/0/00ba767f5d2a929394ea3be193b1f79074a1c4bc/1578161264_binary/clickhouse - # Then do: - chmod a+x clickhouse - -1. دانلود تنظیمات: - - - - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.xml - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/users.xml - mkdir config.d - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/path.xml -O config.d/path.xml - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/log_to_console.xml -O config.d/log_to_console.xml - -1. دانلود فایل معیار: - - - - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/benchmark-new.sh - chmod a+x benchmark-new.sh - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/queries.sql - -1. دانلود داده های تست با توجه به [یاندکسمجموعه داده های متریکا](../getting-started/example-datasets/metrica.md) دستورالعمل (“hits” جدول حاوی 100 میلیون ردیف). - - - - wget https://datasets.clickhouse.tech/hits/partitions/hits_100m_obfuscated_v1.tar.xz - tar xvf hits_100m_obfuscated_v1.tar.xz -C . - mv hits_100m_obfuscated_v1/* . - -1. اجرای کارساز: - - - - ./clickhouse server - -1. داده ها را بررسی کنید: در ترمینال دیگر به سرور مراجعه کنید - - - - ./clickhouse client --query "SELECT count() FROM hits_100m_obfuscated" - 100000000 - -1. ویرایش benchmark-new.sh تغییر `clickhouse-client` به `./clickhouse client` و اضافه کردن `–-max_memory_usage 100000000000` پارامتر. - - - - mcedit benchmark-new.sh - -1. اجرای معیار: - - - - ./benchmark-new.sh hits_100m_obfuscated - -1. ارسال اعداد و اطلاعات در مورد پیکربندی سخت افزار خود را به clickhouse-feedback@yandex-team.com - -همه نتایج در اینجا منتشر شده: https://clickhouse.فناوری / معیار / سخت افزار/ diff --git a/docs/fa/operations/quotas.md b/docs/fa/operations/quotas.md deleted file mode 100644 index 63d77aafa84..00000000000 --- a/docs/fa/operations/quotas.md +++ /dev/null @@ -1,112 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 51 -toc_title: "\u0633\u0647\u0645\u06CC\u0647" ---- - -# سهمیه {#quotas} - -سهمیه به شما اجازه محدود کردن استفاده از منابع بیش از یک دوره از زمان و یا پیگیری استفاده از منابع. -سهمیه در پیکربندی کاربر راه اندازی, که معمولا ‘users.xml’. - -این سیستم همچنین دارای یک ویژگی برای محدود کردن پیچیدگی یک پرس و جو واحد. بخش را ببینید “Restrictions on query complexity”). - -در مقابل به پرس و جو محدودیت پیچیدگی, سهمیه: - -- محل محدودیت در مجموعه ای از نمایش داده شد که می تواند بیش از یک دوره از زمان اجرا, به جای محدود کردن یک پرس و جو. -- حساب برای منابع صرف شده در تمام سرور از راه دور برای پردازش پرس و جو توزیع شده است. - -بیایید به بخش ‘users.xml’ فایل که سهمیه را تعریف می کند. - -``` xml - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - -``` - -به طور پیش فرض, سهمیه ردیابی مصرف منابع برای هر ساعت, بدون محدود کردن استفاده. -مصرف منابع محاسبه شده برای هر فاصله خروجی به ورود به سیستم سرور بعد از هر درخواست است. - -``` xml - - - - - 3600 - - 1000 - 100 - 1000000000 - 100000000000 - 900 - - - - 86400 - - 10000 - 1000 - 5000000000 - 500000000000 - 7200 - - -``` - -برای ‘statbox’ سهمیه, محدودیت برای هر ساعت و برای هر مجموعه 24 ساعت ها (86,400 ثانیه). فاصله زمانی شمارش شده است, با شروع از یک لحظه ثابت پیاده سازی تعریف شده در زمان. به عبارت دیگر فاصله 24 ساعته لزوما در نیمه شب شروع نمی شود. - -هنگامی که فاصله به پایان می رسد تمام مقادیر جمع شده پاک می شوند. برای ساعت بعد محاسبه سهمیه بیش از شروع می شود. - -در اینجا مقدار است که می تواند محدود می شود: - -`queries` – The total number of requests. - -`errors` – The number of queries that threw an exception. - -`result_rows` – The total number of rows given as a result. - -`read_rows` – The total number of source rows read from tables for running the query on all remote servers. - -`execution_time` – The total query execution time, in seconds (wall time). - -اگر حد برای حداقل یک فاصله زمانی بیش از, یک استثنا با یک متن که در مورد محدودیت بیش از حد شد پرتاب, که فاصله, و هنگامی که فاصله جدید شروع می شود (هنگامی که نمایش داده شد را می توان دوباره ارسال). - -سهمیه می توانید استفاده کنید “quota key” ویژگی به گزارش منابع برای کلید های متعدد به طور مستقل. در اینجا یک مثال از این است: - -``` xml - - - - -``` - -سهمیه به کاربران در اختصاص داده ‘users’ بخش پیکربندی. بخش را ببینید “Access rights”. - -برای پردازش پرس و جو توزیع, مقدار انباشته شده بر روی سرور درخواست ذخیره می شود. بنابراین اگر کاربر می رود به سرور دیگر, سهمیه وجود خواهد داشت “start over”. - -هنگامی که سرور دوباره راه اندازی شده است, سهمیه تنظیم مجدد. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/quotas/) diff --git a/docs/fa/operations/requirements.md b/docs/fa/operations/requirements.md deleted file mode 100644 index f116f59208b..00000000000 --- a/docs/fa/operations/requirements.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 44 -toc_title: "\u0627\u0644\u0632\u0627\u0645\u0627\u062A" ---- - -# الزامات {#requirements} - -## CPU {#cpu} - -برای نصب و راه اندازی از بسته های پیش ساخته دب, استفاده از یک پردازنده با معماری ایکس86_64 و پشتیبانی برای سوس 4.2 دستورالعمل. برای اجرای ClickHouse با پردازنده های که پشتیبانی نمی کند SSE 4.2 یا AArch64 یا PowerPC64LE معماری شما باید ساخت ClickHouse از منابع. - -تاتر پیاده سازی پردازش داده های موازی و با استفاده از تمام منابع سخت افزاری در دسترس. در هنگام انتخاب یک پردازنده, را به حساب که فاحشه خانه کار می کند موثر تر در تنظیمات با تعداد زیادی از هسته اما نرخ ساعت پایین تر از در تنظیمات با هسته کمتر و نرخ ساعت بالاتر. مثلا, 16 هسته با 2600 مگاهرتز بهتر از است 8 هسته با 3600 مگاهرتز. - -توصیه می شود برای استفاده **افزایش توربو** و **بیش از حد نخ** تکنولوژیها. این به طور قابل توجهی بهبود عملکرد با حجم کار معمولی. - -## RAM {#ram} - -ما توصیه می کنیم با استفاده از حداقل 4 گیگابایت رم برای انجام نمایش داده شد غیر بدیهی است. سرور کلیک می توانید با مقدار بسیار کوچکتر از رم اجرا, اما نیاز به حافظه برای پردازش نمایش داده شد. - -حجم مورد نیاز رم بستگی دارد: - -- پیچیدگی نمایش داده شد. -- مقدار داده هایی که در نمایش داده شد پردازش شده است. - -برای محاسبه حجم مورد نیاز رم, شما باید اندازه داده های موقت برای تخمین [GROUP BY](../sql-reference/statements/select/group-by.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select/distinct.md#select-distinct), [JOIN](../sql-reference/statements/select/join.md#select-join) و عملیات دیگر استفاده می کنید. - -تاتر می توانید حافظه خارجی برای داده های موقت استفاده. ببینید [گروه در حافظه خارجی](../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory) برای اطلاعات بیشتر. - -## تعویض پرونده {#swap-file} - -غیر فعال کردن فایل مبادله برای محیط های تولید. - -## زیرسیستم ذخیره سازی {#storage-subsystem} - -شما باید 2 گیگابایت فضای دیسک رایگان برای نصب کلیک کنید. - -حجم ذخیره سازی مورد نیاز برای داده های خود را باید به طور جداگانه محاسبه می شود. ارزیابی باید شامل موارد زیر باشد: - -- تخمین حجم داده ها. - - شما می توانید یک نمونه از داده ها و اندازه متوسط یک ردیف از. سپس مقدار ضرب شده توسط تعدادی از ردیف شما برنامه ای برای ذخیره. - -- ضریب فشرده سازی داده ها. - - برای تخمین ضریب فشرده سازی داده ها, بار یک نمونه از داده های خود را به خانه, و مقایسه اندازه واقعی از داده ها با اندازه جدول ذخیره شده. مثلا, داده های کلیک استریم است که معمولا توسط فشرده 6-10 بار. - -برای محاسبه حجم نهایی داده ها ذخیره می شود, اعمال ضریب فشرده سازی به حجم داده های تخمین زده شده. اگر شما قصد دارید برای ذخیره داده ها در چند کپی, سپس ضرب حجم تخمین زده شده توسط تعدادی از کپی. - -## شبکه {#network} - -در صورت امکان از شبکه های 10 گرم یا کلاس بالاتر استفاده کنید. - -پهنای باند شبکه برای پردازش نمایش داده شد توزیع با مقدار زیادی از داده های متوسط بسیار مهم است. علاوه بر این, سرعت شبکه را تحت تاثیر قرار فرایندهای تکرار. - -## نرم افزار {#software} - -کلیک خانه در درجه اول برای خانواده لینوکس از سیستم عامل توسعه یافته است. توزیع لینوکس توصیه شده اوبونتو است. این `tzdata` بسته باید در سیستم نصب شود. - -تاتر همچنین می توانید در دیگر خانواده سیستم عامل کار. مشاهده اطلاعات در [شروع کار](../getting-started/index.md) بخش از اسناد و مدارک. diff --git a/docs/fa/operations/server-configuration-parameters/index.md b/docs/fa/operations/server-configuration-parameters/index.md deleted file mode 100644 index 94b493419d4..00000000000 --- a/docs/fa/operations/server-configuration-parameters/index.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u067E\u0627\u0631\u0627\u0645\u062A\u0631\u0647\u0627\u06CC \u067E\ - \u06CC\u06A9\u0631\u0628\u0646\u062F\u06CC \u0633\u0631\u0648\u0631" -toc_priority: 54 -toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" ---- - -# پارامترهای پیکربندی سرور {#server-settings} - -این بخش شامل شرح تنظیمات سرور است که نمی تواند در سطح جلسه یا پرس و جو تغییر کند. - -این تنظیمات در ذخیره می شود `config.xml` فایل بر روی سرور کلیک. - -تنظیمات دیگر در توصیف “[تنظیمات](../settings/index.md#session-settings-intro)” بخش. - -قبل از مطالعه تنظیمات, خواندن [پروندههای پیکربندی](../configuration-files.md#configuration_files) بخش و توجه داشته باشید استفاده از تعویض ( `incl` و `optional` صفات). - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/server_configuration_parameters/) diff --git a/docs/fa/operations/server-configuration-parameters/settings.md b/docs/fa/operations/server-configuration-parameters/settings.md deleted file mode 100644 index b57a9d1bfa8..00000000000 --- a/docs/fa/operations/server-configuration-parameters/settings.md +++ /dev/null @@ -1,907 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 57 -toc_title: "\u062A\u0646\u0638\u06CC\u0645\u0627\u062A \u06A9\u0627\u0631\u06AF\u0632\ - \u0627\u0631" ---- - -# تنظیمات کارگزار {#server-settings} - -## ساختن و احراز هویت اکانتهای دستگاه {#builtin-dictionaries-reload-interval} - -فاصله در ثانیه قبل از بارگذاری ساخته شده است در لغت نامه. - -مخزن بارگذاری مجدد ساخته شده است در لغت نامه در هر ثانیه ایکس. این امکان ویرایش واژهنامهها را فراهم میکند “on the fly” بدون راه اندازی مجدد سرور. - -مقدار پیش فرض: 3600. - -**مثال** - -``` xml -3600 -``` - -## فشردهسازی {#server-settings-compression} - -تنظیمات فشرده سازی داده ها برای [ادغام](../../engines/table-engines/mergetree-family/mergetree.md)- جدول موتور . - -!!! warning "اخطار" - اگر شما فقط شروع به استفاده از خانه کلیک استفاده نکنید. - -قالب پیکربندی: - -``` xml - - - ... - ... - ... - - ... - -``` - -`` زمینه: - -- `min_part_size` – The minimum size of a data part. -- `min_part_size_ratio` – The ratio of the data part size to the table size. -- `method` – Compression method. Acceptable values: `lz4` یا `zstd`. - -شما می توانید چند پیکربندی کنید `` بخش. - -اقدامات زمانی که شرایط ملاقات می شوند: - -- اگر بخشی از داده ها منطبق یک مجموعه شرایط, تاتر با استفاده از روش فشرده سازی مشخص. -- اگر یک بخش داده منطبق مجموعه شرایط متعدد, خانه رعیتی با استفاده از اولین مجموعه شرایط همسان. - -اگر هیچ شرایطی برای یک بخش داده ملاقات, خانه عروسکی با استفاده از `lz4` فشردهسازی. - -**مثال** - -``` xml - - - 10000000000 - 0.01 - zstd - - -``` - -## & تنظیمات {#default-database} - -پایگاه داده به طور پیش فرض. - -برای دریافت یک لیست از پایگاه داده, استفاده از [SHOW DATABASES](../../sql-reference/statements/show.md#show-databases) پرس و جو. - -**مثال** - -``` xml -default -``` - -## قصور {#default-profile} - -تنظیمات پیش فرض مشخصات. - -پروفایل های تنظیمات در فایل مشخص شده در پارامتر واقع شده است `user_config`. - -**مثال** - -``` xml -default -``` - -## دیکشنامهای {#server_configuration_parameters-dictionaries_config} - -مسیر به فایل پیکربندی برای لغت نامه های خارجی. - -مسیر: - -- مشخص کردن مسیر مطلق و یا مسیر نسبت به فایل پیکربندی سرور. -- مسیر می تواند حاوی نویسه عام \* و?. - -همچنین نگاه کنید به “[واژهنامهها خارجی](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md)”. - -**مثال** - -``` xml -*_dictionary.xml -``` - -## _بارگیری کامل {#server_configuration_parameters-dictionaries_lazy_load} - -بارگذاری تنبل از لغت نامه. - -اگر `true` سپس هر فرهنگ لغت در اولین استفاده ایجاد می شود. اگر ایجاد فرهنگ لغت شکست خورده, تابع بود که با استفاده از فرهنگ لغت می اندازد یک استثنا. - -اگر `false`, تمام لغت نامه ها ایجاد می شوند زمانی که سرور شروع می شود, و اگر یک خطا وجود دارد, سرور خاموش. - -به طور پیش فرض است `true`. - -**مثال** - -``` xml -true -``` - -## قالب_شکلمات شیمی {#server_configuration_parameters-format_schema_path} - -مسیر به دایرکتوری با طرح برای داده های ورودی, مانند طرحواره برای [کاپپروتو](../../interfaces/formats.md#capnproto) قالب. - -**مثال** - -``` xml - - format_schemas/ -``` - -## گرافیت {#server_configuration_parameters-graphite} - -ارسال داده به [گرافیت](https://github.com/graphite-project). - -تنظیمات: - -- host – The Graphite server. -- port – The port on the Graphite server. -- interval – The interval for sending, in seconds. -- timeout – The timeout for sending data, in seconds. -- root_path – Prefix for keys. -- metrics – Sending data from the [سیستم.متریک](../../operations/system-tables.md#system_tables-metrics) جدول -- events – Sending deltas data accumulated for the time period from the [سیستم.رویدادها](../../operations/system-tables.md#system_tables-events) جدول -- events_cumulative – Sending cumulative data from the [سیستم.رویدادها](../../operations/system-tables.md#system_tables-events) جدول -- asynchronous_metrics – Sending data from the [سیستم._نامهنویسی ناهمزمان](../../operations/system-tables.md#system_tables-asynchronous_metrics) جدول - -شما می توانید چند پیکربندی کنید `` بند. برای مثال شما می توانید از این برای ارسال داده های مختلف در فواصل مختلف استفاده کنید. - -**مثال** - -``` xml - - localhost - 42000 - 0.1 - 60 - one_min - true - true - false - true - -``` - -## لغزش _ نمودار {#server_configuration_parameters-graphite-rollup} - -تنظیمات برای نازک شدن داده ها برای گرافیت. - -برای اطلاعات بیشتر, دیدن [نمودار](../../engines/table-engines/mergetree-family/graphitemergetree.md). - -**مثال** - -``` xml - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - -``` - -## _وارد کردن/پشتیبانی {#http-porthttps-port} - -درگاه برای اتصال به کارساز بالای صفحه) ها (. - -اگر `https_port` مشخص شده است, [openSSL](#server_configuration_parameters-openssl) باید پیکربندی شود. - -اگر `http_port` مشخص شده است, پیکربندی اپنسسل نادیده گرفته شده است حتی اگر قرار است. - -**مثال** - -``` xml -9999 -``` - -## نقلقولهای جدید از این نویسنده {#server_configuration_parameters-http_server_default_response} - -صفحه ای که به طور پیش فرض نشان داده شده است زمانی که شما دسترسی به سرور قام کلیک. -مقدار پیش فرض است “Ok.” (با خوراک خط در پایان) - -**مثال** - -باز می شود `https://tabix.io/` هنگام دسترسی `http://localhost: http_port`. - -``` xml - -
]]> - -``` - -## شامل _فروم {#server_configuration_parameters-include_from} - -مسیر به فایل با تعویض. - -برای کسب اطلاعات بیشتر به بخش مراجعه کنید “[پروندههای پیکربندی](../configuration-files.md#configuration_files)”. - -**مثال** - -``` xml -/etc/metrica.xml -``` - -## _صادر کردن {#interserver-http-port} - -پورت برای تبادل اطلاعات بین سرور های فاحشه خانه. - -**مثال** - -``` xml -9009 -``` - -## حذف جستجو {#interserver-http-host} - -نام میزبان است که می تواند توسط سرور های دیگر برای دسترسی به این سرور استفاده می شود. - -اگر حذف, این است که در همان راه به عنوان تعریف `hostname-f` فرمان. - -مفید برای شکستن دور از یک رابط شبکه خاص. - -**مثال** - -``` xml -example.yandex.ru -``` - -## پتانسیلهای متقابل {#server-settings-interserver-http-credentials} - -نام کاربری و رمز عبور مورد استفاده برای تصدیق در طول [تکرار](../../engines/table-engines/mergetree-family/replication.md) با تکرار \* موتورهای. این اعتبار تنها برای ارتباط بین کپی استفاده می شود و ربطی به اعتبار برای مشتریان خانه عروسکی هستند. سرور چک کردن این اعتبار برای اتصال کپی و استفاده از اعتبار همان هنگام اتصال به دیگر کپی. بنابراین, این اعتبار باید همین کار را برای همه کپی در یک خوشه مجموعه. -به طور پیش فرض احراز هویت استفاده نمی شود. - -این بخش شامل پارامترهای زیر است: - -- `user` — username. -- `password` — password. - -**مثال** - -``` xml - - admin - 222 - -``` - -## حفاظت از حریم خصوصی {#keep-alive-timeout} - -تعداد ثانیه که تاتر منتظر درخواست های دریافتی قبل از بستن اتصال. به طور پیش فرض به 3 ثانیه. - -**مثال** - -``` xml -3 -``` - -## _نوست فهرست {#server_configuration_parameters-listen_host} - -محدودیت در میزبان که درخواست می توانید از. اگر می خواهید سرور برای پاسخ به همه انها مشخص شود `::`. - -مثالها: - -``` xml -::1 -127.0.0.1 -``` - -## چوبگر {#server_configuration_parameters-logger} - -تنظیمات ورود به سیستم. - -کلید: - -- level – Logging level. Acceptable values: `trace`, `debug`, `information`, `warning`, `error`. -- log – The log file. Contains all the entries according to `level`. -- errorlog – Error log file. -- size – Size of the file. Applies to `log`و`errorlog`. هنگامی که فایل می رسد `size`, بایگانی کلیک هوس و تغییر نام, و ایجاد یک فایل ورود به سیستم جدید را در خود جای. -- count – The number of archived log files that ClickHouse stores. - -**مثال** - -``` xml - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - -``` - -نوشتن به وبلاگ نیز پشتیبانی می کند. پیکربندی مثال: - -``` xml - - 1 - -
syslog.remote:10514
- myhost.local - LOG_LOCAL6 - syslog - - -``` - -کلید: - -- use_syslog — Required setting if you want to write to the syslog. -- address — The host\[:port\] of syslogd. If omitted, the local daemon is used. -- hostname — Optional. The name of the host that logs are sent from. -- facility — [کلمه کلیدی تسهیلات سیسلوگ](https://en.wikipedia.org/wiki/Syslog#Facility) در حروف بزرگ با “LOG_” پیشوند: (`LOG_USER`, `LOG_DAEMON`, `LOG_LOCAL3`, و به همین ترتیب). - مقدار پیشفرض: `LOG_USER` اگر `address` مشخص شده است, `LOG_DAEMON otherwise.` -- format – Message format. Possible values: `bsd` و `syslog.` - -## & کلاندارها {#macros} - -تعویض پارامتر برای جداول تکرار. - -می توان حذف اگر جداول تکرار استفاده نمی شود. - -برای کسب اطلاعات بیشتر به بخش مراجعه کنید “[ایجاد جداول تکرار شده](../../engines/table-engines/mergetree-family/replication.md)”. - -**مثال** - -``` xml - -``` - -## نشاندار کردن _چ_سیز {#server-mark-cache-size} - -اندازه تقریبی (به بایت) کش علامت های استفاده شده توسط موتورهای جدول [ادغام](../../engines/table-engines/mergetree-family/mergetree.md) خانواده - -کش برای سرور به اشتراک گذاشته و حافظه به عنوان مورد نیاز اختصاص داده است. اندازه کش باید حداقل 5368709120 باشد. - -**مثال** - -``` xml -5368709120 -``` - -## م_قیاس تصویر {#max-concurrent-queries} - -حداکثر تعداد درخواست به طور همزمان پردازش. - -**مثال** - -``` xml -100 -``` - -## _تنامههای بیشینه {#max-connections} - -حداکثر تعداد اتصالات ورودی. - -**مثال** - -``` xml -4096 -``` - -## _موضوعات بیشینه {#max-open-files} - -حداکثر تعداد فایل های باز. - -به طور پیش فرض: `maximum`. - -ما توصیه می کنیم با استفاده از این گزینه در سیستم عامل مک ایکس از `getrlimit()` تابع یک مقدار نادرست می گرداند. - -**مثال** - -``` xml -262144 -``` - -## حداکثر_طب_ضز_توقف {#max-table-size-to-drop} - -محدودیت در حذف جداول. - -اگر اندازه یک [ادغام](../../engines/table-engines/mergetree-family/mergetree.md) جدول بیش از `max_table_size_to_drop` با استفاده از پرس و جو قطره نمی توانید حذف کنید. - -اگر شما هنوز هم نیاز به حذف جدول بدون راه اندازی مجدد سرور کلیک, ایجاد `/flags/force_drop_table` فایل و اجرای پرس و جو قطره. - -مقدار پیش فرض: 50 گیگابایت. - -ارزش 0 بدان معنی است که شما می توانید تمام جداول بدون هیچ گونه محدودیت حذف. - -**مثال** - -``` xml -0 -``` - -## ادغام {#server_configuration_parameters-merge_tree} - -تنظیم زیبا برای جداول در [ادغام](../../engines/table-engines/mergetree-family/mergetree.md). - -برای کسب اطلاعات بیشتر, دیدن ادغام.فایل هدر ساعت. - -**مثال** - -``` xml - - 5 - -``` - -## openSSL {#server_configuration_parameters-openssl} - -SSL client/server configuration. - -پشتیبانی از اس اس ال توسط `libpoco` کتابخونه. رابط در فایل شرح داده شده است [سوسمنگر.ه](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h) - -کلید برای تنظیمات سرور / مشتری: - -- privateKeyFile – The path to the file with the secret key of the PEM certificate. The file may contain a key and certificate at the same time. -- certificateFile – The path to the client/server certificate file in PEM format. You can omit it if `privateKeyFile` شامل گواهی. -- caConfig – The path to the file or directory that contains trusted root certificates. -- verificationMode – The method for checking the node's certificates. Details are in the description of the [متن](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) کلاس. مقادیر ممکن: `none`, `relaxed`, `strict`, `once`. -- verificationDepth – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value. -- loadDefaultCAFile – Indicates that built-in CA certificates for OpenSSL will be used. Acceptable values: `true`, `false`. \| -- cipherList – Supported OpenSSL encryptions. For example: `ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH`. -- cacheSessions – Enables or disables caching sessions. Must be used in combination with `sessionIdContext`. مقادیر قابل قبول: `true`, `false`. -- sessionIdContext – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed `SSL_MAX_SSL_SESSION_ID_LENGTH`. این پارامتر همیشه توصیه می شود از این کمک می کند تا جلوگیری از مشکلات هر دو اگر سرور حافظه پنهان جلسه و اگر مشتری درخواست ذخیره. مقدار پیشفرض: `${application.name}`. -- sessionCacheSize – The maximum number of sessions that the server caches. Default value: 1024\*20. 0 – Unlimited sessions. -- sessionTimeout – Time for caching the session on the server. -- extendedVerification – Automatically extended verification of certificates after the session ends. Acceptable values: `true`, `false`. -- requireTLSv1 – Require a TLSv1 connection. Acceptable values: `true`, `false`. -- requireTLSv1_1 – Require a TLSv1.1 connection. Acceptable values: `true`, `false`. -- requireTLSv1 – Require a TLSv1.2 connection. Acceptable values: `true`, `false`. -- fips – Activates OpenSSL FIPS mode. Supported if the library's OpenSSL version supports FIPS. -- privateKeyPassphraseHandler – Class (PrivateKeyPassphraseHandler subclass) that requests the passphrase for accessing the private key. For example: ``, `KeyFileHandler`, `test`, ``. -- invalidCertificateHandler – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` ConsoleCertificateHandler ` . -- disableProtocols – Protocols that are not allowed to use. -- preferServerCiphers – Preferred server ciphers on the client. - -**مثال تنظیمات:** - -``` xml - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - -``` - -## _خروج {#server_configuration_parameters-part-log} - -وقایع ورود به سیستم که با مرتبط [ادغام](../../engines/table-engines/mergetree-family/mergetree.md). برای مثال, اضافه کردن یا ادغام داده ها. شما می توانید ورود به سیستم برای شبیه سازی الگوریتم های ادغام و مقایسه ویژگی های خود استفاده کنید. شما می توانید روند ادغام تجسم. - -نمایش داده شد در سیستم وارد [سیستم._خروج](../../operations/system-tables.md#system_tables-part-log) جدول, نه در یک فایل جداگانه. شما می توانید نام این جدول را در پیکربندی `table` پارامتر (پایین را ببینید). - -از پارامترهای زیر برای پیکربندی ورود استفاده کنید: - -- `database` – Name of the database. -- `table` – Name of the system table. -- `partition_by` – Sets a [کلید پارتیشن بندی سفارشی](../../engines/table-engines/mergetree-family/custom-partitioning-key.md). -- `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. - -**مثال** - -``` xml - - system - part_log
- toMonday(event_date) - 7500 - -``` - -## مسیر {#server_configuration_parameters-path} - -مسیر به دایرکتوری حاوی داده. - -!!! note "یادداشت" - اسلش الزامی است. - -**مثال** - -``` xml -/var/lib/clickhouse/ -``` - -## پرومتیوس {#server_configuration_parameters-prometheus} - -افشای معیارهای داده ها برای خراش دادن از [پرومتیوس](https://prometheus.io). - -تنظیمات: - -- `endpoint` – HTTP endpoint for scraping metrics by prometheus server. Start from ‘/’. -- `port` – Port for `endpoint`. -- `metrics` – Flag that sets to expose metrics from the [سیستم.متریک](../system-tables.md#system_tables-metrics) جدول -- `events` – Flag that sets to expose metrics from the [سیستم.رویدادها](../system-tables.md#system_tables-events) جدول -- `asynchronous_metrics` – Flag that sets to expose current metrics values from the [سیستم._نامهنویسی ناهمزمان](../system-tables.md#system_tables-asynchronous_metrics) جدول - -**مثال** - -``` xml - - /metrics - 8001 - true - true - true - -``` - -## _خروج {#server_configuration_parameters-query-log} - -تنظیم برای ورود به سیستم نمایش داده شد با دریافت [_ترکیب = 1](../settings/settings.md) تنظیمات. - -نمایش داده شد در سیستم وارد [سیستم._خروج](../../operations/system-tables.md#system_tables-query_log) جدول, نه در یک فایل جداگانه. شما می توانید نام جدول را در `table` پارامتر (پایین را ببینید). - -از پارامترهای زیر برای پیکربندی ورود استفاده کنید: - -- `database` – Name of the database. -- `table` – Name of the system table the queries will be logged in. -- `partition_by` – Sets a [کلید پارتیشن بندی سفارشی](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) برای یک جدول. -- `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. - -اگه جدول وجود نداشته باشه. اگر ساختار ورود به سیستم پرس و جو تغییر زمانی که سرور فاحشه خانه به روز شد, جدول با ساختار قدیمی تغییر نام داد, و یک جدول جدید به طور خودکار ایجاد شده است. - -**مثال** - -``` xml - - system - query_log
- toMonday(event_date) - 7500 - -``` - -## _ر_خروج {#server_configuration_parameters-query-thread-log} - -تنظیم برای ورود به سیستم موضوعات نمایش داده شد دریافت شده با [& پایین: 1](../settings/settings.md#settings-log-query-threads) تنظیمات. - -نمایش داده شد در سیستم وارد [سیستم._ر_خروج](../../operations/system-tables.md#system_tables-query-thread-log) جدول, نه در یک فایل جداگانه. شما می توانید نام جدول را در `table` پارامتر (پایین را ببینید). - -از پارامترهای زیر برای پیکربندی ورود استفاده کنید: - -- `database` – Name of the database. -- `table` – Name of the system table the queries will be logged in. -- `partition_by` – Sets a [کلید پارتیشن بندی سفارشی](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) برای یک جدول سیستم. -- `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. - -اگه جدول وجود نداشته باشه. اگر ساختار پرس و جو موضوع ورود به سیستم تغییر زمانی که سرور فاحشه خانه به روز شد, جدول با ساختار قدیمی تغییر نام داد, و یک جدول جدید به طور خودکار ایجاد شده است. - -**مثال** - -``` xml - - system - query_thread_log
- toMonday(event_date) - 7500 - -``` - -## _قطع {#server_configuration_parameters-trace_log} - -تنظیمات برای [_قطع](../../operations/system-tables.md#system_tables-trace_log) عملیات جدول سیستم. - -پارامترها: - -- `database` — Database for storing a table. -- `table` — Table name. -- `partition_by` — [کلید پارتیشن بندی سفارشی](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) برای یک جدول سیستم. -- `flush_interval_milliseconds` — Interval for flushing data from the buffer in memory to the table. - -فایل پیکربندی پیش فرض سرور `config.xml` شامل بخش تنظیمات زیر است: - -``` xml - - system - trace_log
- toYYYYMM(event_date) - 7500 - -``` - -## _منبع {#query-masking-rules} - -قوانین مبتنی بر عبارت منظم, خواهد شد که به نمایش داده شد و همچنین تمام پیام های ورود به سیستم قبل از ذخیره سازی در سیاهههای مربوط به سرور اعمال, -`system.query_log`, `system.text_log`, `system.processes` جدول, و در سیاهههای مربوط به مشتری ارسال. که اجازه می دهد تا جلوگیری از -نشت اطلاعات حساس از پرس و جو گذاشتن (مانند نام, ایمیل, شخصی -شناسه و یا شماره کارت اعتباری) به سیاهههای مربوط. - -**مثال** - -``` xml - - - hide SSN - (^|\D)\d{3}-\d{2}-\d{4}($|\D) - 000-00-0000 - - -``` - -زمینه پیکربندی: -- `name` - نام قانون (اختیاری) -- `regexp` - تکرار 2 عبارت منظم سازگار (اجباری) -- `replace` - رشته جایگزینی برای داده های حساس (اختیاری به طور پیش فرض-شش ستاره) - -قوانین پوشش به کل پرس و جو اعمال می شود (برای جلوگیری از نشت اطلاعات حساس از نمایش داده شد ناقص / غیر تجزیه). - -`system.events` جدول شمارنده `QueryMaskingRulesMatch` که تعداد کلی از پرس و جو پوشش قوانین مسابقات. - -برای نمایش داده شد توزیع هر سرور باید به طور جداگانه پیکربندی شود, در غیر این صورت, فرعی به دیگر منتقل -گره ها بدون پوشش ذخیره می شوند. - -## دور دور {#server-settings-remote-servers} - -پیکربندی خوشه های مورد استفاده توسط [توزیع شده](../../engines/table-engines/special/distributed.md) موتور جدول و توسط `cluster` تابع جدول. - -**مثال** - -``` xml - -``` - -برای ارزش `incl` ویژگی, بخش را ببینید “[پروندههای پیکربندی](../configuration-files.md#configuration_files)”. - -**همچنین نگاه کنید به** - -- [در حال بارگذاری](../settings/settings.md#settings-skip_unavailable_shards) - -## منطقهی زمانی {#server_configuration_parameters-timezone} - -منطقه زمانی سرور. - -مشخص شده به عنوان شناساگر ایانا برای منطقه زمانی یو تی سی یا موقعیت جغرافیایی (مثلا افریقا / ابیجان). - -منطقه زمانی برای تبدیل بین فرمت های رشته و تاریخ ساعت لازم است که زمینه های تاریخ ساعت خروجی به فرمت متن (چاپ شده بر روی صفحه نمایش و یا در یک فایل), و هنگامی که گرفتن تاریخ ساعت از یک رشته. علاوه بر این, منطقه زمانی در توابع است که با زمان و تاریخ کار می کنند در صورتی که منطقه زمانی در پارامترهای ورودی دریافت نمی استفاده. - -**مثال** - -``` xml -Europe/Moscow -``` - -## _صادر کردن {#server_configuration_parameters-tcp_port} - -پورت برای برقراری ارتباط با مشتریان بیش از پروتکل تی سی پی. - -**مثال** - -``` xml -9000 -``` - -## _شروع مجدد {#server_configuration_parameters-tcp_port_secure} - -پورت تی سی پی برای برقراری ارتباط امن با مشتریان. با استفاده از [OpenSSL](#server_configuration_parameters-openssl) تنظیمات. - -**مقادیر ممکن** - -عدد صحیح مثبت. - -**مقدار پیشفرض** - -``` xml -9440 -``` - -## _وارد کردن {#server_configuration_parameters-mysql_port} - -پورت برای برقراری ارتباط با مشتریان بیش از پروتکل خروجی زیر. - -**مقادیر ممکن** - -عدد صحیح مثبت. - -مثال - -``` xml -9004 -``` - -## _مخفی کردن {#server-settings-tmp_path} - -مسیر به داده های موقت برای پردازش نمایش داده شد بزرگ است. - -!!! note "یادداشت" - اسلش الزامی است. - -**مثال** - -``` xml -/var/lib/clickhouse/tmp/ -``` - -## پیدا کردن موقعیت جغرافیایی از روی شبکه {#server-settings-tmp-policy} - -سیاست از [`storage_configuration`](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) برای ذخیره فایل های موقت. -اگر تنظیم نشود [`tmp_path`](#server-settings-tmp_path) استفاده شده است, در غیر این صورت نادیده گرفته شده است. - -!!! note "یادداشت" - - `move_factor` نادیده گرفته شده است -- `keep_free_space_bytes` نادیده گرفته شده است -- `max_data_part_size_bytes` نادیده گرفته شده است -- شما باید دقیقا یک جلد در این سیاست داشته باشید - -## _بالا {#server-settings-uncompressed_cache_size} - -اندازه کش (به بایت) برای داده های غیر فشرده استفاده شده توسط موتورهای جدول از [ادغام](../../engines/table-engines/mergetree-family/mergetree.md). - -یک کش مشترک برای سرور وجود دارد. حافظه در تقاضا اختصاص داده. کش در صورتی که گزینه استفاده می شود [همترازی پایین](../settings/settings.md#setting-use_uncompressed_cache) فعال است. - -کش غیر فشرده سودمند برای نمایش داده شد بسیار کوتاه در موارد فردی است. - -**مثال** - -``` xml -8589934592 -``` - -## _مخفی کردن _صفحه {#server_configuration_parameters-user_files_path} - -دایرکتوری با فایل های کاربر. مورد استفاده در تابع جدول [پرونده()](../../sql-reference/table-functions/file.md). - -**مثال** - -``` xml -/var/lib/clickhouse/user_files/ -``` - -## _تنفورد {#users-config} - -مسیر پروندهی شامل: - -- تنظیمات کاربر. -- حقوق دسترسی. -- پروفایل تنظیمات. -- تنظیمات سهمیه. - -**مثال** - -``` xml -users.xml -``` - -## باغ وحش {#server-settings_zookeeper} - -شامل تنظیماتی است که اجازه می دهد تا کلیک برای ارتباط برقرار کردن با یک [باغ وحش](http://zookeeper.apache.org/) خوشه خوشه. - -کلیک هاوس با استفاده از باغ وحش برای ذخیره سازی ابرداده از کپی در هنگام استفاده از جداول تکرار. اگر جداول تکرار استفاده نمی شود, این بخش از پارامترها را می توان حذف. - -این بخش شامل پارامترهای زیر است: - -- `node` — ZooKeeper endpoint. You can set multiple endpoints. - - به عنوان مثال: - - - -``` xml - - example_host - 2181 - -``` - - The `index` attribute specifies the node order when trying to connect to the ZooKeeper cluster. - -- `session_timeout` — Maximum timeout for the client session in milliseconds. -- `root` — The [حالت](http://zookeeper.apache.org/doc/r3.5.5/zookeeperOver.html#Nodes+and+ephemeral+nodes) استفاده شده است که به عنوان ریشه برای znodes استفاده شده توسط ClickHouse سرور. اختیاری. -- `identity` — User and password, that can be required by ZooKeeper to give access to requested znodes. Optional. - -**پیکربندی نمونه** - -``` xml - - - example1 - 2181 - - - example2 - 2181 - - 30000 - 10000 - - /path/to/zookeeper/node - - user:password - -``` - -**همچنین نگاه کنید به** - -- [تکرار](../../engines/table-engines/mergetree-family/replication.md) -- [راهنمای برنامه نویس باغ وحش](http://zookeeper.apache.org/doc/current/zookeeperProgrammers.html) - -## سرویس پرداخت درونبرنامهای پلی {#server-settings-use_minimalistic_part_header_in_zookeeper} - -روش ذخیره سازی برای هدر بخش داده ها در باغ وحش. - -این تنظیم فقط در مورد `MergeTree` خانواده این را می توان مشخص کرد: - -- در سطح جهانی در [ادغام](#server_configuration_parameters-merge_tree) بخش از `config.xml` پرونده. - - تاتر با استفاده از تنظیمات برای تمام جداول بر روی سرور. شما می توانید تنظیمات را در هر زمان تغییر دهید. جداول موجود رفتار خود را تغییر دهید زمانی که تنظیمات تغییر می کند. - -- برای هر جدول. - - هنگام ایجاد یک جدول مربوطه را مشخص کنید [تنظیم موتور](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). رفتار یک جدول موجود با این تنظیم تغییر نمی کند, حتی اگر تغییرات تنظیم جهانی. - -**مقادیر ممکن** - -- 0 — Functionality is turned off. -- 1 — Functionality is turned on. - -اگر `use_minimalistic_part_header_in_zookeeper = 1` پس [تکرار](../../engines/table-engines/mergetree-family/replication.md) جداول هدر قطعات داده را با استفاده از یک واحد ذخیره می کنند `znode`. اگر جدول شامل بسیاری از ستون, این روش ذخیره سازی به طور قابل توجهی کاهش می دهد حجم داده های ذخیره شده در باغ وحش. - -!!! attention "توجه" - پس از استفاده از `use_minimalistic_part_header_in_zookeeper = 1` شما نمیتوانید سرور کلیک را به نسخه ای که از این تنظیم پشتیبانی نمی کند ارتقا دهید. مراقب باشید در هنگام به روز رسانی تاتر بر روی سرور در یک خوشه. همه سرورها را در یک زمان ارتقا ندهید. این امن تر است برای تست نسخه های جدید از خانه رعیتی در یک محیط تست, و یا فقط در چند سرور از یک خوشه. - - Data part headers already stored with this setting can't be restored to their previous (non-compact) representation. - -**مقدار پیشفرض:** 0. - -## نمایش سایت {#server-settings-disable-internal-dns-cache} - -غیر فعال کش دی ان اس داخلی. توصیه شده برای کارخانه کلیک در سیستم -با زیرساخت های اغلب در حال تغییر مانند کوبرنتس. - -**مقدار پیشفرض:** 0. - -## پیدا کردن موقعیت جغرافیایی از روی شبکه {#server-settings-dns-cache-update-period} - -دوره به روز رسانی نشانی های اینترنتی ذخیره شده در کش دی ان اس داخلی خانه (در ثانیه). -به روز رسانی همزمان انجام, در یک موضوع سیستم جداگانه. - -**مقدار پیشفرض**: 15. - -## _پوشه دستیابی {#access_control_path} - -مسیر را به یک پوشه که یک سرور کلیک ذخیره کاربر و نقش تنظیمات ایجاد شده توسط دستورات گذاشتن. - -مقدار پیشفرض: `/var/lib/clickhouse/access/`. - -**همچنین نگاه کنید به** - -- [کنترل دسترسی و مدیریت حساب](../access-rights.md#access-control) - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/server_configuration_parameters/settings/) diff --git a/docs/fa/operations/settings/constraints-on-settings.md b/docs/fa/operations/settings/constraints-on-settings.md deleted file mode 100644 index 809dccdbec5..00000000000 --- a/docs/fa/operations/settings/constraints-on-settings.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 62 -toc_title: "\u0645\u062D\u062F\u0648\u062F\u06CC\u062A \u062F\u0631 \u062A\u0646\u0638\ - \u06CC\u0645\u0627\u062A" ---- - -# محدودیت در تنظیمات {#constraints-on-settings} - -محدودیت در تنظیمات را می توان در تعریف `profiles` بخش از `user.xml` فایل پیکربندی و منع کاربران از تغییر برخی از تنظیمات با `SET` پرس و جو. -محدودیت ها به صورت زیر تعریف می شوند: - -``` xml - - - - - lower_boundary - - - upper_boundary - - - lower_boundary - upper_boundary - - - - - - - -``` - -اگر کاربر تلاش می کند به نقض محدودیت یک استثنا پرتاب می شود و تنظیم تغییر نکرده است. -سه نوع محدودیت پشتیبانی می شوند: `min`, `max`, `readonly`. این `min` و `max` محدودیت مشخص مرزهای بالا و پایین برای یک محیط عددی و می تواند در ترکیب استفاده می شود. این `readonly` محدودیت مشخص می کند که کاربر می تواند تنظیمات مربوطه را تغییر دهید و در همه. - -**مثال:** اجازه بدهید `users.xml` شامل خطوط: - -``` xml - - - 10000000000 - 0 - ... - - - 5000000000 - 20000000000 - - - - - - - -``` - -نمایش داده شد زیر همه استثنا پرتاب: - -``` sql -SET max_memory_usage=20000000001; -SET max_memory_usage=4999999999; -SET force_index_by_date=1; -``` - -``` text -Code: 452, e.displayText() = DB::Exception: Setting max_memory_usage should not be greater than 20000000000. -Code: 452, e.displayText() = DB::Exception: Setting max_memory_usage should not be less than 5000000000. -Code: 452, e.displayText() = DB::Exception: Setting force_index_by_date should not be changed. -``` - -**یادداشت:** این `default` مشخصات است دست زدن به ویژه: همه محدودیت های تعریف شده برای `default` مشخصات تبدیل به محدودیت های پیش فرض, بنابراین محدود کردن تمام کاربران تا زمانی که به صراحت برای این کاربران باطل. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/settings/constraints_on_settings/) diff --git a/docs/fa/operations/settings/index.md b/docs/fa/operations/settings/index.md deleted file mode 100644 index 8dba6ab6542..00000000000 --- a/docs/fa/operations/settings/index.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u062A\u0646\u0638\u06CC\u0645\u0627\u062A" -toc_priority: 55 -toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" ---- - -# تنظیمات {#session-settings-intro} - -راه های متعدد را به تمام تنظیمات شرح داده شده در این بخش از اسناد و مدارک وجود دارد. - -تنظیمات در لایه پیکربندی, بنابراین هر لایه های بعدی دوباره تعریف تنظیمات قبلی. - -راه های پیکربندی تنظیمات به ترتیب اولویت: - -- تنظیمات در `users.xml` فایل پیکربندی سرور. - - تنظیم در عنصر ``. - -- تنظیمات جلسه. - - ارسال `SET setting=value` از مشتری کنسول کلیک در حالت تعاملی. - به طور مشابه, شما می توانید جلسات کلیک در پروتکل قام استفاده. برای انجام این, شما نیاز به مشخص `session_id` پارامتر قام. - -- تنظیمات پرس و جو. - - - هنگام شروع مشتری کنسول کلیک در حالت غیر تعاملی, تنظیم پارامتر راه اندازی `--setting=value`. - - هنگام استفاده از پارامترهای سیستم (`URL?setting_1=value&setting_2=value...`). - -تنظیمات است که تنها می تواند در فایل پیکربندی سرور ساخته شده در این بخش پوشش داده نمی شود. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/settings/) diff --git a/docs/fa/operations/settings/permissions-for-queries.md b/docs/fa/operations/settings/permissions-for-queries.md deleted file mode 100644 index 6b49c6ca334..00000000000 --- a/docs/fa/operations/settings/permissions-for-queries.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 58 -toc_title: "\u0645\u062C\u0648\u0632 \u0628\u0631\u0627\u06CC \u0646\u0645\u0627\u06CC\ - \u0634 \u062F\u0627\u062F\u0647 \u0634\u062F" ---- - -# مجوز برای نمایش داده شد {#permissions_for_queries} - -نمایش داده شد در کلیک خانه را می توان به انواع مختلفی تقسیم شده است: - -1. خواندن نمایش داده شد داده: `SELECT`, `SHOW`, `DESCRIBE`, `EXISTS`. -2. نوشتن نمایش داده شد داده ها: `INSERT`, `OPTIMIZE`. -3. تغییر پرسوجوی تنظیمات: `SET`, `USE`. -4. [DDL](https://en.wikipedia.org/wiki/Data_definition_language) نمایش داده شد: `CREATE`, `ALTER`, `RENAME`, `ATTACH`, `DETACH`, `DROP` `TRUNCATE`. -5. `KILL QUERY`. - -تنظیمات زیر تنظیم مجوز کاربر بر اساس نوع پرس و جو: - -- [فقط خواندنی](#settings_readonly) — Restricts permissions for all types of queries except DDL queries. -- [اجازه دادن به _نشانی](#settings_allow_ddl) — Restricts permissions for DDL queries. - -`KILL QUERY` را می توان با هر تنظیمات انجام می شود. - -## فقط خواندنی {#settings_readonly} - -محدود مجوز برای خواندن داده ها, نوشتن داده ها و تغییر تنظیمات نمایش داده شد. - -ببینید که چگونه نمایش داده شد به انواع تقسیم [بالا](#permissions_for_queries). - -مقادیر ممکن: - -- 0 — All queries are allowed. -- 1 — Only read data queries are allowed. -- 2 — Read data and change settings queries are allowed. - -پس از تنظیم `readonly = 1` کاربر نمیتواند تغییر کند `readonly` و `allow_ddl` تنظیمات در جلسه فعلی. - -هنگام استفاده از `GET` روش در [رابط قام](../../interfaces/http.md), `readonly = 1` به طور خودکار تنظیم شده است. برای تغییر داده ها از `POST` روش. - -تنظیم `readonly = 1` منع کاربر از تغییر تمام تنظیمات. یک راه برای منع کاربر وجود دارد -از تغییر تنظیمات تنها خاص, برای اطلاعات بیشتر ببینید [محدودیت در تنظیمات](constraints-on-settings.md). - -مقدار پیشفرض: 0 - -## اجازه دادن به _نشانی {#settings_allow_ddl} - -اجازه می دهد یا رد می کند [DDL](https://en.wikipedia.org/wiki/Data_definition_language) نمایش داده شد. - -ببینید که چگونه نمایش داده شد به انواع تقسیم [بالا](#permissions_for_queries). - -مقادیر ممکن: - -- 0 — DDL queries are not allowed. -- 1 — DDL queries are allowed. - -شما نمی توانید اجرا کنید `SET allow_ddl = 1` اگر `allow_ddl = 0` برای جلسه فعلی. - -مقدار پیشفرض: 1 - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/settings/permissions_for_queries/) diff --git a/docs/fa/operations/settings/query-complexity.md b/docs/fa/operations/settings/query-complexity.md deleted file mode 100644 index e78193e671e..00000000000 --- a/docs/fa/operations/settings/query-complexity.md +++ /dev/null @@ -1,302 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 59 -toc_title: "\u0645\u062D\u062F\u0648\u062F\u06CC\u062A \u062F\u0631 \u067E\u06CC\u0686\ - \u06CC\u062F\u06AF\u06CC \u067E\u0631\u0633 \u0648 \u062C\u0648" ---- - -# محدودیت در پیچیدگی پرس و جو {#restrictions-on-query-complexity} - -محدودیت در پیچیدگی پرس و جو بخشی از تنظیمات. -برای اجرای امن تر از رابط کاربر استفاده می شود. -تقریبا تمام محدودیت ها فقط برای اعمال `SELECT`. برای پردازش پرس و جو توزیع, محدودیت بر روی هر سرور به طور جداگانه اعمال. - -خانه را کلیک کنید چک محدودیت برای قطعات داده, نه برای هر سطر. این بدان معنی است که شما می توانید ارزش محدودیت با اندازه بخش داده ها تجاوز. - -محدودیت در “maximum amount of something” می توانید مقدار را 0, که به معنی “unrestricted”. -اکثر محدودیت ها نیز دارند ‘overflow_mode’ محیط, به این معنی چه باید بکنید هنگامی که از حد فراتر رفته است. -این می تواند یکی از دو مقدار را: `throw` یا `break`. محدودیت در تجمع (کد _شورت_فلو_وشه گروه) نیز ارزش داشته باشد `any`. - -`throw` – Throw an exception (default). - -`break` – Stop executing the query and return the partial result, as if the source data ran out. - -`any (only for group_by_overflow_mode)` – Continuing aggregation for the keys that got into the set, but don't add new keys to the set. - -## _کاساژ بیشینه {#settings_max_memory_usage} - -حداکثر مقدار رم برای استفاده برای اجرای پرس و جو بر روی یک سرور واحد. - -در فایل پیکربندی پیش فرض, حداکثر است 10 گیگابایت. - -تنظیم می کند حجم حافظه در دسترس و یا حجم کل حافظه بر روی دستگاه در نظر نمی. -محدودیت شامل یک پرس و جو تنها در یک سرور. -شما می توانید استفاده کنید `SHOW PROCESSLIST` برای دیدن مصرف حافظه فعلی برای هر پرس و جو. -بعلاوه, مصرف حافظه اوج برای هر پرس و جو ردیابی و نوشته شده به ورود به سیستم. - -استفاده از حافظه برای ایالت های توابع مجموع خاص نظارت نیست. - -استفاده از حافظه به طور کامل برای ایالت ها از توابع کل ردیابی نیست `min`, `max`, `any`, `anyLast`, `argMin`, `argMax` از `String` و `Array` بحث کردن. - -مصرف حافظه نیز توسط پارامترها محدود شده است `max_memory_usage_for_user` و `max_memory_usage_for_all_queries`. - -## _شمارهگیر بیشینه {#max-memory-usage-for-user} - -حداکثر مقدار رم برای استفاده برای اجرای نمایش داده شد کاربر بر روی یک سرور واحد. - -مقادیر پیش فرض در تعریف [تنظیمات.ه](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.h#L288). به طور پیش فرض مقدار محدود نمی شود (`max_memory_usage_for_user = 0`). - -همچنین نگاه کنید به شرح [_کاساژ بیشینه](#settings_max_memory_usage). - -## _شیشه بخاطر _خروج {#max-memory-usage-for-all-queries} - -حداکثر مقدار رم برای استفاده برای اجرای تمام نمایش داده شد بر روی یک سرور واحد. - -مقادیر پیش فرض در تعریف [تنظیمات.ه](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.h#L289). به طور پیش فرض مقدار محدود نمی شود (`max_memory_usage_for_all_queries = 0`). - -همچنین نگاه کنید به شرح [_کاساژ بیشینه](#settings_max_memory_usage). - -## _گذرواژههای _ورود {#max-rows-to-read} - -محدودیت های زیر را می توان در هر بلوک بررسی (به جای در هر سطر). به این معنا که, محدودیت را می توان شکسته کمی. -هنگامی که در حال اجرا یک پرس و جو در موضوعات مختلف, محدودیت های زیر به هر موضوع اعمال می شود به طور جداگانه. - -حداکثر تعداد ردیف است که می تواند از یک جدول زمانی که در حال اجرا یک پرس و جو به عنوان خوانده شده. - -## _مخفی کردن {#max-bytes-to-read} - -حداکثر تعداد بایت (داده های غیر فشرده) است که می تواند از یک جدول به عنوان خوانده شده در هنگام اجرای یک پرس و جو. - -## _ورود به سیستم {#read-overflow-mode} - -چه باید بکنید هنگامی که حجم داده ها به عنوان خوانده شده بیش از یکی از محدودیت های: ‘throw’ یا ‘break’. به طور پیش فرض, پرتاب. - -## _رو_تو_گروهها {#settings-max-rows-to-group-by} - -حداکثر تعداد کلید منحصر به فرد دریافت شده از تجمع. این تنظیم به شما امکان مصرف حافظه محدود در هنگام جمع. - -## _شماره _شماره گروه {#group-by-overflow-mode} - -چه باید بکنید هنگامی که تعدادی از کلید های منحصر به فرد برای تجمع بیش از حد: ‘throw’, ‘break’ یا ‘any’. به طور پیش فرض, پرتاب. -با استفاده از ‘any’ ارزش شما اجازه می دهد یک تقریب از گروه های اجرا. کیفیت این تقریب بستگی به ماهیت استاتیک داده ها دارد. - -## ا_فزون_بر_گونهی_گونهی زیر_گروهها {#settings-max_bytes_before_external_group_by} - -فعالسازی یا غیرفعالسازی اعدام `GROUP BY` بند در حافظه خارجی. ببینید [گروه در حافظه خارجی](../../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory). - -مقادیر ممکن: - -- حداکثر حجم رم (به بایت) است که می تواند توسط تک استفاده می شود [GROUP BY](../../sql-reference/statements/select/group-by.md#select-group-by-clause) عمل -- 0 — `GROUP BY` در حافظه خارجی غیر فعال. - -مقدار پیش فرض: 0. - -## _شماره بیشینه {#max-rows-to-sort} - -حداکثر تعداد ردیف قبل از مرتب سازی. این اجازه می دهد تا شما را به محدود کردن مصرف حافظه در هنگام مرتب سازی. - -## ا_سلایدی {#max-bytes-to-sort} - -حداکثر تعداد بایت قبل از مرتب سازی. - -## کد_و_وشهیابی {#sort-overflow-mode} - -چه باید بکنید اگر تعداد ردیف قبل از مرتب سازی دریافت بیش از یکی از محدودیت: ‘throw’ یا ‘break’. به طور پیش فرض, پرتاب. - -## بارشهای بیشینه {#setting-max_result_rows} - -محدود در تعداد ردیف در نتیجه. همچنین برای زیرمجموعه بررسی, و بر روی سرور از راه دور در هنگام اجرای بخش هایی از یک پرس و جو توزیع. - -## حداکثر_زمین بایت {#max-result-bytes} - -محدود در تعداد بایت در نتیجه. همان تنظیمات قبلی. - -## _شماره حاصل {#result-overflow-mode} - -چه باید بکنید اگر حجم نتیجه بیش از یکی از محدودیت های: ‘throw’ یا ‘break’. به طور پیش فرض, پرتاب. - -با استفاده از ‘break’ شبیه به استفاده از حد است. `Break` قطع اعدام تنها در سطح بلوک. این به این معنی است که مقدار ردیف بازگشت بیشتر از [بارشهای بیشینه](#setting-max_result_rows) چندین [ت_مایش بیشینه](settings.md#setting-max_block_size) و بستگی دارد [_مخفی کردن](settings.md#settings-max_threads). - -مثال: - -``` sql -SET max_threads = 3, max_block_size = 3333; -SET max_result_rows = 3334, result_overflow_mode = 'break'; - -SELECT * -FROM numbers_mt(100000) -FORMAT Null; -``` - -نتیجه: - -``` text -6666 rows in set. ... -``` - -## زمان _شنامهی حداکثر {#max-execution-time} - -حداکثر زمان اجرای پرس و جو در ثانیه. -در این زمان برای یکی از مراحل مرتب سازی بررسی نمی شود و یا هنگام ادغام و نهایی کردن توابع کلی. - -## _شروع مجدد {#timeout-overflow-mode} - -چه باید بکنید اگر پرس و جو اجرا می شود بیش از ‘max_execution_time’: ‘throw’ یا ‘break’. به طور پیش فرض, پرتاب. - -## _شروع مجدد {#min-execution-speed} - -سرعت اجرای حداقل در ردیف در هر ثانیه. بررسی در هر بلوک داده زمانی که ‘timeout_before_checking_execution_speed’ انقضا مییابد. اگر سرعت اجرای پایین تر است, یک استثنا پرتاب می شود. - -## ا_فزونهها {#min-execution-speed-bytes} - -حداقل تعداد بایت اعدام در هر ثانیه. بررسی در هر بلوک داده زمانی که ‘timeout_before_checking_execution_speed’ انقضا مییابد. اگر سرعت اجرای پایین تر است, یک استثنا پرتاب می شود. - -## حداکثر_حاقسازی سرعت {#max-execution-speed} - -حداکثر تعداد ردیف اعدام در هر ثانیه. بررسی در هر بلوک داده زمانی که ‘timeout_before_checking_execution_speed’ انقضا مییابد. اگر سرعت اجرای بالا است, سرعت اجرای کاهش خواهد یافت. - -## حداکثر_کشن_پیمایههای سرعت {#max-execution-speed-bytes} - -حداکثر تعداد بایت اعدام در هر ثانیه. بررسی در هر بلوک داده زمانی که ‘timeout_before_checking_execution_speed’ انقضا مییابد. اگر سرعت اجرای بالا است, سرعت اجرای کاهش خواهد یافت. - -## جستجو {#timeout-before-checking-execution-speed} - -چک که سرعت اجرای بیش از حد کند نیست (کمتر از ‘min_execution_speed’), پس از زمان مشخص شده در ثانیه تمام شده است. - -## _رنگ _ورود {#max-columns-to-read} - -حداکثر تعداد ستون است که می تواند از یک جدول در یک پرس و جو به عنوان خوانده شده. اگر پرس و جو نیاز به خواندن تعداد بیشتری از ستون, این می اندازد یک استثنا. - -## _رنگ بیشینه {#max-temporary-columns} - -حداکثر تعداد ستون موقت است که باید در رم در همان زمان نگه داشته شود که در حال اجرا یک پرس و جو, از جمله ستون ثابت. اگر ستون موقت بیش از این وجود دارد, این یک استثنا می اندازد. - -## _رنگ {#max-temporary-non-const-columns} - -همان چیزی که به عنوان ‘max_temporary_columns’, اما بدون شمارش ستون ثابت. -توجه داشته باشید که ستون های ثابت در حال اجرا یک پرس و جو نسبتا اغلب تشکیل, اما نیاز به حدود صفر منابع محاسباتی. - -## حداکثر {#max-subquery-depth} - -حداکثر عمق تودرتو از کارخانه های فرعی. اگر کارخانه های فرعی عمیق تر, یک استثنا پرتاب می شود. به طور پیش فرض, 100. - -## حداکثر _پیپیلین {#max-pipeline-depth} - -حداکثر عمق خط لوله. مربوط به تعدادی از تحولات که هر بلوک داده می رود از طریق در طول پردازش پرس و جو. شمارش در محدوده یک سرور واحد. اگر عمق خط لوله بیشتر است, یک استثنا پرتاب می شود. به طور پیش فرض 1000. - -## _ص_خلاف {#max-ast-depth} - -حداکثر عمق تودرتو از یک درخت نحوی پرس و جو. اگر بیش از, یک استثنا پرتاب می شود. -در این زمان در تجزیه بررسی نمی شود اما تنها پس از تجزیه پرس و جو. به این معنا که, یک درخت نحوی است که بیش از حد عمیق می تواند در طول تجزیه ایجاد, اما پرس و جو شکست مواجه خواهد شد. به طور پیش فرض 1000. - -## _محلولات حداکثر {#max-ast-elements} - -حداکثر تعداد عناصر در یک درخت نحوی پرس و جو. اگر بیش از, یک استثنا پرتاب می شود. -در همان راه به عنوان تنظیمات قبلی تنها پس از تجزیه پرس و جو بررسی می شود. به طور پیش فرض 50000. - -## _رو_ تنظیم {#max-rows-in-set} - -حداکثر تعداد ردیف برای یک مجموعه داده ها در بند در ایجاد شده از یک خرده فروشی. - -## تنظیم _سریع {#max-bytes-in-set} - -حداکثر تعداد بایت (داده های غیر فشرده) استفاده شده توسط یک مجموعه در بند در ایجاد شده از یک خرده فروشی. - -## _حالت تنظیم {#set-overflow-mode} - -چه باید بکنید هنگامی که مقدار داده ها بیش از یکی از محدودیت های: ‘throw’ یا ‘break’. به طور پیش فرض, پرتاب. - -## حوزه _کاربری مکس {#max-rows-in-distinct} - -حداکثر تعداد ردیف های مختلف در هنگام استفاده از متمایز. - -## مک_بتس_ حوزه {#max-bytes-in-distinct} - -حداکثر تعداد بایت استفاده شده توسط یک جدول هش در هنگام استفاده متمایز. - -## _شروع مجدد {#distinct-overflow-mode} - -چه باید بکنید هنگامی که مقدار داده ها بیش از یکی از محدودیت های: ‘throw’ یا ‘break’. به طور پیش فرض, پرتاب. - -## ترجمههای بیشینه {#max-rows-to-transfer} - -حداکثر تعداد ردیف است که می تواند به یک سرور از راه دور منتقل می شود و یا ذخیره شده در یک جدول موقت در هنگام استفاده از جهانی در. - -## ترجمههای بیشینه {#max-bytes-to-transfer} - -حداکثر تعداد بایت (داده های غیر فشرده) است که می تواند به یک سرور از راه دور منتقل می شود و یا ذخیره شده در یک جدول موقت در هنگام استفاده از جهانی در. - -## _شروع مجدد {#transfer-overflow-mode} - -چه باید بکنید هنگامی که مقدار داده ها بیش از یکی از محدودیت های: ‘throw’ یا ‘break’. به طور پیش فرض, پرتاب. - -## _پاک کردن _روشن گرافیک {#settings-max_rows_in_join} - -محدودیت تعداد ردیف در جدول هش استفاده شده است که در هنگام پیوستن به جداول. - -این تنظیمات در مورد [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) عملیات و [پیوستن](../../engines/table-engines/special/join.md) موتور جدول. - -اگر یک پرس و جو شامل چند می پیوندد, خانه چک این تنظیم برای هر نتیجه متوسط. - -تاتر می توانید با اقدامات مختلف ادامه دهید زمانی که از حد رسیده است. استفاده از [_شروع مجدد](#settings-join_overflow_mode) تنظیم برای انتخاب عمل. - -مقادیر ممکن: - -- عدد صحیح مثبت. -- 0 — Unlimited number of rows. - -مقدار پیش فرض: 0. - -## _پویش همیشگی {#settings-max_bytes_in_join} - -محدودیت اندازه در بایت از جدول هش استفاده می شود در هنگام پیوستن به جداول. - -این تنظیمات در مورد [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) عملیات و [پیوستن به موتور جدول](../../engines/table-engines/special/join.md). - -اگر پرس و جو شامل می پیوندد, کلیک چک این تنظیمات برای هر نتیجه متوسط. - -تاتر می توانید با اقدامات مختلف ادامه دهید زمانی که از حد رسیده است. استفاده [_شروع مجدد](#settings-join_overflow_mode) تنظیمات برای انتخاب عمل. - -مقادیر ممکن: - -- عدد صحیح مثبت. -- 0 — Memory control is disabled. - -مقدار پیش فرض: 0. - -## _شروع مجدد {#settings-join_overflow_mode} - -تعریف می کند که چه عمل کلیک انجام زمانی که هر یک از محدودیت های زیر ملحق رسیده است: - -- [_پویش همیشگی](#settings-max_bytes_in_join) -- [_پاک کردن _روشن گرافیک](#settings-max_rows_in_join) - -مقادیر ممکن: - -- `THROW` — ClickHouse throws an exception and breaks operation. -- `BREAK` — ClickHouse breaks operation and doesn't throw an exception. - -مقدار پیشفرض: `THROW`. - -**همچنین نگاه کنید به** - -- [پیوستن بند](../../sql-reference/statements/select/join.md#select-join) -- [پیوستن به موتور جدول](../../engines/table-engines/special/join.md) - -## _مسدود کردن بیشینه {#max-partitions-per-insert-block} - -حداکثر تعداد پارتیشن در یک بلوک قرار داده شده را محدود می کند. - -- عدد صحیح مثبت. -- 0 — Unlimited number of partitions. - -مقدار پیش فرض: 100. - -**اطلاعات دقیق** - -هنگام قرار دادن داده ها, تاتر محاسبه تعداد پارتیشن در بلوک قرار داده. اگر تعداد پارتیشن ها بیش از `max_partitions_per_insert_block`, خانه را کلیک می اندازد یک استثنا با متن زیر: - -> “Too many partitions for single INSERT block (more than” اطلاعات دقیق “). The limit is controlled by ‘max_partitions_per_insert_block’ setting. A large number of partitions is a common misconception. It will lead to severe negative performance impact, including slow server startup, slow INSERT queries and slow SELECT queries. Recommended total number of partitions for a table is under 1000..10000. Please note, that partitioning is not intended to speed up SELECT queries (ORDER BY key is sufficient to make range queries fast). Partitions are intended for data manipulation (DROP PARTITION, etc).” - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/settings/query_complexity/) diff --git a/docs/fa/operations/settings/settings-profiles.md b/docs/fa/operations/settings/settings-profiles.md deleted file mode 100644 index de6f40ba015..00000000000 --- a/docs/fa/operations/settings/settings-profiles.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 61 -toc_title: "\u067E\u0631\u0648\u0641\u0627\u06CC\u0644 \u062A\u0646\u0638\u06CC\u0645\ - \u0627\u062A" ---- - -# پروفایل تنظیمات {#settings-profiles} - -مشخصات تنظیمات مجموعه ای از تنظیمات گروه بندی شده تحت همین نام است. - -!!! note "اطلاعات" - فاحشه خانه نیز پشتیبانی می کند [گردش کار مبتنی بر مربع](../access-rights.md#access-control) برای مدیریت پروفایل تنظیمات. ما توصیه می کنیم از این استفاده کنید. - -مشخصات می توانید هر نام دارند. مشخصات می توانید هر نام دارند. شما می توانید مشخصات مشابه برای کاربران مختلف را مشخص کنید. مهم ترین چیز شما می توانید در مشخصات تنظیمات ارسال شده است `readonly=1`, که تضمین می کند فقط خواندنی دسترسی. - -پروفایل تنظیمات می توانید از یکدیگر به ارث می برند. برای استفاده از ارث, نشان می دهد یک یا چند `profile` تنظیمات قبل از تنظیمات دیگر که در مشخصات ذکر شده. در صورتی که یک تنظیم در پروفایل های مختلف تعریف شده, از تعریف استفاده شده است. - -برای اعمال تمام تنظیمات در یک پروفایل, تنظیم `profile` تنظیمات. - -مثال: - -نصب `web` پرونده. - -``` sql -SET profile = 'web' -``` - -پروفایل تنظیمات در فایل پیکربندی کاربر اعلام کرد. این است که معمولا `users.xml`. - -مثال: - -``` xml - - - - - - 8 - - - - - 1000000000 - 100000000000 - - 1000000 - any - - 1000000 - 1000000000 - - 100000 - 100000000 - break - - 600 - 1000000 - 15 - - 25 - 100 - 50 - - 2 - 25 - 50 - 100 - - 1 - - -``` - -به عنوان مثال دو پروفایل مشخص: `default` و `web`. - -این `default` مشخصات دارای یک هدف خاص: همیشه باید وجود داشته باشد و در هنگام شروع سرور اعمال می شود. به عبارت دیگر `default` مشخصات شامل تنظیمات پیش فرض. - -این `web` مشخصات یک پروفایل به طور منظم است که می تواند با استفاده از مجموعه است `SET` پرسوجو یا استفاده از یک پارامتر نشانی وب در پرسوجو اچتیتیپی. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/settings/settings_profiles/) diff --git a/docs/fa/operations/settings/settings-users.md b/docs/fa/operations/settings/settings-users.md deleted file mode 100644 index c73f945e25a..00000000000 --- a/docs/fa/operations/settings/settings-users.md +++ /dev/null @@ -1,164 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 63 -toc_title: "\u062A\u0646\u0638\u06CC\u0645\u0627\u062A \u06A9\u0627\u0631\u0628\u0631" ---- - -# تنظیمات کاربر {#user-settings} - -این `users` بخش از `user.xml` فایل پیکربندی شامل تنظیمات کاربر. - -!!! note "اطلاعات" - فاحشه خانه نیز پشتیبانی می کند [گردش کار مبتنی بر مربع](../access-rights.md#access-control) برای مدیریت کاربران. ما توصیه می کنیم از این استفاده کنید. - -ساختار `users` بخش: - -``` xml - - - - - - - - 0|1 - - - - - profile_name - - default - - - - - expression - - - - - - -``` - -### نام / رمز عبور {#user-namepassword} - -رمز عبور را می توان در متن یا در شی256 (فرمت سحر و جادو) مشخص شده است. - -- برای اختصاص دادن رمز عبور به متن (**توصیه نمی شود**), جای خود را در یک `password` عنصر. - - به عنوان مثال, `qwerty`. رمز عبور را می توان خالی گذاشت. - - - -- برای اختصاص دادن رمز عبور با استفاده از هش ش256 در یک `password_sha256_hex` عنصر. - - به عنوان مثال, `65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5`. - - نمونه ای از نحوه تولید رمز عبور از پوسته: - - PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-' - - خط اول نتیجه رمز عبور است. خط دوم مربوط به هش ش256 است. - - - -- برای سازگاری با مشتریان خروجی زیر, رمز عبور را می توان در دو شی1 هش مشخص. محل را در `password_double_sha1_hex` عنصر. - - به عنوان مثال, `08b4a0f1de6ad37da17359e592c8d74788a83eb0`. - - نمونه ای از نحوه تولید رمز عبور از پوسته: - - PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-' - - خط اول نتیجه رمز عبور است. خط دوم مربوط به هش دو شی1 است. - -### مدیریت دسترسی {#access_management-user-setting} - -این تنظیم را قادر می سازد از غیر فعال با استفاده از گذاشتن محور [کنترل دسترسی و مدیریت حساب](../access-rights.md#access-control) برای کاربر. - -مقادیر ممکن: - -- 0 — Disabled. -- 1 — Enabled. - -مقدار پیش فرض: 0. - -### نام / شبکه {#user-namenetworks} - -لیست شبکه هایی که کاربر می تواند به سرور کلیک متصل شود. - -هر عنصر از لیست می توانید یکی از اشکال زیر را داشته باشد: - -- `` — IP address or network mask. - - مثالها: `213.180.204.3`, `10.0.0.1/8`, `10.0.0.1/255.255.255.0`, `2a02:6b8::3`, `2a02:6b8::3/64`, `2a02:6b8::3/ffff:ffff:ffff:ffff::`. - -- `` — Hostname. - - مثال: `example01.host.ru`. - - برای بررسی دسترسی یک پرسوجوی دی ان اس انجام میشود و تمامی نشانیهای اینترنتی برگشتی با نشانی همکار مقایسه میشوند. - -- `` — Regular expression for hostnames. - - مثال, `^example\d\d-\d\d-\d\.host\.ru$` - - برای بررسی دسترسی [جستجو](https://en.wikipedia.org/wiki/Reverse_DNS_lookup) برای نشانی همکار انجام می شود و سپس عبارت منظم مشخص شده اعمال می شود. سپس پرس و جو دی ان اس دیگری برای نتایج پرس و جو انجام می شود و تمامی نشانیهای دریافتی با نشانی همکار مقایسه می شوند. ما قویا توصیه می کنیم که عبارت منظم به پایان می رسد با$. - -تمام نتایج درخواست دی ان اس ذخیره سازی تا زمانی که سرور راه اندازی مجدد. - -**مثالها** - -برای باز کردن دسترسی برای کاربر از هر شبکه مشخص کنید: - -``` xml -::/0 -``` - -!!! warning "اخطار" - این نا امن برای باز کردن دسترسی از هر شبکه مگر اینکه شما یک فایروال به درستی پیکربندی و یا سرور به طور مستقیم به اینترنت متصل نیست. - -برای باز کردن دسترسی فقط از جایل هاست مشخص کنید: - -``` xml -::1 -127.0.0.1 -``` - -### نام / پروفایل {#user-nameprofile} - -شما می توانید یک پروفایل تنظیمات برای کاربر اختصاص دهید. پروفایل های تنظیمات در یک بخش جداگانه از پیکربندی `users.xml` پرونده. برای کسب اطلاعات بیشتر, دیدن [پروفایل تنظیمات](settings-profiles.md). - -### نام / سهمیه {#user-namequota} - -سهمیه اجازه می دهد شما را به پیگیری و یا محدود کردن استفاده از منابع بیش از یک دوره از زمان. سهمیه در پیکربندی `quotas` -بخش از `users.xml` فایل پیکربندی. - -شما می توانید یک سهمیه تعیین شده برای کاربر اختصاص. برای شرح مفصلی از پیکربندی سهمیه, دیدن [سهمیه](../quotas.md#quotas). - -### نام/پایگاه های داده {#user-namedatabases} - -در این بخش می توانید ردیف هایی را که توسط کلیک برای بازگشت هستند محدود کنید `SELECT` نمایش داده شد ساخته شده توسط کاربر فعلی, در نتیجه اجرای امنیت سطح ردیف پایه. - -**مثال** - -پیکربندی زیر نیروهای که کاربر `user1` فقط می توانید ردیف ها را ببینید `table1` به عنوان نتیجه `SELECT` نمایش داده شد که ارزش `id` درست است 1000. - -``` xml - - - - - id = 1000 - - - - -``` - -این `filter` می تواند هر بیان و در نتیجه [UInt8](../../sql-reference/data-types/int-uint.md)- نوع ارزش. این حالت معمولا شامل مقایسه و اپراتورهای منطقی. سطرها از `database_name.table1` از کجا نتایج فیلتر به 0 برای این کاربر بازگشت نیست. فیلتر کردن با ناسازگار است `PREWHERE` عملیات و معلولین `WHERE→PREWHERE` بهینهسازی. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/settings/settings_users/) diff --git a/docs/fa/operations/settings/settings.md b/docs/fa/operations/settings/settings.md deleted file mode 100644 index 61cb2a9793f..00000000000 --- a/docs/fa/operations/settings/settings.md +++ /dev/null @@ -1,1254 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd ---- - -# تنظیمات {#settings} - -## _شماره توزیع شده {#distributed-product-mode} - -تغییر رفتار [توزیع subqueries](../../sql-reference/operators/in.md). - -ClickHouse applies this setting when the query contains the product of distributed tables, i.e. when the query for a distributed table contains a non-GLOBAL subquery for the distributed table. - -محدودیت ها: - -- تنها برای اعمال در و پیوستن به subqueries. -- فقط اگر از بخش با استفاده از یک جدول توزیع حاوی بیش از یک سفال. -- اگر خرده فروشی مربوط به یک جدول توزیع حاوی بیش از یک سفال. -- برای ارزش جدول استفاده نمی شود [دور](../../sql-reference/table-functions/remote.md) تابع. - -مقادیر ممکن: - -- `deny` — Default value. Prohibits using these types of subqueries (returns the “Double-distributed in/JOIN subqueries is denied” استثنا). -- `local` — Replaces the database and table in the subquery with local ones for the destination server (shard), leaving the normal `IN`/`JOIN.` -- `global` — Replaces the `IN`/`JOIN` پرسوجو با `GLOBAL IN`/`GLOBAL JOIN.` -- `allow` — Allows the use of these types of subqueries. - -## تغییر در حسابهای کاربری دستگاه {#enable-optimize-predicate-expression} - -تبدیل به پیش فرض فشار در `SELECT` نمایش داده شد. - -افت فشار پیش فرض ممکن است به طور قابل توجهی کاهش ترافیک شبکه برای نمایش داده شد توزیع شده است. - -مقادیر ممکن: - -- 0 — Disabled. -- 1 — Enabled. - -مقدار پیش فرض: 1. - -استفاده - -پرس و جو های زیر را در نظر بگیرید: - -1. `SELECT count() FROM test_table WHERE date = '2018-10-10'` -2. `SELECT count() FROM (SELECT * FROM test_table) WHERE date = '2018-10-10'` - -اگر `enable_optimize_predicate_expression = 1`, سپس زمان اجرای این نمایش داده شد برابر است چرا که کلیکاوس اعمال می شود `WHERE` به خرده فروشی در هنگام پردازش. - -اگر `enable_optimize_predicate_expression = 0` سپس زمان اجرای پرس و جو دوم بسیار طولانی تر است زیرا `WHERE` بند مربوط به تمام داده ها پس از اتمام زیرخاکری. - -## شناسه بسته: {#settings-fallback_to_stale_replicas_for_distributed_queries} - -نیروهای پرس و جو به ماکت خارج از تاریخ اگر داده به روز شده در دسترس نیست. ببینید [تکرار](../../engines/table-engines/mergetree-family/replication.md). - -تاتر انتخاب مناسب ترین از کپی منسوخ شده از جدول. - -مورد استفاده در هنگام انجام `SELECT` از یک جدول توزیع شده است که اشاره به جداول تکرار. - -به طور پیش فرض, 1 (فعال). - -## اجبار {#settings-force_index_by_date} - -غیرفعال اجرای پرس و جو در صورتی که شاخص می تواند بر اساس تاریخ استفاده نمی شود. - -با جداول در خانواده ادغام کار می کند. - -اگر `force_index_by_date=1` چک چه پرس و جو وضعیت کلید تاریخ است که می تواند مورد استفاده قرار گیرد برای محدود کردن محدوده داده. اگر هیچ شرایط مناسب وجود دارد, این یک استثنا می اندازد. با این حال, بررسی نمی کند که وضعیت مقدار داده ها به خواندن را کاهش می دهد. مثلا, شرایط `Date != ' 2000-01-01 '` قابل قبول است حتی زمانی که منطبق بر تمام داده ها در جدول (به عنوان مثال در حال اجرا پرس و جو نیاز به اسکن کامل). برای کسب اطلاعات بیشتر در مورد محدوده داده ها در جداول ادغام, دیدن [ادغام](../../engines/table-engines/mergetree-family/mergetree.md). - -## اجبار {#force-primary-key} - -غیر فعال اعدام پرس و جو اگر نمایه سازی توسط کلید اصلی امکان پذیر نیست. - -با جداول در خانواده ادغام کار می کند. - -اگر `force_primary_key=1` چک خانه را ببینید اگر پرس و جو شرایط کلیدی اولیه است که می تواند مورد استفاده قرار گیرد برای محدود کردن محدوده داده است. اگر هیچ شرایط مناسب وجود دارد, این یک استثنا می اندازد. با این حال, بررسی نمی کند که وضعیت مقدار داده ها به خواندن را کاهش می دهد. برای کسب اطلاعات بیشتر در مورد محدوده داده ها در جداول ادغام, دیدن [ادغام](../../engines/table-engines/mergetree-family/mergetree.md). - -## قالب_نما {#format-schema} - -این پارامتر زمانی مفید است که شما با استفاده از فرمت های که نیاز به یک تعریف طرح, مانند [سروان نیا](https://capnproto.org/) یا [Protobuf](https://developers.google.com/protocol-buffers/). ارزش بستگی به فرمت. - -## دادههای سایت {#fsync-metadata} - -فعالسازی یا غیرفعال کردن [فوسینک](http://pubs.opengroup.org/onlinepubs/9699919799/functions/fsync.html) هنگام نوشتن `.sql` پرونده ها فعال به طور پیش فرض. - -این را حس می کند به غیر فعال کردن اگر سرور دارای میلیون ها جدول کوچک است که به طور مداوم در حال ایجاد و نابود شده است. - -## نصب و راه اندازی {#settings-enable_http_compression} - -را قادر می سازد و یا غیر فعال فشرده سازی داده ها در پاسخ به درخواست قام. - -برای کسب اطلاعات بیشتر, خواندن [توصیف واسط قام](../../interfaces/http.md). - -مقادیر ممکن: - -- 0 — Disabled. -- 1 — Enabled. - -مقدار پیش فرض: 0. - -## _تنظیم مجدد به حالت اولیه {#settings-http_zlib_compression_level} - -سطح فشرده سازی داده ها را در پاسخ به درخواست قام تنظیم می کند [قابلیت تنظیم صدا = 1](#settings-enable_http_compression). - -مقادیر ممکن: اعداد از 1 تا 9. - -مقدار پیش فرض: 3. - -## تغییر در حسابهای کاربری دستگاه {#settings-http_native_compression_disable_checksumming_on_decompress} - -را قادر می سازد و یا غیر فعال تایید کنترلی زمانی که از حالت فشرده خارج کردن اطلاعات ارسال قام از مشتری. فقط برای فرمت فشرده سازی بومی کلیک (با استفاده نمی شود `gzip` یا `deflate`). - -برای کسب اطلاعات بیشتر, خواندن [توصیف واسط قام](../../interfaces/http.md). - -مقادیر ممکن: - -- 0 — Disabled. -- 1 — Enabled. - -مقدار پیش فرض: 0. - -## نمایش سایت {#settings-send_progress_in_http_headers} - -فعالسازی یا غیرفعال کردن `X-ClickHouse-Progress` هدرهای پاسخ قام در `clickhouse-server` پاسخ. - -برای کسب اطلاعات بیشتر, خواندن [توصیف واسط قام](../../interfaces/http.md). - -مقادیر ممکن: - -- 0 — Disabled. -- 1 — Enabled. - -مقدار پیش فرض: 0. - -## عناصر {#setting-max_http_get_redirects} - -محدودیت حداکثر تعداد قام از رازک تغییر مسیر برای [URL](../../engines/table-engines/special/url.md)- جدول موتور . تنظیمات مربوط به هر دو نوع جداول: کسانی که ایجاد شده توسط [CREATE TABLE](../../sql-reference/statements/create.md#create-table-query) پرس و جو و توسط [نشانی وب](../../sql-reference/table-functions/url.md) تابع جدول. - -مقادیر ممکن: - -- هر عدد صحیح مثبت رازک. -- 0 — No hops allowed. - -مقدار پیش فرض: 0. - -## وارد کردن _فرست_مرزیابی _نمایش مجدد {#settings-input_format_allow_errors_num} - -حداکثر تعداد خطاهای قابل قبول را در هنگام خواندن از فرمت های متنی تنظیم می کند). - -مقدار پیش فرض 0 است. - -همیشه با جفت `input_format_allow_errors_ratio`. - -اگر یک خطا رخ داده است در حالی که خواندن ردیف اما ضد خطا است که هنوز هم کمتر از `input_format_allow_errors_num`, خانه را نادیده می گیرد ردیف و حرکت بر روی یک بعدی. - -اگر هر دو `input_format_allow_errors_num` و `input_format_allow_errors_ratio` بیش از حد, فاحشه خانه می اندازد یک استثنا. - -## ثبت نام {#settings-input_format_allow_errors_ratio} - -حداکثر درصد خطاها را در هنگام خواندن از فرمت های متنی تنظیم می کند.). -درصد خطاها به عنوان یک عدد ممیز شناور بین 0 و 1 تنظیم شده است. - -مقدار پیش فرض 0 است. - -همیشه با جفت `input_format_allow_errors_num`. - -اگر یک خطا رخ داده است در حالی که خواندن ردیف اما ضد خطا است که هنوز هم کمتر از `input_format_allow_errors_ratio`, خانه را نادیده می گیرد ردیف و حرکت بر روی یک بعدی. - -اگر هر دو `input_format_allow_errors_num` و `input_format_allow_errors_ratio` بیش از حد, فاحشه خانه می اندازد یک استثنا. - -## در حال خواندن: {#settings-input_format_values_interpret_expressions} - -را قادر می سازد و یا غیر فعال تجزیه کننده کامل گذاشتن اگر تجزیه کننده جریان سریع نمی تواند تجزیه داده ها. این تنظیم فقط برای [مقادیر](../../interfaces/formats.md#data-format-values) فرمت در درج داده ها. برای کسب اطلاعات بیشتر در مورد تجزیه نحو, دیدن [نحو](../../sql-reference/syntax.md) بخش. - -مقادیر ممکن: - -- 0 — Disabled. - - در این مورد, شما باید داده های فرمت شده را فراهم. دیدن [فرشها](../../interfaces/formats.md) بخش. - -- 1 — Enabled. - - در این مورد, شما می توانید یک عبارت گذاشتن به عنوان یک ارزش استفاده, اما درج داده است این راه بسیار کندتر. اگر شما وارد کردن داده های فرمت شده تنها, سپس کلیک کنیدهاوس رفتار به عنوان اگر مقدار تنظیم است 0. - -مقدار پیش فرض: 1. - -مثال استفاده - -درج [DateTime](../../sql-reference/data-types/datetime.md) ارزش نوع با تنظیمات مختلف. - -``` sql -SET input_format_values_interpret_expressions = 0; -INSERT INTO datetime_t VALUES (now()) -``` - -``` text -Exception on client: -Code: 27. DB::Exception: Cannot parse input: expected ) before: now()): (at row 1) -``` - -``` sql -SET input_format_values_interpret_expressions = 1; -INSERT INTO datetime_t VALUES (now()) -``` - -``` text -Ok. -``` - -پرس و جو گذشته معادل به شرح زیر است: - -``` sql -SET input_format_values_interpret_expressions = 0; -INSERT INTO datetime_t SELECT now() -``` - -``` text -Ok. -``` - -## در حال خواندن: {#settings-input_format_values_deduce_templates_of_expressions} - -را قادر می سازد و یا غیر فعال کسر الگو برای عبارات گذاشتن در [مقادیر](../../interfaces/formats.md#data-format-values) قالب. این اجازه می دهد تجزیه و تفسیر عبارات در `Values` بسیار سریع تر اگر عبارات در ردیف متوالی همان ساختار. تاتر تلاش می کند به استنباط قالب یک عبارت, تجزیه ردیف زیر با استفاده از این الگو و ارزیابی بیان در یک دسته از ردیف موفقیت تجزیه. - -مقادیر ممکن: - -- 0 — Disabled. -- 1 — Enabled. - -مقدار پیش فرض: 1. - -برای پرس و جو زیر: - -``` sql -INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), (upper('Values')), ... -``` - -- اگر `input_format_values_interpret_expressions=1` و `format_values_deduce_templates_of_expressions=0`, عبارات به طور جداگانه برای هر سطر تفسیر (این برای تعداد زیادی از ردیف بسیار کند است). -- اگر `input_format_values_interpret_expressions=0` و `format_values_deduce_templates_of_expressions=1`, عبارات در اولین, ردیف دوم و سوم با استفاده از الگو تجزیه `lower(String)` و با هم تفسیر, بیان در ردیف جلو با قالب دیگری تجزیه (`upper(String)`). -- اگر `input_format_values_interpret_expressions=1` و `format_values_deduce_templates_of_expressions=1`, همان است که در مورد قبلی, بلکه اجازه می دهد تا عقب نشینی به تفسیر عبارات به طور جداگانه اگر این امکان وجود ندارد به استنباط الگو. - -## وارد کردن _تماس_عول_ایجاد _شکلتهای _شخصی {#settings-input-format-values-accurate-types-of-literals} - -این تنظیم تنها زمانی استفاده می شود `input_format_values_deduce_templates_of_expressions = 1`. این می تواند رخ دهد, که عبارت برای برخی از ستون دارای ساختار مشابه, اما حاوی لیتر عددی از انواع مختلف, به عنوان مثال - -``` sql -(..., abs(0), ...), -- UInt64 literal -(..., abs(3.141592654), ...), -- Float64 literal -(..., abs(-1), ...), -- Int64 literal -``` - -مقادیر ممکن: - -- 0 — Disabled. - - In this case, ClickHouse may use a more general type for some literals (e.g., `Float64` یا `Int64` به جای `UInt64` برای `42`), اما ممکن است مشکلات سرریز و دقت شود. - -- 1 — Enabled. - - در این مورد, تاتر چک نوع واقعی تحت اللفظی و با استفاده از یک قالب بیان از نوع مربوطه. در بعضی موارد, ممکن است به طور قابل توجهی کاهش سرعت ارزیابی بیان در `Values`. - -مقدار پیش فرض: 1. - -## _پوشه های ورودی و خروجی {#session_settings-input_format_defaults_for_omitted_fields} - -هنگام انجام `INSERT` نمایش داده شد, جایگزین مقادیر ستون ورودی حذف شده با مقادیر پیش فرض از ستون مربوطه. این گزینه فقط برای اعمال [جیسانچرو](../../interfaces/formats.md#jsoneachrow), [CSV](../../interfaces/formats.md#csv) و [جدول دار](../../interfaces/formats.md#tabseparated) فرمتها. - -!!! note "یادداشت" - هنگامی که این گزینه فعال است, ابرداده جدول طولانی از سرور به مشتری ارسال. این مصرف منابع محاسباتی اضافی بر روی سرور و می تواند عملکرد را کاهش دهد. - -مقادیر ممکن: - -- 0 — Disabled. -- 1 — Enabled. - -مقدار پیش فرض: 1. - -## پیشسو {#settings-input-format-tsv-empty-as-default} - -هنگامی که فعال, جایگزین زمینه های ورودی خالی در فیلم با مقادیر پیش فرض. برای عبارات پیش فرض پیچیده `input_format_defaults_for_omitted_fields` باید بیش از حد فعال شود. - -غیر فعال به طور پیش فرض. - -## خرابی در حذف گواهینامهها {#settings-input-format-null-as-default} - -را قادر می سازد و یا غیر فعال با استفاده از مقادیر پیش فرض اگر داده های ورودی شامل `NULL`, اما نوع داده از ستون مربوطه در نمی `Nullable(T)` (برای فرمت های ورودی متن). - -## _دفتر_صرفههای شناسنامهی ورودی {#settings-input-format-skip-unknown-fields} - -را قادر می سازد و یا غیر فعال پرش درج داده های اضافی. - -در هنگام نوشتن داده ها, تاتر می اندازد یک استثنا اگر داده های ورودی حاوی ستون که در جدول هدف وجود ندارد. اگر پرش فعال است, تاتر می کند داده های اضافی وارد کنید و یک استثنا پرتاب نمی. - -فرمت های پشتیبانی شده: - -- [جیسانچرو](../../interfaces/formats.md#jsoneachrow) -- [اطلاعات دقیق](../../interfaces/formats.md#csvwithnames) -- [اطلاعات دقیق](../../interfaces/formats.md#tabseparatedwithnames) -- [TSKV](../../interfaces/formats.md#tskv) - -مقادیر ممکن: - -- 0 — Disabled. -- 1 — Enabled. - -مقدار پیش فرض: 0. - -## تغییر _کم_تر_تنظیم مجدد _جنسان {#settings-input_format_import_nested_json} - -درج دادههای جسون را با اشیای تو در تو فعال یا غیرفعال میکند. - -فرمت های پشتیبانی شده: - -- [جیسانچرو](../../interfaces/formats.md#jsoneachrow) - -مقادیر ممکن: - -- 0 — Disabled. -- 1 — Enabled. - -مقدار پیش فرض: 0. - -همچنین نگاه کنید به: - -- [استفاده از ساختارهای تو در تو](../../interfaces/formats.md#jsoneachrow-nested) با `JSONEachRow` قالب. - -## _فرست_ام_امنمایش گذرواژه {#settings-input-format-with-names-use-header} - -را قادر می سازد و یا غیر فعال چک کردن سفارش ستون در هنگام قرار دادن داده ها. - -برای بهبود عملکرد درج, توصیه می کنیم غیر فعال کردن این چک اگر شما اطمینان حاصل کنید که سفارش ستون از داده های ورودی همان است که در جدول هدف است. - -فرمت های پشتیبانی شده: - -- [اطلاعات دقیق](../../interfaces/formats.md#csvwithnames) -- [اطلاعات دقیق](../../interfaces/formats.md#tabseparatedwithnames) - -مقادیر ممکن: - -- 0 — Disabled. -- 1 — Enabled. - -مقدار پیش فرض: 1. - -## تغییر _شماره {#settings-date_time_input_format} - -اجازه می دهد تا انتخاب تجزیه کننده از نمایش متن از تاریخ و زمان. - -تنظیمات برای اعمال نمی شود [توابع تاریخ و زمان](../../sql-reference/functions/date-time-functions.md). - -مقادیر ممکن: - -- `'best_effort'` — Enables extended parsing. - - تاتر می توانید پایه تجزیه `YYYY-MM-DD HH:MM:SS` فرمت و همه [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) فرمت های تاریخ و زمان. به عنوان مثال, `'2018-06-08T01:02:03.000Z'`. - -- `'basic'` — Use basic parser. - - تاتر می توانید تنها پایه تجزیه `YYYY-MM-DD HH:MM:SS` قالب. به عنوان مثال, `'2019-08-20 10:18:56'`. - -مقدار پیشفرض: `'basic'`. - -همچنین نگاه کنید به: - -- [نوع داده حسگر ناحیه رنگی.](../../sql-reference/data-types/datetime.md) -- [توابع برای کار با تاریخ و زمان.](../../sql-reference/functions/date-time-functions.md) - -## بررسی اجمالی {#settings-join_default_strictness} - -مجموعه سختی پیش فرض برای [تاریخ بند](../../sql-reference/statements/select/join.md#select-join). - -مقادیر ممکن: - -- `ALL` — If the right table has several matching rows, ClickHouse creates a [محصول دکارتی](https://en.wikipedia.org/wiki/Cartesian_product) از تطبیق ردیف. این طبیعی است `JOIN` رفتار از استاندارد گذاشتن. -- `ANY` — If the right table has several matching rows, only the first one found is joined. If the right table has only one matching row, the results of `ANY` و `ALL` یکسان هستند. -- `ASOF` — For joining sequences with an uncertain match. -- `Empty string` — If `ALL` یا `ANY` در پرس و جو مشخص نشده است, خانه عروسکی می اندازد یک استثنا. - -مقدار پیشفرض: `ALL`. - -## نمایش سایت {#settings-join_any_take_last_row} - -تغییرات رفتار پیوستن به عملیات با `ANY` سخت بودن. - -!!! warning "توجه" - این تنظیم فقط برای `JOIN` عملیات با [پیوستن](../../engines/table-engines/special/join.md) جداول موتور. - -مقادیر ممکن: - -- 0 — If the right table has more than one matching row, only the first one found is joined. -- 1 — If the right table has more than one matching row, only the last one found is joined. - -مقدار پیش فرض: 0. - -همچنین نگاه کنید به: - -- [پیوستن بند](../../sql-reference/statements/select/join.md#select-join) -- [پیوستن به موتور جدول](../../engines/table-engines/special/join.md) -- [بررسی اجمالی](#settings-join_default_strictness) - -## ارزشهای خبری عبارتند از: {#join_use_nulls} - -نوع را تنظیم می کند [JOIN](../../sql-reference/statements/select/join.md) رفتار هنگامی که ادغام جداول سلول های خالی ممکن است ظاهر شود. کلیک هاوس بر اساس این تنظیم متفاوت است. - -مقادیر ممکن: - -- 0 — The empty cells are filled with the default value of the corresponding field type. -- 1 — `JOIN` رفتار به همان شیوه به عنوان در گذاشتن استاندارد. نوع زمینه مربوطه به تبدیل [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable) سلول های خالی پر شده اند [NULL](../../sql-reference/syntax.md). - -مقدار پیش فرض: 0. - -## ت_مایش بیشینه {#setting-max_block_size} - -در خانه, داده ها توسط بلوک های پردازش (مجموعه ای از قطعات ستون). چرخه پردازش داخلی برای یک بلوک به اندازه کافی موثر هستند, اما هزینه های قابل توجه در هر بلوک وجود دارد. این `max_block_size` تنظیم یک توصیه برای چه اندازه بلوک (در تعداد ردیف) برای بارگذاری از جداول است. اندازه بلوک نباید بیش از حد کوچک, به طوری که هزینه در هر بلوک هنوز هم قابل توجه است, اما نه بیش از حد بزرگ به طوری که پرس و جو با محدودیت است که پس از اولین بلوک به سرعت پردازش تکمیل. هدف این است که برای جلوگیری از مصرف حافظه بیش از حد در هنگام استخراج تعداد زیادی از ستون ها در موضوعات مختلف و برای حفظ حداقل برخی از محل کش. - -مقدار پیش فرض: 65,536. - -بلوک اندازه `max_block_size` همیشه از جدول لود نمی. اگر واضح است که داده های کمتر نیاز به بازیابی یک بلوک کوچکتر پردازش می شود. - -## ترجی_حات {#preferred-block-size-bytes} - -مورد استفاده برای همان هدف به عنوان `max_block_size` اما با تطبیق تعداد سطرها در بلوک اندازه بلوک توصیه شده را در بایت تنظیم می کند. -با این حال, اندازه بلوک نمی تواند بیش از `max_block_size` ردیف -به طور پیش فرض: 1,000,000. تنها در هنگام خواندن از موتورهای ادغام کار می کند. - -## ادغام _تر_م_را_م_مایش مجدد {#setting-merge-tree-min-rows-for-concurrent-read} - -اگر تعداد ردیف از یک فایل از یک خوانده شود [ادغام](../../engines/table-engines/mergetree-family/mergetree.md) جدول بیش از `merge_tree_min_rows_for_concurrent_read` سپس کلیک کنیدهاوس تلاش می کند برای انجام خواندن همزمان از این فایل در موضوعات مختلف. - -مقادیر ممکن: - -- هر عدد صحیح مثبت. - -مقدار پیش فرض: 163840. - -## _انتقال به _انتقال به _شخصی {#setting-merge-tree-min-bytes-for-concurrent-read} - -اگر تعداد بایت برای خواندن از یک فایل از یک [ادغام](../../engines/table-engines/mergetree-family/mergetree.md)- جدول موتور بیش از `merge_tree_min_bytes_for_concurrent_read` سپس کلیک کنیدهاوس تلاش می کند به صورت همزمان از این فایل در موضوعات مختلف به عنوان خوانده شده. - -مقدار ممکن: - -- هر عدد صحیح مثبت. - -مقدار پیش فرض: 251658240. - -## ادغام _تر_م_را_م_را_مایش مجدد {#setting-merge-tree-min-rows-for-seek} - -اگر فاصله بین دو بلوک داده در یک فایل خوانده شود کمتر از `merge_tree_min_rows_for_seek` ردیف, سپس کلیک می کند از طریق فایل به دنبال ندارد اما می خواند پی در پی داده ها. - -مقادیر ممکن: - -- هر عدد صحیح مثبت. - -مقدار پیش فرض: 0. - -## ادغام _تر_حضربه _ترکمال {#setting-merge-tree-min-bytes-for-seek} - -اگر فاصله بین دو بلوک داده در یک فایل خوانده شود کمتر از `merge_tree_min_bytes_for_seek` بایت, سپس پی در پی تاتر می خواند طیف وسیعی از فایل است که شامل هر دو بلوک, در نتیجه اجتناب اضافی به دنبال. - -مقادیر ممکن: - -- هر عدد صحیح مثبت. - -مقدار پیش فرض: 0. - -## ادغام _تر_کوارسی_یندگرمانی {#setting-merge-tree-coarse-index-granularity} - -هنگامی که جستجو برای داده ها, تاتر چک علامت داده ها در فایل شاخص. اگر فاحشه خانه می یابد که کلید های مورد نیاز در برخی از محدوده هستند, این تقسیم این محدوده به `merge_tree_coarse_index_granularity` موشک و جستجو کلید های مورد نیاز وجود دارد به صورت بازگشتی. - -مقادیر ممکن: - -- هر عدد صحیح حتی مثبت. - -مقدار پیش فرض: 8. - -## _انتقال به _انتقال {#setting-merge-tree-max-rows-to-use-cache} - -اگر کلیک خانه باید بیش از خواندن `merge_tree_max_rows_to_use_cache` ردیف ها در یک پرس و جو از کش بلوک های غیر فشرده استفاده نمی کنند. - -ذخیره سازی داده های ذخیره شده بلوک های غیر فشرده برای نمایش داده شد. تاتر با استفاده از این کش برای سرعت بخشیدن به پاسخ به نمایش داده شد کوچک تکرار شده است. این تنظیم محافظت از کش از سطل زباله توسط نمایش داده شد که مقدار زیادی از داده ها به عنوان خوانده شده. این [_بالا](../server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) تنظیم سرور اندازه کش از بلوک های غیر فشرده را تعریف می کند. - -مقادیر ممکن: - -- هر عدد صحیح مثبت. - -Default value: 128 ✕ 8192. - -## _انتقال به _انتقال {#setting-merge-tree-max-bytes-to-use-cache} - -اگر کلیک خانه باید بیش از خواندن `merge_tree_max_bytes_to_use_cache` بایت در یک پرس و جو, این کش از بلوک های غیر فشرده استفاده نمی. - -ذخیره سازی داده های ذخیره شده بلوک های غیر فشرده برای نمایش داده شد. تاتر با استفاده از این کش برای سرعت بخشیدن به پاسخ به نمایش داده شد کوچک تکرار شده است. این تنظیم محافظت از کش از سطل زباله توسط نمایش داده شد که مقدار زیادی از داده ها به عنوان خوانده شده. این [_بالا](../server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) تنظیم سرور اندازه کش از بلوک های غیر فشرده را تعریف می کند. - -مقدار ممکن: - -- هر عدد صحیح مثبت. - -مقدار پیش فرض: 2013265920. - -## _عنوان _تو_میشه {#settings-min-bytes-to-use-direct-io} - -حداقل حجم داده های مورد نیاز برای استفاده از دسترسی مستقیم به دیسک ذخیره سازی. - -تاتر با استفاده از این تنظیم در هنگام خواندن داده ها از جداول. اگر حجم کل ذخیره سازی از تمام داده ها به عنوان خوانده شده بیش از `min_bytes_to_use_direct_io` بایت, سپس کلیک هاوس می خواند داده ها از دیسک ذخیره سازی با `O_DIRECT` انتخاب - -مقادیر ممکن: - -- 0 — Direct I/O is disabled. -- عدد صحیح مثبت. - -مقدار پیش فرض: 0. - -## _خروج {#settings-log-queries} - -راه اندازی ورود به سیستم پرس و جو. - -نمایش داده شد با توجه به قوانین در به کلیک خانه فرستاده می شود [_خروج](../server-configuration-parameters/settings.md#server_configuration_parameters-query-log) پارامتر پیکربندی سرور. - -مثال: - -``` text -log_queries=1 -``` - -## _قاب کردن _نوع {#settings-log-queries-min-type} - -`query_log` حداقل نوع برای ورود به سیستم. - -مقادیر ممکن: -- `QUERY_START` (`=1`) -- `QUERY_FINISH` (`=2`) -- `EXCEPTION_BEFORE_START` (`=3`) -- `EXCEPTION_WHILE_PROCESSING` (`=4`) - -مقدار پیشفرض: `QUERY_START`. - -می توان برای محدود کردن که `query_log`, می گویند شما جالب تنها در اشتباهات هستند, سپس شما می توانید استفاده کنید `EXCEPTION_WHILE_PROCESSING`: - -``` text -log_queries_min_type='EXCEPTION_WHILE_PROCESSING' -``` - -## باز کردن {#settings-log-query-threads} - -راه اندازی موضوعات پرس و جو ورود به سیستم. - -نمایش داده شد' موضوعات runned توسط ClickHouse با راه اندازی این سیستم هستند با توجه به قوانین در [_ر_خروج](../server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) پارامتر پیکربندی سرور. - -مثال: - -``` text -log_query_threads=1 -``` - -## ا_فزونهها {#settings-max_insert_block_size} - -اندازه بلوک به شکل برای درج به یک جدول. -این تنظیم فقط در مواردی که سرور بلوک را تشکیل می دهد اعمال می شود. -برای مثال برای درج از طریق رابط اچ.تی. تی. پی سرور تجزیه فرمت داده ها و اشکال بلوک از اندازه مشخص شده است. -اما هنگامی که با استفاده از کلیک-مشتری تجزیه داده های خود را و ‘max_insert_block_size’ تنظیم بر روی سرور به اندازه بلوک قرار داده تاثیر نمی گذارد. -تنظیمات نیز یک هدف در هنگام استفاده از درج را انتخاب کنید ندارد, از داده ها با استفاده از بلوک های مشابه که پس از انتخاب تشکیل قرار داده. - -مقدار پیش فرض: 1,048,576. - -به طور پیش فرض کمی بیش از `max_block_size`. دلیل این کار این است زیرا موتورهای جدول خاص (`*MergeTree`) بخش داده ها بر روی دیسک برای هر بلوک قرار داده شده است که یک نهاد نسبتا بزرگ را تشکیل می دهند. به طور مشابه, `*MergeTree` جداول مرتب سازی بر داده ها در هنگام درج و اندازه بلوک به اندازه کافی بزرگ اجازه می دهد مرتب سازی داده های بیشتر در رم. - -## _معرض _سبک_ز_وز {#min-insert-block-size-rows} - -مجموعه حداقل تعداد ردیف در بلوک است که می تواند به یک جدول توسط یک قرار داده `INSERT` پرس و جو. بلوک های کوچکتر به اندازه به موارد بزرگتر له می شوند. - -مقادیر ممکن: - -- عدد صحیح مثبت. -- 0 — Squashing disabled. - -مقدار پیش فرض: 1048576. - -## ا_فزونهها {#min-insert-block-size-bytes} - -مجموعه حداقل تعداد بایت در بلوک است که می تواند به یک جدول توسط یک قرار داده `INSERT` پرس و جو. بلوک های کوچکتر به اندازه به موارد بزرگتر له می شوند. - -مقادیر ممکن: - -- عدد صحیح مثبت. -- 0 — Squashing disabled. - -مقدار پیش فرض: 268435456. - -## _شروع مجدد _شروع مجدد _شروع مجدد _کاربری {#settings-max_replica_delay_for_distributed_queries} - -غیرفعال تاخیر کپی برای نمایش داده شد توزیع شده است. ببینید [تکرار](../../engines/table-engines/mergetree-family/replication.md). - -زمان را در عرض چند ثانیه تنظیم می کند. اگر یک ماکت نشدم بیش از ارزش مجموعه, این ماکت استفاده نمی شود. - -مقدار پیش فرض: 300. - -مورد استفاده در هنگام انجام `SELECT` از یک جدول توزیع شده است که اشاره به جداول تکرار. - -## _مخفی کردن {#settings-max_threads} - -حداکثر تعداد موضوعات پردازش پرس و جو, به جز موضوعات برای بازیابی داده ها از سرور از راه دور (دیدن ‘max_distributed_connections’ پارامتر). - -این پارامتر شامل موضوعات است که انجام همان مراحل از خط لوله پردازش پرس و جو به صورت موازی. -مثلا, در هنگام خواندن از یک جدول, اگر ممکن است به ارزیابی عبارات با توابع, فیلتر با کجا و از پیش جمع شده برای گروه به صورت موازی با استفاده از حداقل ‘max_threads’ تعداد موضوعات, سپس ‘max_threads’ استفاده می شود. - -مقدار پیش فرض: تعداد هسته های پردازنده فیزیکی. - -اگر کمتر از یک پرس و جو را انتخاب کنید به طور معمول بر روی یک سرور در یک زمان اجرا, تنظیم این پارامتر به یک مقدار کمی کمتر از تعداد واقعی هسته پردازنده. - -برای نمایش داده شد که به سرعت به دلیل محدودیت تکمیل, شما می توانید یک مجموعه پایین تر ‘max_threads’. مثلا , اگر تعداد لازم از نوشته در هر بلوک و حداکثر _سرخ واقع = 8, سپس 8 بلوک بازیابی می شوند, اگر چه این امر می توانست به اندازه کافی برای خواندن فقط یک بوده است. - -کوچکتر `max_threads` ارزش حافظه کمتر مصرف می شود. - -## ا_فزونهها {#settings-max-insert-threads} - -حداکثر تعداد موضوعات برای اجرای `INSERT SELECT` پرس و جو. - -مقادیر ممکن: - -- 0 (or 1) — `INSERT SELECT` بدون اجرای موازی. -- عدد صحیح مثبت. بزرگتر از 1. - -مقدار پیش فرض: 0. - -موازی `INSERT SELECT` اثر تنها در صورتی که `SELECT` بخش به صورت موازی اجرا, دیدن [_مخفی کردن](#settings-max_threads) تنظیمات. -مقادیر بالاتر به استفاده از حافظه بالاتر منجر شود. - -## _بزرگنمایی {#max-compress-block-size} - -حداکثر اندازه بلوک از داده های غیر فشرده قبل از فشرده سازی برای نوشتن به یک جدول. به طور پیش فرض, 1,048,576 (1 مگابایت). اگر اندازه کاهش می یابد, میزان فشرده سازی به طور قابل توجهی کاهش می یابد, سرعت فشرده سازی و رفع فشار کمی با توجه به محل کش را افزایش می دهد, و مصرف حافظه کاهش می یابد. معمولا وجود دارد هر دلیلی برای تغییر این تنظیم نیست. - -هنوز بلوک برای فشرده سازی اشتباه نیست (یک تکه از حافظه متشکل از بایت) با بلوک برای پردازش پرس و جو (مجموعه ای از ردیف از یک جدول). - -## _بزرگنمایی {#min-compress-block-size} - -برای [ادغام](../../engines/table-engines/mergetree-family/mergetree.md)"جداول . به منظور کاهش زمان تاخیر در هنگام پردازش نمایش داده شد, یک بلوک فشرده شده است در هنگام نوشتن علامت بعدی اگر اندازه خود را حداقل ‘min_compress_block_size’. به طور پیش فرض 65,536. - -اندازه واقعی بلوک, اگر داده غیر فشرده کمتر از است ‘max_compress_block_size’, کمتر از این مقدار و کمتر از حجم داده ها برای یک علامت. - -بیایید نگاهی به عنوان مثال. فرض کنیم که ‘index_granularity’ در طول ایجاد جدول به 8192 تنظیم شد. - -ما در حال نوشتن یک ستون نوع 32 (4 بایت در هر مقدار). هنگام نوشتن 8192 ردیف, کل خواهد بود 32 کیلوبایت داده. پس min_compress_block_size = 65,536 یک فشرده بلوک تشکیل خواهد شد برای هر دو نشانه است. - -ما در حال نوشتن یک ستون نشانی اینترنتی با نوع رشته (اندازه متوسط 60 بایت در هر مقدار). هنگام نوشتن 8192 ردیف, متوسط خواهد بود کمی کمتر از 500 کیلوبایت داده. پس از این بیش از است 65,536, یک بلوک فشرده خواهد شد برای هر علامت تشکیل. در این مورد, در هنگام خواندن داده ها از دیسک در طیف وسیعی از یک علامت, اطلاعات اضافی نمی خواهد از حالت فشرده خارج شود. - -معمولا وجود دارد هر دلیلی برای تغییر این تنظیم نیست. - -## بیشینه_کرکی_سیز {#settings-max_query_size} - -حداکثر بخشی از پرس و جو است که می تواند به رم برای تجزیه با تجزیه کننده گذاشتن گرفته شده است. -پرس و جو درج همچنین شامل داده ها برای درج است که توسط تجزیه کننده جریان جداگانه پردازش (که مصرف درجه(1) رم), است که در این محدودیت شامل نمی شود. - -مقدار پیش فرض: 256 کیلوبایت. - -## فعالسازی _دلای {#interactive-delay} - -فاصله در میکروثانیه برای بررسی اینکه اجرای درخواست لغو شده است و ارسال پیشرفت. - -مقدار پیش فرض: 100,000 (چک برای لغو و پیشرفت می فرستد ده بار در ثانیه). - -## connect_timeout, receive_timeout, send_timeout {#connect-timeout-receive-timeout-send-timeout} - -وقفه در ثانیه بر روی سوکت مورد استفاده برای برقراری ارتباط با مشتری. - -مقدار پیش فرض: 10, 300, 300. - -## _انتقال به _ار_خروج {#cancel-http-readonly-queries-on-client-close} - -Cancels HTTP read-only queries (e.g. SELECT) when a client closes the connection without waiting for the response. - -مقدار پیشفرض: 0 - -## پول_نتروال {#poll-interval} - -قفل در یک حلقه انتظار برای تعداد مشخصی از ثانیه. - -مقدار پیش فرض: 10. - -## _ادغام گیر {#max-distributed-connections} - -حداکثر تعداد اتصالات همزمان با سرور از راه دور برای پردازش توزیع از یک پرس و جو تنها به یک جدول توزیع. ما توصیه می کنیم تنظیم یک مقدار کمتر از تعداد سرور در خوشه. - -مقدار پیش فرض: 1024. - -پارامترهای زیر فقط هنگام ایجاد جداول توزیع شده (و هنگام راه اندازی یک سرور) استفاده می شود بنابراین هیچ دلیلی برای تغییر در زمان اجرا وجود ندارد. - -## نمایش سایت {#distributed-connections-pool-size} - -حداکثر تعداد اتصالات همزمان با سرور از راه دور برای پردازش توزیع از همه نمایش داده شد به یک جدول توزیع شده است. ما توصیه می کنیم تنظیم یک مقدار کمتر از تعداد سرور در خوشه. - -مقدار پیش فرض: 1024. - -## _انتقال به _مزاح_اف_کننده {#connect-timeout-with-failover-ms} - -فاصله در میلی ثانیه برای اتصال به یک سرور از راه دور برای یک موتور جدول توزیع, اگر ‘shard’ و ‘replica’ بخش ها در تعریف خوشه استفاده می شود. -اگر ناموفق, چندین تلاش برای اتصال به کپی های مختلف ساخته شده. - -مقدار پیش فرض: 50. - -## قابلیت اتصال به شبکه {#connections-with-failover-max-tries} - -حداکثر تعداد تلاش اتصال با هر ماکت برای موتور جدول توزیع. - -مقدار پیش فرض: 3. - -## افراط {#extremes} - -اینکه مقادیر شدید (حداقل و حداکثر در ستون یک نتیجه پرس و جو) شمارش شود. می پذیرد 0 یا 1. به طور پیش فرض, 0 (غیر فعال). -برای کسب اطلاعات بیشتر به بخش مراجعه کنید “Extreme values”. - -## همترازی پایین {#setting-use_uncompressed_cache} - -اینکه از یک کش از بلوکهای غیر فشرده استفاده شود یا خیر. می پذیرد 0 یا 1. به طور پیش فرض, 0 (غیر فعال). -با استفاده از کش غیر فشرده (فقط برای جداول در خانواده ادغام) می تواند به طور قابل توجهی کاهش زمان تاخیر و افزایش توان در هنگام کار با تعداد زیادی از نمایش داده شد کوتاه است. فعال کردن این تنظیم برای کاربرانی که ارسال درخواست کوتاه مکرر. همچنین با توجه به پرداخت [_بالا](../server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) configuration parameter (only set in the config file) – the size of uncompressed cache blocks. By default, it is 8 GiB. The uncompressed cache is filled in as needed and the least-used data is automatically deleted. - -برای نمایش داده شد که خواندن حداقل حجم تا حدودی زیادی از داده ها (یک میلیون ردیف یا بیشتر) غیر فشرده کش غیر فعال است به طور خودکار به صرفه جویی در فضا برای واقعا کوچک نمایش داده شد. این به این معنی است که شما می توانید نگه دارید ‘use_uncompressed_cache’ تنظیم همیشه به مجموعه 1. - -## جایگزینی _خروج {#replace-running-query} - -هنگام استفاده از رابط قام ‘query_id’ پارامتر را می توان گذشت. این هر رشته که به عنوان شناسه پرس و جو در خدمت است. -اگر پرس و جو از همان کاربر با همان ‘query_id’ در حال حاضر در این زمان وجود دارد, رفتار بستگی به ‘replace_running_query’ پارامتر. - -`0` (default) – Throw an exception (don't allow the query to run if a query with the same ‘query_id’ در حال حاضر در حال اجرا). - -`1` – Cancel the old query and start running the new one. - -یاندکسمتریکا با استفاده از این پارامتر را به 1 برای اجرای پیشنهادات خود را برای شرایط تقسیم بندی. پس از ورود به شخصیت بعدی, اگر پرس و جو قدیمی هنوز تمام نشده است, باید لغو شود. - -## _خاله جریان {#stream-flush-interval-ms} - -این نسخهها کار میکند برای جداول با جریان در مورد یک ایست, و یا زمانی که یک موضوع تولید [ا_فزونهها](#settings-max_insert_block_size) ردیف - -مقدار پیش فرض 7500 است. - -کوچکتر ارزش, اطلاعات بیشتر به جدول سرخ. تنظیم مقدار خیلی کم منجر به عملکرد ضعیف می شود. - -## _تبالسازی {#settings-load_balancing} - -تعیین الگوریتم انتخاب کپی است که برای پردازش پرس و جو توزیع استفاده. - -کلیک هاوس از الگوریتم های زیر برای انتخاب کپی ها پشتیبانی می کند: - -- [تصادفی](#load_balancing-random) (به طور پیش فرض) -- [نزدیکترین نام میزبان](#load_balancing-nearest_hostname) -- [به ترتیب](#load_balancing-in_order) -- [اول یا تصادفی](#load_balancing-first_or_random) - -### تصادفی (به طور پیش فرض) {#load_balancing-random} - -``` sql -load_balancing = random -``` - -تعداد خطاها برای هر ماکت شمارش. پرس و جو با کمترین خطاها به ماکت ارسال می شود و اگر چندین مورد از این موارد وجود داشته باشد به هر کسی. -معایب: نزدیکی سرور برای خود اختصاص نمی; اگر کپی داده های مختلف, شما همچنین می خواهد داده های مختلف. - -### نزدیکترین نام میزبان {#load_balancing-nearest_hostname} - -``` sql -load_balancing = nearest_hostname -``` - -The number of errors is counted for each replica. Every 5 minutes, the number of errors is integrally divided by 2. Thus, the number of errors is calculated for a recent time with exponential smoothing. If there is one replica with a minimal number of errors (i.e. errors occurred recently on the other replicas), the query is sent to it. If there are multiple replicas with the same minimal number of errors, the query is sent to the replica with a hostname that is most similar to the server's hostname in the config file (for the number of different characters in identical positions, up to the minimum length of both hostnames). - -مثلا example01-01-1 و example01-01-2.yandex.ru متفاوت هستند در یک موقعیت در حالی که example01-01-1 و example01-02-2 متفاوت در دو مکان است. -این روش ممکن است ابتدایی به نظر برسد اما اطلاعات خارجی در مورد توپولوژی شبکه نیاز ندارد و نشانی های اینترنتی را مقایسه نمی کند که برای نشانیهای اینترنتی6 پیچیده خواهد بود. - -بدین ترتیب, اگر کپی معادل وجود دارد, نزدیک ترین یک به نام ترجیح داده می شود. -ما همچنین می توانیم فرض کنیم که در هنگام ارسال یک پرس و جو به همان سرور در صورت عدم وجود شکست توزیع پرس و جو نیز به همان سرور. بنابراین حتی اگر داده های مختلف بر روی کپی قرار داده شده, پرس و جو بیشتر همان نتایج بازگشت. - -### به ترتیب {#load_balancing-in_order} - -``` sql -load_balancing = in_order -``` - -کپی با همان تعداد از اشتباهات در همان جهت قابل دسترسی هستند که در پیکربندی مشخص شده است. -این روش مناسب است که شما می دانید دقیقا همان است که ماکت ترجیح داده شده است. - -### اول یا تصادفی {#load_balancing-first_or_random} - -``` sql -load_balancing = first_or_random -``` - -این الگوریتم را انتخاب اولین ماکت در مجموعه و یا یک ماکت تصادفی اگر برای اولین بار در دسترس نیست. این در تنظیم توپولوژی متقابل تکرار موثر, اما بی فایده در تنظیمات دیگر. - -این `first_or_random` الگوریتم حل مشکل از `in_order` الگوریتم. با `in_order`, اگر یک ماکت پایین می رود, یک بعدی می شود یک بار دو برابر در حالی که کپی باقی مانده رسیدگی به مقدار معمول از ترافیک. هنگام استفاده از `first_or_random` الگوریتم, بار به طور مساوی در میان کپی که هنوز هم در دسترس هستند توزیع. - -## پیشفرض {#settings-prefer-localhost-replica} - -را قادر می سازد / غیر فعال ترجیح با استفاده از ماکت مجنون زمانی که پردازش نمایش داده شد توزیع شده است. - -مقادیر ممکن: - -- 1 — ClickHouse always sends a query to the localhost replica if it exists. -- 0 — ClickHouse uses the balancing strategy specified by the [_تبالسازی](#settings-load_balancing) تنظیمات. - -مقدار پیش فرض: 1. - -!!! warning "اخطار" - غیر فعال کردن این تنظیم در صورت استفاده [بیشینه_راپرال_راپیکال](#settings-max_parallel_replicas). - -## کد _ورود {#totals-mode} - -چگونه برای محاسبه مجموع زمانی که نیاز است در حال حاضر به عنوان زمانی که max_rows_to_group_by و group_by_overflow_mode = ‘any’ حضور دارند. -بخش را ببینید “WITH TOTALS modifier”. - -## در حال بارگذاری {#totals-auto-threshold} - -نگهبان `totals_mode = 'auto'`. -بخش را ببینید “WITH TOTALS modifier”. - -## بیشینه_راپرال_راپیکال {#settings-max_parallel_replicas} - -حداکثر تعداد کپی برای هر سفال در هنگام اجرای یک پرس و جو. -برای سازگاری (برای دریافت بخش های مختلف از تقسیم داده های مشابه), این گزینه تنها کار می کند زمانی که کلید نمونه گیری قرار است. -تاخیر المثنی کنترل نمی شود. - -## کامپایل {#compile} - -فعال کردن مجموعه ای از نمایش داده شد. به طور پیش فرض, 0 (غیر فعال). - -تدوین تنها برای بخشی از خط لوله پرس و جو پردازش استفاده می شود: برای مرحله اول تجمع (گروه های). -اگر این بخش از خط لوله وارد شده بود, پرس و جو ممکن است سریع تر با توجه به استقرار چرخه های کوتاه و محدود تماس تابع جمع اجرا. حداکثر بهبود عملکرد (تا چهار برابر سریعتر در موارد نادر) برای نمایش داده شد با توابع چند دانه ساده دیده می شود. معمولا افزایش عملکرد ناچیز است. در موارد بسیار نادر, ممکن است کم کردن سرعت اجرای پرس و جو. - -## _کوچکنمایی {#min-count-to-compile} - -چند بار به طور بالقوه استفاده از یک تکه وارد شده از کد قبل از در حال اجرا تلفیقی. به طور پیش فرض, 3. -For testing, the value can be set to 0: compilation runs synchronously and the query waits for the end of the compilation process before continuing execution. For all other cases, use values ​​starting with 1. Compilation normally takes about 5-10 seconds. -اگر مقدار است 1 یا بیشتر, تلفیقی ناهمگام در یک موضوع جداگانه رخ می دهد. نتیجه به محض این که حاضر است از جمله نمایش داده شد که در حال حاضر در حال اجرا استفاده می شود. - -کد کامپایل شده برای هر ترکیب های مختلف از توابع کل مورد استفاده در پرس و جو و نوع کلید در گروه بند مورد نیاز است. -The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they don't use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted. - -## خروجی _فرمان_جسون_کوات_64بیت_تنظیمی {#session_settings-output_format_json_quote_64bit_integers} - -اگر مقدار درست است صحیح به نظر می رسد در نقل قول ها در هنگام استفاده از JSON\* Int64 و UInt64 فرمت (برای سازگاری بیشتر با جاوا اسکریپت پیاده سازی); در غیر این صورت اعداد صحیح هستند و خروجی بدون نقل قول. - -## _مخفی کردن _قابلیت _جدید {#settings-format_csv_delimiter} - -شخصیت به عنوان یک جداساز در داده های سی سی. وی تفسیر شده است. به طور پیش فرض, جداساز است `,`. - -## _فرستادن به _کوچکنمایی {#settings-input_format_csv_unquoted_null_literal_as_null} - -برای فرمت ورودی سی اس وی را قادر می سازد و یا غیر فعال تجزیه بدون نقل `NULL` به عنوان تحت اللفظی (مترادف برای `\N`). - -## _انتقال به _شروع مجدد {#settings-output-format-csv-crlf-end-of-line} - -استفاده از داس/ویندوز-سبک خط جدا کننده (CRLF) در CSV به جای یونیکس سبک (LF). - -## _فرستادن در_م_مایش از _برخط {#settings-output-format-tsv-crlf-end-of-line} - -استفاده از توضیحات / ویندوز به سبک خط جدا (سازمان تنظیم مقررات) در واحد پشتیبانی فنی فنی فنی مهندسی به جای سبک یونیکس. - -## _معامله {#settings-insert_quorum} - -را قادر می سازد حد نصاب می نویسد. - -- اگر `insert_quorum < 2`, حد نصاب می نویسد غیر فعال هستند. -- اگر `insert_quorum >= 2`, حد نصاب می نویسد فعال هستند. - -مقدار پیش فرض: 0. - -حد نصاب می نویسد - -`INSERT` موفق تنها زمانی که تاتر موفق به درستی ارسال داده ها به `insert_quorum` از کپی در طول `insert_quorum_timeout`. اگر به هر دلیلی تعدادی از کپی با موفق می نویسد می کند از دسترس نیست `insert_quorum` نوشتن در نظر گرفته شده است شکست خورده و خانه را حذف بلوک قرار داده شده از تمام کپی که داده ها در حال حاضر نوشته شده است. - -همه تکرار در حد نصاب سازگار هستند, به عنوان مثال, حاوی اطلاعات از همه قبلی `INSERT` نمایش داده شد. این `INSERT` دنباله خطی است. - -هنگام خواندن داده های نوشته شده از `insert_quorum` شما می توانید از [مورد احترام](#settings-select_sequential_consistency) انتخاب - -تاتر تولید یک استثنا - -- اگر تعداد کپی های موجود در زمان پرس و جو کمتر از `insert_quorum`. -- در تلاش برای نوشتن داده ها زمانی که بلوک قبلی هنوز در وارد نشده است `insert_quorum` از کپی. این وضعیت ممکن است رخ دهد اگر کاربر تلاش می کند برای انجام یک `INSERT` قبل از قبلی با `insert_quorum` کامل شده است. - -همچنین نگاه کنید به: - -- [_بههنگامسازی](#settings-insert_quorum_timeout) -- [مورد احترام](#settings-select_sequential_consistency) - -## _بههنگامسازی {#settings-insert_quorum_timeout} - -ارسال به فاصله حد نصاب در ثانیه. اگر ایست را تصویب کرده است و بدون نوشتن صورت گرفته است و در عین حال, تاتر یک استثنا تولید و مشتری باید پرس و جو تکرار برای نوشتن همان بلوک به همان و یا هر ماکت دیگر. - -مقدار پیش فرض: 60 ثانیه. - -همچنین نگاه کنید به: - -- [_معامله](#settings-insert_quorum) -- [مورد احترام](#settings-select_sequential_consistency) - -## مورد احترام {#settings-select_sequential_consistency} - -قوام متوالی را فعال یا غیرفعال می کند `SELECT` نمایش داده شد: - -مقادیر ممکن: - -- 0 — Disabled. -- 1 — Enabled. - -مقدار پیش فرض: 0. - -استفاده - -هنگامی که قوام پی در پی فعال است, تاتر اجازه می دهد تا مشتری برای اجرای `SELECT` پرس و جو فقط برای کسانی که کپی که حاوی داده ها از همه قبلی `INSERT` نمایش داده شد اجرا با `insert_quorum`. اگر مشتری اشاره به یک ماکت بخشی, تاتر یک استثنا تولید. پرس و جو را انتخاب کنید داده است که هنوز به حد نصاب کپی نوشته نشده است را شامل نمی شود. - -همچنین نگاه کنید به: - -- [_معامله](#settings-insert_quorum) -- [_بههنگامسازی](#settings-insert_quorum_timeout) - -## _تنظیم مجدد به حالت اولیه {#settings-insert-deduplicate} - -امکان حذف یا غیرفعال کردن مسدود کردن تقسیم بندی `INSERT` (برای تکرار\* جداول). - -مقادیر ممکن: - -- 0 — Disabled. -- 1 — Enabled. - -مقدار پیش فرض: 1. - -به طور پیش فرض بلوک ها به جداول تکرار شده توسط `INSERT` بیانیه تقسیم شده است (نگاه کنید به [تکرار داده ها](../../engines/table-engines/mergetree-family/replication.md)). - -## دریافت حسابهای کاربری دستگاه {#settings-deduplicate-blocks-in-dependent-materialized-views} - -را قادر می سازد و یا غیر فعال بررسی تقسیم بندی برای نمایش محقق که دریافت داده ها از تکرار\* جداول. - -مقادیر ممکن: - - 0 — Disabled. - 1 — Enabled. - -مقدار پیش فرض: 0. - -استفاده - -به طور پیش فرض, تقسیم بندی برای نمایش تحقق انجام نشده است اما بالادست انجام, در جدول منبع. -اگر یک بلوک قرار داده شده است به دلیل تقسیم بندی در جدول منبع قلم, وجود خواهد داشت بدون درج به نمایش مواد متصل. این رفتار وجود دارد برای فعال کردن درج داده ها بسیار جمع به نمایش محقق, برای مواردی که بلوک های قرار داده شده همان پس از تجمع مشاهده محقق اما مشتق شده از درج های مختلف را به جدول منبع. -همزمان, این رفتار “breaks” `INSERT` حق تقدم. اگر یک `INSERT` به جدول اصلی موفق بود و `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won't receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` اجازه می دهد تا برای تغییر این رفتار. در تلاش مجدد, یک دیدگاه محقق درج تکرار دریافت خواهد کرد و بررسی تکرار به خودی خود انجام, -نادیده گرفتن نتیجه چک برای جدول منبع, و ردیف به دلیل شکست اول از دست داده وارد. - -## ویژ_گیها {#settings-max-network-bytes} - -محدودیت حجم داده ها (به بایت) است که دریافت و یا انتقال بر روی شبکه در هنگام اجرای یک پرس و جو. این تنظیم در مورد هر پرس و جو فردی. - -مقادیر ممکن: - -- عدد صحیح مثبت. -- 0 — Data volume control is disabled. - -مقدار پیش فرض: 0. - -## _عرض {#settings-max-network-bandwidth} - -محدودیت سرعت تبادل داده ها بر روی شبکه در بایت در هر ثانیه. این تنظیم در مورد هر پرس و جو. - -مقادیر ممکن: - -- عدد صحیح مثبت. -- 0 — Bandwidth control is disabled. - -مقدار پیش فرض: 0. - -## _شمارهگیر بیشینه {#settings-max-network-bandwidth-for-user} - -محدودیت سرعت تبادل داده ها بر روی شبکه در بایت در هر ثانیه. این تنظیم به تمام نمایش داده شد همزمان در حال اجرا انجام شده توسط یک کاربر اعمال می شود. - -مقادیر ممکن: - -- عدد صحیح مثبت. -- 0 — Control of the data speed is disabled. - -مقدار پیش فرض: 0. - -## _شمارهگیرها {#settings-max-network-bandwidth-for-all-users} - -محدودیت سرعت است که داده ها در بیش از شبکه در بایت در هر ثانیه رد و بدل. این تنظیم در مورد تمام نمایش داده شد به صورت همزمان در حال اجرا بر روی سرور. - -مقادیر ممکن: - -- عدد صحیح مثبت. -- 0 — Control of the data speed is disabled. - -مقدار پیش فرض: 0. - -## ا_فزونهها {#settings-count_distinct_implementation} - -مشخص می کند که کدام یک از `uniq*` توابع باید برای انجام [COUNT(DISTINCT …)](../../sql-reference/aggregate-functions/reference.md#agg_function-count) ساخت و ساز. - -مقادیر ممکن: - -- [دانشگاه](../../sql-reference/aggregate-functions/reference.md#agg_function-uniq) -- [مخلوط نشده](../../sql-reference/aggregate-functions/reference.md#agg_function-uniqcombined) -- [نیم قرن 64](../../sql-reference/aggregate-functions/reference.md#agg_function-uniqcombined64) -- [یونقلل12](../../sql-reference/aggregate-functions/reference.md#agg_function-uniqhll12) -- [قرارداد اتحادیه](../../sql-reference/aggregate-functions/reference.md#agg_function-uniqexact) - -مقدار پیشفرض: `uniqExact`. - -## در حال بارگذاری {#settings-skip_unavailable_shards} - -را قادر می سازد و یا غیر فعال سکوت پرش از خرده ریز در دسترس نیست. - -سفال در دسترس نیست در نظر گرفته اگر همه کپی خود را در دسترس نیست. ماکت در موارد زیر در دسترس نیست: - -- تاتر نمی تواند به هر دلیلی به ماکت متصل شود. - - هنگام اتصال به یک ماکت, تاتر انجام چندین تلاش. اگر تمام این تلاش شکست, ماکت در دسترس نیست در نظر گرفته شده است. - -- بدل نمی تواند از طریق دی ان اس حل شود. - - اگر نام میزبان ماکت را نمی توان از طریق دی ان اس حل و فصل, می تواند شرایط زیر نشان می دهد: - - - میزبان ماکت هیچ سابقه دی ان اس. این می تواند در سیستم های با دی ان اس پویا رخ می دهد, مثلا, [کوبرنتس](https://kubernetes.io), جایی که گره می تواند در طول خرابی قابل حل, و این یک خطا نیست. - - - خطای پیکربندی. فایل پیکربندی کلیک شامل یک نام میزبان اشتباه است. - -مقادیر ممکن: - -- 1 — skipping enabled. - - اگر یک سفال در دسترس نیست, خانه را برمی گرداند در نتیجه بر اساس داده های بخشی می کند و مشکلات در دسترس بودن گره گزارش نمی. - -- 0 — skipping disabled. - - اگر یک سفال در دسترس نیست, تاتر می اندازد یک استثنا. - -مقدار پیش فرض: 0. - -## افراد زیر در این افزونه مشارکت کردهاند {#settings-optimize_skip_unused_shards} - -فعال یا غیر فعال پرش استفاده نشده خرده ریز برای انتخاب نمایش داده شد که sharding شرط کلیدی در PREWHERE/که در آن (به فرض که داده ها توزیع شده است sharding کلیدی در غیر این صورت هیچ چیز). - -مقدار پیشفرض: 0 - -## افراد زیر در این افزونه مشارکت کردهاند {#settings-force_optimize_skip_unused_shards} - -فعال یا غیرفعال اجرای پرس و جو اگر [`optimize_skip_unused_shards`](#settings-optimize_skip_unused_shards) فعال و پرش از خرده ریز استفاده نشده امکان پذیر نیست. اگر پرش امکان پذیر نیست و تنظیمات را فعال کنید استثنا پرتاب خواهد شد. - -مقادیر ممکن: - -- 0-معلول (نمی اندازد) -- 1-غیر فعال کردن اجرای پرس و جو تنها در صورتی که جدول دارای کلید شارژ -- 2-غیر فعال کردن اجرای پرس و جو بدون در نظر گرفتن کلید شاردینگ برای جدول تعریف شده است - -مقدار پیشفرض: 0 - -## ا_فزون_ف_کوپ {#setting-optimize_throw_if_noop} - -را قادر می سازد و یا غیر فعال پرتاب یک استثنا اگر یک [OPTIMIZE](../../sql-reference/statements/misc.md#misc_operations-optimize) پرس و جو یک ادغام انجام نمی. - -به طور پیش فرض, `OPTIMIZE` حتی اگر هیچ کاری انجام نمی دهد با موفقیت باز می گردد. این تنظیم به شما اجازه می دهد تا این شرایط را متمایز کنید و دلیل را در یک پیام استثنا دریافت کنید. - -مقادیر ممکن: - -- 1 — Throwing an exception is enabled. -- 0 — Throwing an exception is disabled. - -مقدار پیش فرض: 0. - -## در حال بارگذاری {#settings-distributed_replica_error_half_life} - -- نوع: ثانیه -- مقدار پیشفرض: 60 ثانیه - -کنترل خطاهای چگونه سریع در جداول توزیع صفر. اگر یک ماکت برای برخی از زمان در دسترس نیست, تجمع می یابد 5 اشتباهات, و توزیع _راپیرار_لفا_لایف تنظیم شده است 1 دوم, سپس ماکت در نظر گرفته شده است طبیعی 3 ثانیه پس از خطا گذشته. - -همچنین نگاه کنید به: - -- [موتور جدول توزیع شده است](../../engines/table-engines/special/distributed.md) -- [نمایش سایت](#settings-distributed_replica_error_cap) - -## نمایش سایت {#settings-distributed_replica_error_cap} - -- نوع: امضا نشده -- مقدار پیش فرض: 1000 - -تعداد خطا از هر ماکت است که در این مقدار پوش, جلوگیری از یک ماکت از تجمع بیش از حد بسیاری از اشتباهات. - -همچنین نگاه کنید به: - -- [موتور جدول توزیع شده است](../../engines/table-engines/special/distributed.md) -- [در حال بارگذاری](#settings-distributed_replica_error_half_life) - -## در حال بارگذاری {#distributed_directory_monitor_sleep_time_ms} - -فاصله پایه برای [توزیع شده](../../engines/table-engines/special/distributed.md) موتور جدول برای ارسال داده ها. فاصله واقعی نمایی رشد می کند در صورت خطا. - -مقادیر ممکن: - -- عدد صحیح مثبت میلی ثانیه. - -مقدار پیش فرض: 100 میلی ثانیه. - -## در حال بارگذاری {#distributed_directory_monitor_max_sleep_time_ms} - -حداکثر فاصله برای [توزیع شده](../../engines/table-engines/special/distributed.md) موتور جدول برای ارسال داده ها. محدودیت رشد نمایی از فاصله تعیین شده در [در حال بارگذاری](#distributed_directory_monitor_sleep_time_ms) تنظیمات. - -مقادیر ممکن: - -- عدد صحیح مثبت میلی ثانیه. - -مقدار پیش فرض: 30000 میلی ثانیه (30 ثانیه). - -## نمایش سایت {#distributed_directory_monitor_batch_inserts} - -را قادر می سازد/غیر فعال ارسال داده های درج شده در دسته. - -هنگام ارسال دسته ای فعال است [توزیع شده](../../engines/table-engines/special/distributed.md) موتور جدول تلاش می کند چندین فایل داده های درج شده را در یک عملیات به جای ارسال جداگانه ارسال کند. دسته ای ارسال را بهبود می بخشد عملکرد خوشه با استفاده بهتر از سرور و شبکه منابع. - -مقادیر ممکن: - -- 1 — Enabled. -- 0 — Disabled. - -مقدار پیش فرض: 0. - -## _شخصیت {#setting-os-thread-priority} - -اولویت را تنظیم می کند ([خوبه](https://en.wikipedia.org/wiki/Nice_(Unix))) برای موضوعات که نمایش داده شد را اجرا کند . زمانبندی سیستم عامل این اولویت در نظر هنگام انتخاب موضوع بعدی به اجرا در هر هسته پردازنده در دسترس است. - -!!! warning "اخطار" - برای استفاده از این تنظیم, شما نیاز به تنظیم `CAP_SYS_NICE` توان. این `clickhouse-server` بسته بندی در هنگام نصب تنظیم می شود. برخی از محیط های مجازی اجازه نمی دهد که شما را به مجموعه `CAP_SYS_NICE` توان. در این مورد, `clickhouse-server` در ابتدا پیامی در موردش نشان می دهد. - -مقادیر ممکن: - -- شما می توانید مقادیر را در محدوده تنظیم کنید `[-20, 19]`. - -مقادیر پایین تر به معنای اولویت بالاتر است. رشتهها با کم `nice` مقادیر اولویت اغلب از موضوعات با ارزش های بالا اجرا می شود. مقادیر بالا برای پرس و جو های غیر تعاملی طولانی تر ترجیح داده می شود زیرا اجازه می دهد تا به سرعت منابع را به نفع نمایش های تعاملی کوتاه در هنگام ورود به دست بدهند. - -مقدار پیش فرض: 0. - -## جستجو {#query_profiler_real_time_period_ns} - -دوره را برای یک تایمر ساعت واقعی تنظیم می کند [پروفیل پرس و جو](../../operations/optimizing-performance/sampling-query-profiler.md). تایمر ساعت واقعی شمارش زمان دیوار ساعت. - -مقادیر ممکن: - -- عدد صحیح مثبت در nanoseconds. - - مقادیر توصیه شده: - - - 10000000 (100 times a second) nanoseconds and less for single queries. - - 1000000000 (once a second) for cluster-wide profiling. - -- 0 برای خاموش کردن تایمر. - -نوع: [UInt64](../../sql-reference/data-types/int-uint.md). - -مقدار پیش فرض: 1000000000 نانو ثانیه (یک بار در ثانیه). - -همچنین نگاه کنید به: - -- جدول سیستم [_قطع](../../operations/system-tables.md#system_tables-trace_log) - -## ایران در تهران {#query_profiler_cpu_time_period_ns} - -دوره را برای تایمر ساعت پردازنده تنظیم می کند [پروفیل پرس و جو](../../operations/optimizing-performance/sampling-query-profiler.md). این تایمر شمارش تنها زمان پردازنده. - -مقادیر ممکن: - -- عدد صحیح مثبت نانو ثانیه. - - مقادیر توصیه شده: - - - 10000000 (100 times a second) nanoseconds and more for single queries. - - 1000000000 (once a second) for cluster-wide profiling. - -- 0 برای خاموش کردن تایمر. - -نوع: [UInt64](../../sql-reference/data-types/int-uint.md). - -مقدار پیش فرض: 1000000000 نانو ثانیه. - -همچنین نگاه کنید به: - -- جدول سیستم [_قطع](../../operations/system-tables.md#system_tables-trace_log) - -## اجازه دادن به _فعال کردن اختلال در عملکرد {#settings-allow_introspection_functions} - -فعالسازی از کارانداختن [توابع درون گونه](../../sql-reference/functions/introspection.md) برای پروفایل پرس و جو. - -مقادیر ممکن: - -- 1 — Introspection functions enabled. -- 0 — Introspection functions disabled. - -مقدار پیش فرض: 0. - -**همچنین نگاه کنید به** - -- [پروفایل پرس و جو نمونه برداری](../optimizing-performance/sampling-query-profiler.md) -- جدول سیستم [_قطع](../../operations/system-tables.md#system_tables-trace_log) - -## وارد_فرمت_پارلل_درپارس {#input-format-parallel-parsing} - -- نوع: بولی -- مقدار پیشفرض: درست - -فعال کردن نظم حفظ تجزیه موازی از فرمت های داده. پشتیبانی تنها برای TSV TKSV CSV و JSONEachRow فرمت های. - -## _حداقل کردن _بیتس_برای_پرال_درپارس {#min-chunk-bytes-for-parallel-parsing} - -- نوع: امضا نشده -- مقدار پیشفرض: 1 مگابایت - -حداقل اندازه تکه در بایت, که هر موضوع به صورت موازی تجزیه خواهد شد. - -## _فرماندگی لبه بام {#settings-output_format_avro_codec} - -مجموعه کدک فشرده سازی مورد استفاده برای خروجی فایل اورو. - -نوع: رشته - -مقادیر ممکن: - -- `null` — No compression -- `deflate` — Compress with Deflate (zlib) -- `snappy` — Compress with [روح](https://google.github.io/snappy/) - -مقدار پیشفرض: `snappy` (در صورت موجود بودن) یا `deflate`. - -## _فرماندگی لبه چشم {#settings-output_format_avro_sync_interval} - -مجموعه حداقل اندازه داده (در بایت) بین نشانگر هماهنگ سازی برای فایل خروجی هواپیما. - -نوع: امضا نشده - -مقادیر ممکن: 32 (32 بایت) - 1073741824 (1 دستگاه گوارش) - -مقدار پیش فرض: 32768 (32 کیلوبایت) - -## باز کردن _نمایش مجدد {#settings-format_avro_schema_registry_url} - -نشانی اینترنتی رجیستری طرحواره را برای استفاده تنظیم میکند [هشدار داده می شود](../../interfaces/formats.md#data-format-avro-confluent) قالب - -نوع: نشانی وب - -مقدار پیشفرض: خالی - -## پس زمینه {#background_pool_size} - -مجموعه تعدادی از موضوعات انجام عملیات پس زمینه در موتورهای جدول (مثلا, ادغام در [موتور ادغام](../../engines/table-engines/mergetree-family/index.md) جدول). این تنظیم در شروع سرور کلیک استفاده می شود و نمی تواند در یک جلسه کاربر تغییر کند. با تنظیم این تنظیم شما پردازنده و دیسک بار مدیریت. اندازه استخر کوچکتر با بهره گیری از پردازنده و دیسک منابع کمتر, اما فرایندهای پس زمینه پیشرفت کندتر که در نهایت ممکن است تاثیر عملکرد پرس و جو. - -مقادیر ممکن: - -- هر عدد صحیح مثبت. - -مقدار پیش فرض: 16. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/fa/operations/system-tables.md b/docs/fa/operations/system-tables.md deleted file mode 100644 index d634e6dc541..00000000000 --- a/docs/fa/operations/system-tables.md +++ /dev/null @@ -1,1168 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 52 -toc_title: "\u062C\u062F\u0627\u0648\u0644 \u0633\u06CC\u0633\u062A\u0645" ---- - -# جداول سیستم {#system-tables} - -جداول سیستم برای اجرای بخشی از قابلیت های سیستم استفاده می شود و برای دسترسی به اطلاعات در مورد چگونگی کار سیستم. -شما می توانید یک جدول سیستم را حذف کنید (اما شما می توانید جدا انجام). -جداول سیستم فایل های با داده ها بر روی دیسک و یا فایل های با ابرداده ندارد. سرور ایجاد تمام جداول سیستم زمانی که شروع می شود. -جداول سیستم فقط خواندنی. -این در واقع ‘system’ بانک اطلاعات. - -## سیستم._نامهنویسی ناهمزمان {#system_tables-asynchronous_metrics} - -شامل معیارهای که به صورت دوره ای در پس زمینه محاسبه می شود. مثلا, مقدار رم در حال استفاده. - -ستونها: - -- `metric` ([رشته](../sql-reference/data-types/string.md)) — Metric name. -- `value` ([جسم شناور64](../sql-reference/data-types/float.md)) — Metric value. - -**مثال** - -``` sql -SELECT * FROM system.asynchronous_metrics LIMIT 10 -``` - -``` text -┌─metric──────────────────────────────────┬──────value─┐ -│ jemalloc.background_thread.run_interval │ 0 │ -│ jemalloc.background_thread.num_runs │ 0 │ -│ jemalloc.background_thread.num_threads │ 0 │ -│ jemalloc.retained │ 422551552 │ -│ jemalloc.mapped │ 1682989056 │ -│ jemalloc.resident │ 1656446976 │ -│ jemalloc.metadata_thp │ 0 │ -│ jemalloc.metadata │ 10226856 │ -│ UncompressedCacheCells │ 0 │ -│ MarkCacheFiles │ 0 │ -└─────────────────────────────────────────┴────────────┘ -``` - -**همچنین نگاه کنید به** - -- [نظارت](monitoring.md) — Base concepts of ClickHouse monitoring. -- [سیستم.متریک](#system_tables-metrics) — Contains instantly calculated metrics. -- [سیستم.رویدادها](#system_tables-events) — Contains a number of events that have occurred. -- [سیستم._اشکالزدایی](#system_tables-metric_log) — Contains a history of metrics values from tables `system.metrics` и `system.events`. - -## سیستم.خوشه {#system-clusters} - -حاوی اطلاعاتی در مورد خوشه های موجود در فایل پیکربندی و سرورهای موجود در ان. - -ستونها: - -- `cluster` (String) — The cluster name. -- `shard_num` (UInt32) — The shard number in the cluster, starting from 1. -- `shard_weight` (UInt32) — The relative weight of the shard when writing data. -- `replica_num` (UInt32) — The replica number in the shard, starting from 1. -- `host_name` (String) — The host name, as specified in the config. -- `host_address` (String) — The host IP address obtained from DNS. -- `port` (UInt16) — The port to use for connecting to the server. -- `user` (String) — The name of the user for connecting to the server. -- `errors_count` (اوینت32) - تعداد دفعاتی که این میزبان موفق به رسیدن به ماکت. -- `estimated_recovery_time` (اوینت32) - ثانیه به سمت چپ تا زمانی که تعداد خطا ماکت صفر است و در نظر گرفته می شود به حالت عادی. - -لطفا توجه داشته باشید که `errors_count` یک بار در هر پرس و جو به خوشه به روز, ولی `estimated_recovery_time` بر روی تقاضا محاسبه شده است. بنابراین می تواند یک مورد غیر صفر باشد `errors_count` و صفر `estimated_recovery_time`, که پرس و جو بعدی صفر خواهد شد `errors_count` و سعی کنید به استفاده از ماکت به عنوان اگر هیچ خطا. - -**همچنین نگاه کنید به** - -- [موتور جدول توزیع شده است](../engines/table-engines/special/distributed.md) -- [تنظیمات _فرهنگ توزیع میشود](settings/settings.md#settings-distributed_replica_error_cap) -- [پخش _راپیشا_را_را_را_حالف_لایف تنظیم](settings/settings.md#settings-distributed_replica_error_half_life) - -## سیستم.ستونها {#system-columns} - -حاوی اطلاعات در مورد ستون در تمام جداول. - -شما می توانید از این جدول برای دریافت اطلاعات شبیه به [DESCRIBE TABLE](../sql-reference/statements/misc.md#misc-describe-table) پرس و جو, اما برای جداول متعدد در یک بار. - -این `system.columns` جدول شامل ستون های زیر (نوع ستون در براکت نشان داده شده است): - -- `database` (String) — Database name. -- `table` (String) — Table name. -- `name` (String) — Column name. -- `type` (String) — Column type. -- `default_kind` (String) — Expression type (`DEFAULT`, `MATERIALIZED`, `ALIAS`) برای مقدار پیش فرض, و یا یک رشته خالی اگر تعریف نشده است. -- `default_expression` (String) — Expression for the default value, or an empty string if it is not defined. -- `data_compressed_bytes` (UInt64) — The size of compressed data, in bytes. -- `data_uncompressed_bytes` (UInt64) — The size of decompressed data, in bytes. -- `marks_bytes` (UInt64) — The size of marks, in bytes. -- `comment` (String) — Comment on the column, or an empty string if it is not defined. -- `is_in_partition_key` (UInt8) — Flag that indicates whether the column is in the partition expression. -- `is_in_sorting_key` (UInt8) — Flag that indicates whether the column is in the sorting key expression. -- `is_in_primary_key` (UInt8) — Flag that indicates whether the column is in the primary key expression. -- `is_in_sampling_key` (UInt8) — Flag that indicates whether the column is in the sampling key expression. - -## سیستم.یاریدهندکان {#system-contributors} - -حاوی اطلاعات در مورد همکاران. همه مربیان به صورت تصادفی. سفارش تصادفی در زمان اجرای پرس و جو است. - -ستونها: - -- `name` (String) — Contributor (author) name from git log. - -**مثال** - -``` sql -SELECT * FROM system.contributors LIMIT 10 -``` - -``` text -┌─name─────────────┐ -│ Olga Khvostikova │ -│ Max Vetrov │ -│ LiuYangkuan │ -│ svladykin │ -│ zamulla │ -│ Šimon Podlipský │ -│ BayoNet │ -│ Ilya Khomutov │ -│ Amy Krishnevsky │ -│ Loud_Scream │ -└──────────────────┘ -``` - -برای پیدا کردن خود را در جدول, استفاده از یک پرس و جو: - -``` sql -SELECT * FROM system.contributors WHERE name='Olga Khvostikova' -``` - -``` text -┌─name─────────────┐ -│ Olga Khvostikova │ -└──────────────────┘ -``` - -## سیستم.پایگاههای داده {#system-databases} - -این جدول شامل یک ستون رشته ای به نام ‘name’ – the name of a database. -هر پایگاه داده که سرور می داند در مورد یک ورودی مربوطه را در جدول. -این جدول سیستم برای اجرای استفاده می شود `SHOW DATABASES` پرس و جو. - -## سیستم.قطعات مجزا {#system_tables-detached_parts} - -حاوی اطلاعات در مورد قطعات جدا شده از [ادغام](../engines/table-engines/mergetree-family/mergetree.md) میز این `reason` ستون مشخص می کند که چرا بخش جدا شد. برای قطعات کاربر جدا, دلیل خالی است. چنین قطعات را می توان با [ALTER TABLE ATTACH PARTITION\|PART](../sql-reference/statements/alter.md#alter_attach-partition) فرمان. برای توضیحات ستون های دیگر را ببینید [سیستم.قطعات](#system_tables-parts). اگر نام قسمت نامعتبر است, ارزش برخی از ستون ممکن است `NULL`. این قطعات را می توان با حذف [ALTER TABLE DROP DETACHED PART](../sql-reference/statements/alter.md#alter_drop-detached). - -## سیستم.واژهنامهها {#system_tables-dictionaries} - -حاوی اطلاعات در مورد [واژهنامهها خارجی](../sql-reference/dictionaries/external-dictionaries/external-dicts.md). - -ستونها: - -- `database` ([رشته](../sql-reference/data-types/string.md)) — Name of the database containing the dictionary created by DDL query. Empty string for other dictionaries. -- `name` ([رشته](../sql-reference/data-types/string.md)) — [نام واژهنامه](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md). -- `status` ([شمار8](../sql-reference/data-types/enum.md)) — Dictionary status. Possible values: - - `NOT_LOADED` — Dictionary was not loaded because it was not used. - - `LOADED` — Dictionary loaded successfully. - - `FAILED` — Unable to load the dictionary as a result of an error. - - `LOADING` — Dictionary is loading now. - - `LOADED_AND_RELOADING` — Dictionary is loaded successfully, and is being reloaded right now (frequent reasons: [SYSTEM RELOAD DICTIONARY](../sql-reference/statements/system.md#query_language-system-reload-dictionary) پرس و جو, ایست, پیکربندی فرهنگ لغت تغییر کرده است). - - `FAILED_AND_RELOADING` — Could not load the dictionary as a result of an error and is loading now. -- `origin` ([رشته](../sql-reference/data-types/string.md)) — Path to the configuration file that describes the dictionary. -- `type` ([رشته](../sql-reference/data-types/string.md)) — Type of a dictionary allocation. [ذخیره واژهنامهها در حافظه](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md). -- `key` — [نوع کلید](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md#ext_dict_structure-key): کلید عددی ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) or Сomposite key ([رشته](../sql-reference/data-types/string.md)) — form “(type 1, type 2, …, type n)”. -- `attribute.names` ([& حذف](../sql-reference/data-types/array.md)([رشته](../sql-reference/data-types/string.md))) — Array of [نام خصیصه](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md#ext_dict_structure-attributes) فراهم شده توسط فرهنگ لغت. -- `attribute.types` ([& حذف](../sql-reference/data-types/array.md)([رشته](../sql-reference/data-types/string.md))) — Corresponding array of [انواع خصیصه](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md#ext_dict_structure-attributes) که توسط فرهنگ لغت فراهم شده است. -- `bytes_allocated` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Amount of RAM allocated for the dictionary. -- `query_count` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of queries since the dictionary was loaded or since the last successful reboot. -- `hit_rate` ([جسم شناور64](../sql-reference/data-types/float.md)) — For cache dictionaries, the percentage of uses for which the value was in the cache. -- `element_count` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of items stored in the dictionary. -- `load_factor` ([جسم شناور64](../sql-reference/data-types/float.md)) — Percentage filled in the dictionary (for a hashed dictionary, the percentage filled in the hash table). -- `source` ([رشته](../sql-reference/data-types/string.md)) — Text describing the [منبع داده](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) برای فرهنگ لغت. -- `lifetime_min` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Minimum [طول عمر](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md) از فرهنگ لغت در حافظه, پس از کلیک که تلاش می کند به بازنگری فرهنگ لغت (اگر `invalidate_query` قرار است, سپس تنها در صورتی که تغییر کرده است). تنظیم در ثانیه. -- `lifetime_max` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Maximum [طول عمر](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md) از فرهنگ لغت در حافظه, پس از کلیک که تلاش می کند به بازنگری فرهنگ لغت (اگر `invalidate_query` قرار است, سپس تنها در صورتی که تغییر کرده است). تنظیم در ثانیه. -- `loading_start_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Start time for loading the dictionary. -- `last_successful_update_time` ([DateTime](../sql-reference/data-types/datetime.md)) — End time for loading or updating the dictionary. Helps to monitor some troubles with external sources and investigate causes. -- `loading_duration` ([Float32](../sql-reference/data-types/float.md)) — Duration of a dictionary loading. -- `last_exception` ([رشته](../sql-reference/data-types/string.md)) — Text of the error that occurs when creating or reloading the dictionary if the dictionary couldn't be created. - -**مثال** - -پیکربندی فرهنگ لغت. - -``` sql -CREATE DICTIONARY dictdb.dict -( - `key` Int64 DEFAULT -1, - `value_default` String DEFAULT 'world', - `value_expression` String DEFAULT 'xxx' EXPRESSION 'toString(127 * 172)' -) -PRIMARY KEY key -SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dicttbl' DB 'dictdb')) -LIFETIME(MIN 0 MAX 1) -LAYOUT(FLAT()) -``` - -اطمینان حاصل کنید که فرهنگ لغت لود شده است. - -``` sql -SELECT * FROM system.dictionaries -``` - -``` text -┌─database─┬─name─┬─status─┬─origin──────┬─type─┬─key────┬─attribute.names──────────────────────┬─attribute.types─────┬─bytes_allocated─┬─query_count─┬─hit_rate─┬─element_count─┬───────────load_factor─┬─source─────────────────────┬─lifetime_min─┬─lifetime_max─┬──loading_start_time─┌──last_successful_update_time─┬──────loading_duration─┬─last_exception─┐ -│ dictdb │ dict │ LOADED │ dictdb.dict │ Flat │ UInt64 │ ['value_default','value_expression'] │ ['String','String'] │ 74032 │ 0 │ 1 │ 1 │ 0.0004887585532746823 │ ClickHouse: dictdb.dicttbl │ 0 │ 1 │ 2020-03-04 04:17:34 │ 2020-03-04 04:30:34 │ 0.002 │ │ -└──────────┴──────┴────────┴─────────────┴──────┴────────┴──────────────────────────────────────┴─────────────────────┴─────────────────┴─────────────┴──────────┴───────────────┴───────────────────────┴────────────────────────────┴──────────────┴──────────────┴─────────────────────┴──────────────────────────────┘───────────────────────┴────────────────┘ -``` - -## سیستم.رویدادها {#system_tables-events} - -حاوی اطلاعات در مورد تعدادی از حوادث که در سیستم رخ داده است. مثلا, در جدول, شما می توانید پیدا کنید که چگونه بسیاری از `SELECT` نمایش داده شد از سرور کلیک شروع پردازش شد. - -ستونها: - -- `event` ([رشته](../sql-reference/data-types/string.md)) — Event name. -- `value` ([UInt64](../sql-reference/data-types/int-uint.md)) — Number of events occurred. -- `description` ([رشته](../sql-reference/data-types/string.md)) — Event description. - -**مثال** - -``` sql -SELECT * FROM system.events LIMIT 5 -``` - -``` text -┌─event─────────────────────────────────┬─value─┬─description────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ -│ Query │ 12 │ Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries. │ -│ SelectQuery │ 8 │ Same as Query, but only for SELECT queries. │ -│ FileOpen │ 73 │ Number of files opened. │ -│ ReadBufferFromFileDescriptorRead │ 155 │ Number of reads (read/pread) from a file descriptor. Does not include sockets. │ -│ ReadBufferFromFileDescriptorReadBytes │ 9931 │ Number of bytes read from file descriptors. If the file is compressed, this will show the compressed data size. │ -└───────────────────────────────────────┴───────┴────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ -``` - -**همچنین نگاه کنید به** - -- [سیستم._نامهنویسی ناهمزمان](#system_tables-asynchronous_metrics) — Contains periodically calculated metrics. -- [سیستم.متریک](#system_tables-metrics) — Contains instantly calculated metrics. -- [سیستم._اشکالزدایی](#system_tables-metric_log) — Contains a history of metrics values from tables `system.metrics` и `system.events`. -- [نظارت](monitoring.md) — Base concepts of ClickHouse monitoring. - -## سیستم.توابع {#system-functions} - -حاوی اطلاعات در مورد توابع عادی و جمع. - -ستونها: - -- `name`(`String`) – The name of the function. -- `is_aggregate`(`UInt8`) — Whether the function is aggregate. - -## سیستم.بازداشت گرافیت {#system-graphite-retentions} - -حاوی اطلاعات در مورد پارامترها [لغزش _ نمودار](server-configuration-parameters/settings.md#server_configuration_parameters-graphite) که در جداول با استفاده [اطلاعات دقیق](../engines/table-engines/mergetree-family/graphitemergetree.md) موتورها. - -ستونها: - -- `config_name` ) رشته) - `graphite_rollup` نام پارامتر. -- `regexp` (رشته) - یک الگوی برای نام متریک. -- `function` (رشته) - نام تابع جمع. -- `age` (UInt64) - حداقل سن دیتا در ثانیه. -- `precision` (اوینت64) - چگونه دقیقا به تعریف سن داده ها در ثانیه. -- `priority` (UInt16) - الگوی اولویت است. -- `is_default` (UInt8) - آیا الگوی پیش فرض است. -- `Tables.database` (مجموعه (رشته)) - مجموعه ای از نام جداول پایگاه داده که از `config_name` پارامتر. -- `Tables.table` (صف (رشته)) - مجموعه ای از نام جدول که با استفاده از `config_name` پارامتر. - -## سیستم.ادغام {#system-merges} - -حاوی اطلاعات در مورد ادغام و جهش بخشی در حال حاضر در روند برای جداول در خانواده ادغام. - -ستونها: - -- `database` (String) — The name of the database the table is in. -- `table` (String) — Table name. -- `elapsed` (Float64) — The time elapsed (in seconds) since the merge started. -- `progress` (Float64) — The percentage of completed work from 0 to 1. -- `num_parts` (UInt64) — The number of pieces to be merged. -- `result_part_name` (String) — The name of the part that will be formed as the result of merging. -- `is_mutation` (اوینت8) - 1 اگر این فرایند جهش بخشی است. -- `total_size_bytes_compressed` (UInt64) — The total size of the compressed data in the merged chunks. -- `total_size_marks` (UInt64) — The total number of marks in the merged parts. -- `bytes_read_uncompressed` (UInt64) — Number of bytes read, uncompressed. -- `rows_read` (UInt64) — Number of rows read. -- `bytes_written_uncompressed` (UInt64) — Number of bytes written, uncompressed. -- `rows_written` (UInt64) — Number of rows written. - -## سیستم.متریک {#system_tables-metrics} - -شامل معیارهای است که می تواند فورا محاسبه, و یا یک مقدار فعلی. مثلا, تعداد نمایش داده شد به طور همزمان پردازش و یا تاخیر ماکت فعلی. این جدول همیشه به روز. - -ستونها: - -- `metric` ([رشته](../sql-reference/data-types/string.md)) — Metric name. -- `value` ([Int64](../sql-reference/data-types/int-uint.md)) — Metric value. -- `description` ([رشته](../sql-reference/data-types/string.md)) — Metric description. - -لیستی از معیارهای پشتیبانی شده شما می توانید در [همایش های بین المللیپردازنده](https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/CurrentMetrics.cpp) فایل منبع از خانه کلیک. - -**مثال** - -``` sql -SELECT * FROM system.metrics LIMIT 10 -``` - -``` text -┌─metric─────────────────────┬─value─┬─description──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ -│ Query │ 1 │ Number of executing queries │ -│ Merge │ 0 │ Number of executing background merges │ -│ PartMutation │ 0 │ Number of mutations (ALTER DELETE/UPDATE) │ -│ ReplicatedFetch │ 0 │ Number of data parts being fetched from replicas │ -│ ReplicatedSend │ 0 │ Number of data parts being sent to replicas │ -│ ReplicatedChecks │ 0 │ Number of data parts checking for consistency │ -│ BackgroundPoolTask │ 0 │ Number of active tasks in BackgroundProcessingPool (merges, mutations, fetches, or replication queue bookkeeping) │ -│ BackgroundSchedulePoolTask │ 0 │ Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc. │ -│ DiskSpaceReservedForMerge │ 0 │ Disk space reserved for currently running background merges. It is slightly more than the total size of currently merging parts. │ -│ DistributedSend │ 0 │ Number of connections to remote servers sending data that was INSERTed into Distributed tables. Both synchronous and asynchronous mode. │ -└────────────────────────────┴───────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ -``` - -**همچنین نگاه کنید به** - -- [سیستم._نامهنویسی ناهمزمان](#system_tables-asynchronous_metrics) — Contains periodically calculated metrics. -- [سیستم.رویدادها](#system_tables-events) — Contains a number of events that occurred. -- [سیستم._اشکالزدایی](#system_tables-metric_log) — Contains a history of metrics values from tables `system.metrics` и `system.events`. -- [نظارت](monitoring.md) — Base concepts of ClickHouse monitoring. - -## سیستم._اشکالزدایی {#system_tables-metric_log} - -دارای تاریخچه معیارهای ارزش از جداول `system.metrics` و `system.events`, دوره ای به دیسک سرخ. -برای روشن کردن مجموعه تاریخچه معیارهای در `system.metric_log` ایجاد `/etc/clickhouse-server/config.d/metric_log.xml` با محتوای زیر: - -``` xml - - - system - metric_log
- 7500 - 1000 - - -``` - -**مثال** - -``` sql -SELECT * FROM system.metric_log LIMIT 1 FORMAT Vertical; -``` - -``` text -Row 1: -────── -event_date: 2020-02-18 -event_time: 2020-02-18 07:15:33 -milliseconds: 554 -ProfileEvent_Query: 0 -ProfileEvent_SelectQuery: 0 -ProfileEvent_InsertQuery: 0 -ProfileEvent_FileOpen: 0 -ProfileEvent_Seek: 0 -ProfileEvent_ReadBufferFromFileDescriptorRead: 1 -ProfileEvent_ReadBufferFromFileDescriptorReadFailed: 0 -ProfileEvent_ReadBufferFromFileDescriptorReadBytes: 0 -ProfileEvent_WriteBufferFromFileDescriptorWrite: 1 -ProfileEvent_WriteBufferFromFileDescriptorWriteFailed: 0 -ProfileEvent_WriteBufferFromFileDescriptorWriteBytes: 56 -... -CurrentMetric_Query: 0 -CurrentMetric_Merge: 0 -CurrentMetric_PartMutation: 0 -CurrentMetric_ReplicatedFetch: 0 -CurrentMetric_ReplicatedSend: 0 -CurrentMetric_ReplicatedChecks: 0 -... -``` - -**همچنین نگاه کنید به** - -- [سیستم._نامهنویسی ناهمزمان](#system_tables-asynchronous_metrics) — Contains periodically calculated metrics. -- [سیستم.رویدادها](#system_tables-events) — Contains a number of events that occurred. -- [سیستم.متریک](#system_tables-metrics) — Contains instantly calculated metrics. -- [نظارت](monitoring.md) — Base concepts of ClickHouse monitoring. - -## سیستم.اعداد {#system-numbers} - -این جدول شامل یک UInt64 ستون به نام ‘number’ که شامل تقریبا تمام اعداد طبیعی با شروع از صفر. -شما می توانید این جدول برای تست استفاده, و یا اگر شما نیاز به انجام یک جستجو نیروی بی رحم. -بار خوانده شده از این جدول موازی نیست. - -## سیستم._شماره حساب {#system-numbers-mt} - -همان ‘system.numbers’ اما بار خوانده شده موازی هستند. اعداد را می توان در هر سفارش بازگشت. -مورد استفاده برای تست. - -## سیستم.یک {#system-one} - -این جدول شامل یک ردیف با یک ‘dummy’ در زیر8 ستون حاوی مقدار 0. -این جدول استفاده می شود اگر پرس و جو را انتخاب کنید از بند مشخص نیست. -این شبیه میز دوگانه است که در سایر موارد یافت می شود. - -## سیستم.قطعات {#system_tables-parts} - -حاوی اطلاعات در مورد بخش هایی از [ادغام](../engines/table-engines/mergetree-family/mergetree.md) میز - -هر سطر توصیف یک بخش داده. - -ستونها: - -- `partition` (String) – The partition name. To learn what a partition is, see the description of the [ALTER](../sql-reference/statements/alter.md#query_language_queries_alter) پرس و جو. - - فرشها: - - - `YYYYMM` برای پارتیشن بندی خودکار در ماه. - - `any_string` هنگامی که پارتیشن بندی دستی. - -- `name` (`String`) – Name of the data part. - -- `active` (`UInt8`) – Flag that indicates whether the data part is active. If a data part is active, it's used in a table. Otherwise, it's deleted. Inactive data parts remain after merging. - -- `marks` (`UInt64`) – The number of marks. To get the approximate number of rows in a data part, multiply `marks` با دانه دانه دانه شاخص (معمولا 8192) (این اشاره برای دانه دانه تطبیقی کار نمی کند). - -- `rows` (`UInt64`) – The number of rows. - -- `bytes_on_disk` (`UInt64`) – Total size of all the data part files in bytes. - -- `data_compressed_bytes` (`UInt64`) – Total size of compressed data in the data part. All the auxiliary files (for example, files with marks) are not included. - -- `data_uncompressed_bytes` (`UInt64`) – Total size of uncompressed data in the data part. All the auxiliary files (for example, files with marks) are not included. - -- `marks_bytes` (`UInt64`) – The size of the file with marks. - -- `modification_time` (`DateTime`) – The time the directory with the data part was modified. This usually corresponds to the time of data part creation.\| - -- `remove_time` (`DateTime`) – The time when the data part became inactive. - -- `refcount` (`UInt32`) – The number of places where the data part is used. A value greater than 2 indicates that the data part is used in queries or merges. - -- `min_date` (`Date`) – The minimum value of the date key in the data part. - -- `max_date` (`Date`) – The maximum value of the date key in the data part. - -- `min_time` (`DateTime`) – The minimum value of the date and time key in the data part. - -- `max_time`(`DateTime`) – The maximum value of the date and time key in the data part. - -- `partition_id` (`String`) – ID of the partition. - -- `min_block_number` (`UInt64`) – The minimum number of data parts that make up the current part after merging. - -- `max_block_number` (`UInt64`) – The maximum number of data parts that make up the current part after merging. - -- `level` (`UInt32`) – Depth of the merge tree. Zero means that the current part was created by insert rather than by merging other parts. - -- `data_version` (`UInt64`) – Number that is used to determine which mutations should be applied to the data part (mutations with a version higher than `data_version`). - -- `primary_key_bytes_in_memory` (`UInt64`) – The amount of memory (in bytes) used by primary key values. - -- `primary_key_bytes_in_memory_allocated` (`UInt64`) – The amount of memory (in bytes) reserved for primary key values. - -- `is_frozen` (`UInt8`) – Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup doesn't exist. For more details, see [FREEZE PARTITION](../sql-reference/statements/alter.md#alter_freeze-partition) - -- `database` (`String`) – Name of the database. - -- `table` (`String`) – Name of the table. - -- `engine` (`String`) – Name of the table engine without parameters. - -- `path` (`String`) – Absolute path to the folder with data part files. - -- `disk` (`String`) – Name of a disk that stores the data part. - -- `hash_of_all_files` (`String`) – [سیفون128](../sql-reference/functions/hash-functions.md#hash_functions-siphash128) از فایل های فشرده. - -- `hash_of_uncompressed_files` (`String`) – [سیفون128](../sql-reference/functions/hash-functions.md#hash_functions-siphash128) از فایل های غیر فشرده (فایل های با علامت, فایل شاخص و غیره.). - -- `uncompressed_hash_of_compressed_files` (`String`) – [سیفون128](../sql-reference/functions/hash-functions.md#hash_functions-siphash128) از داده ها در فایل های فشرده به عنوان اگر غیر فشرده شد. - -- `bytes` (`UInt64`) – Alias for `bytes_on_disk`. - -- `marks_size` (`UInt64`) – Alias for `marks_bytes`. - -## سیستم._خروج {#system_tables-part-log} - -این `system.part_log` جدول تنها در صورتی ایجاد می شود [_خروج](server-configuration-parameters/settings.md#server_configuration_parameters-part-log) تنظیم سرور مشخص شده است. - -این جدول حاوی اطلاعات در مورد اتفاقاتی که با رخ داده است [قطعات داده](../engines/table-engines/mergetree-family/custom-partitioning-key.md) در [ادغام](../engines/table-engines/mergetree-family/mergetree.md) جداول خانواده, مانند اضافه کردن و یا ادغام داده ها. - -این `system.part_log` جدول شامل ستون های زیر است: - -- `event_type` (Enum) — Type of the event that occurred with the data part. Can have one of the following values: - - `NEW_PART` — Inserting of a new data part. - - `MERGE_PARTS` — Merging of data parts. - - `DOWNLOAD_PART` — Downloading a data part. - - `REMOVE_PART` — Removing or detaching a data part using [DETACH PARTITION](../sql-reference/statements/alter.md#alter_detach-partition). - - `MUTATE_PART` — Mutating of a data part. - - `MOVE_PART` — Moving the data part from the one disk to another one. -- `event_date` (Date) — Event date. -- `event_time` (DateTime) — Event time. -- `duration_ms` (UInt64) — Duration. -- `database` (String) — Name of the database the data part is in. -- `table` (String) — Name of the table the data part is in. -- `part_name` (String) — Name of the data part. -- `partition_id` (String) — ID of the partition that the data part was inserted to. The column takes the ‘all’ ارزش اگر پارتیشن بندی توسط `tuple()`. -- `rows` (UInt64) — The number of rows in the data part. -- `size_in_bytes` (UInt64) — Size of the data part in bytes. -- `merged_from` (Array(String)) — An array of names of the parts which the current part was made up from (after the merge). -- `bytes_uncompressed` (UInt64) — Size of uncompressed bytes. -- `read_rows` (UInt64) — The number of rows was read during the merge. -- `read_bytes` (UInt64) — The number of bytes was read during the merge. -- `error` (UInt16) — The code number of the occurred error. -- `exception` (String) — Text message of the occurred error. - -این `system.part_log` جدول پس از اولین قرار دادن داده ها به ایجاد `MergeTree` جدول - -## سیستم.فرایندها {#system_tables-processes} - -این جدول سیستم برای اجرای استفاده می شود `SHOW PROCESSLIST` پرس و جو. - -ستونها: - -- `user` (String) – The user who made the query. Keep in mind that for distributed processing, queries are sent to remote servers under the `default` کاربر. زمینه شامل نام کاربری برای یک پرس و جو خاص, نه برای پرس و جو که این پرس و جو شروع. -- `address` (String) – The IP address the request was made from. The same for distributed processing. To track where a distributed query was originally made from, look at `system.processes` در سرور درخواست پرس و جو. -- `elapsed` (Float64) – The time in seconds since request execution started. -- `rows_read` (UInt64) – The number of rows read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. -- `bytes_read` (UInt64) – The number of uncompressed bytes read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. -- `total_rows_approx` (UInt64) – The approximation of the total number of rows that should be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known. -- `memory_usage` (UInt64) – Amount of RAM the request uses. It might not include some types of dedicated memory. See the [_کاساژ بیشینه](../operations/settings/query-complexity.md#settings_max_memory_usage) تنظیمات. -- `query` (String) – The query text. For `INSERT` این شامل داده ها برای وارد کردن نیست. -- `query_id` (String) – Query ID, if defined. - -## سیستم._خروج {#system_tables-text_log} - -شامل ورودی ورود به سیستم. سطح ورود به سیستم که می رود به این جدول را می توان با محدود `text_log.level` تنظیم سرور. - -ستونها: - -- `event_date` (`Date`)- تاریخ ورود. -- `event_time` (`DateTime`)- زمان ورود . -- `microseconds` (`UInt32`)- میکروثانیه از ورود. -- `thread_name` (String) — Name of the thread from which the logging was done. -- `thread_id` (UInt64) — OS thread ID. -- `level` (`Enum8`)- ورود به سطح . - - `'Fatal' = 1` - - `'Critical' = 2` - - `'Error' = 3` - - `'Warning' = 4` - - `'Notice' = 5` - - `'Information' = 6` - - `'Debug' = 7` - - `'Trace' = 8` -- `query_id` (`String`)- شناسه پرس و جو . -- `logger_name` (`LowCardinality(String)`) - Name of the logger (i.e. `DDLWorker`) -- `message` (`String`)- پیام خود را. -- `revision` (`UInt32`)- تجدید نظر کلیک کنیدهاوس . -- `source_file` (`LowCardinality(String)`)- فایل منبع که از ورود به سیستم انجام شد . -- `source_line` (`UInt64`)- خط منبع که از ورود به سیستم انجام شد. - -## سیستم._خروج {#system_tables-query_log} - -حاوی اطلاعات در مورد اجرای نمایش داده شد. برای هر پرس و جو, شما می توانید زمان شروع پردازش را ببینید, مدت زمان پردازش, پیام های خطا و اطلاعات دیگر. - -!!! note "یادداشت" - جدول حاوی اطلاعات ورودی برای `INSERT` نمایش داده شد. - -تاتر این جدول را فقط در صورتی ایجاد می کند [_خروج](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) پارامتر سرور مشخص شده است. این پارامتر مجموعه قوانین ورود به سیستم, مانند فاصله ورود به سیستم و یا نام جدول نمایش داده شد خواهد شد وارد سایت شوید. - -برای فعال کردن ورود به سیستم پرس و جو, تنظیم [_خروج](settings/settings.md#settings-log-queries) پارامتر به 1. برای اطلاعات بیشتر [تنظیمات](settings/settings.md) بخش. - -این `system.query_log` جدول ثبت دو نوع نمایش داده شد: - -1. نمایش داده شد اولیه که به طور مستقیم توسط مشتری اجرا شد. -2. کودک نمایش داده شد که توسط دیگر نمایش داده شد (برای اجرای پرس و جو توزیع). برای این نوع از نمایش داده شد, اطلاعات در مورد پدر و مادر نمایش داده شد در نشان داده شده است `initial_*` ستون ها - -ستونها: - -- `type` (`Enum8`) — Type of event that occurred when executing the query. Values: - - `'QueryStart' = 1` — Successful start of query execution. - - `'QueryFinish' = 2` — Successful end of query execution. - - `'ExceptionBeforeStart' = 3` — Exception before the start of query execution. - - `'ExceptionWhileProcessing' = 4` — Exception during the query execution. -- `event_date` (Date) — Query starting date. -- `event_time` (DateTime) — Query starting time. -- `query_start_time` (DateTime) — Start time of query execution. -- `query_duration_ms` (UInt64) — Duration of query execution. -- `read_rows` (UInt64) — Number of read rows. -- `read_bytes` (UInt64) — Number of read bytes. -- `written_rows` (UInt64) — For `INSERT` نمایش داده شد, تعداد ردیف نوشته شده. برای نمایش داده شد دیگر مقدار ستون 0 است. -- `written_bytes` (UInt64) — For `INSERT` نمایش داده شد, تعداد بایت نوشته شده. برای نمایش داده شد دیگر مقدار ستون 0 است. -- `result_rows` (UInt64) — Number of rows in the result. -- `result_bytes` (UInt64) — Number of bytes in the result. -- `memory_usage` (UInt64) — Memory consumption by the query. -- `query` (String) — Query string. -- `exception` (String) — Exception message. -- `stack_trace` (String) — Stack trace (a list of methods called before the error occurred). An empty string, if the query is completed successfully. -- `is_initial_query` (UInt8) — Query type. Possible values: - - 1 — Query was initiated by the client. - - 0 — Query was initiated by another query for distributed query execution. -- `user` (String) — Name of the user who initiated the current query. -- `query_id` (String) — ID of the query. -- `address` (IPv6) — IP address that was used to make the query. -- `port` (UInt16) — The client port that was used to make the query. -- `initial_user` (String) — Name of the user who ran the initial query (for distributed query execution). -- `initial_query_id` (String) — ID of the initial query (for distributed query execution). -- `initial_address` (IPv6) — IP address that the parent query was launched from. -- `initial_port` (UInt16) — The client port that was used to make the parent query. -- `interface` (UInt8) — Interface that the query was initiated from. Possible values: - - 1 — TCP. - - 2 — HTTP. -- `os_user` (String) — OS's username who runs [کلیک مشتری](../interfaces/cli.md). -- `client_hostname` (String) — Hostname of the client machine where the [کلیک مشتری](../interfaces/cli.md) یا یکی دیگر از مشتری تی پی اجرا می شود. -- `client_name` (String) — The [کلیک مشتری](../interfaces/cli.md) یا یکی دیگر از نام مشتری تی پی. -- `client_revision` (UInt32) — Revision of the [کلیک مشتری](../interfaces/cli.md) یا یکی دیگر از مشتری تی پی. -- `client_version_major` (UInt32) — Major version of the [کلیک مشتری](../interfaces/cli.md) یا یکی دیگر از مشتری تی پی. -- `client_version_minor` (UInt32) — Minor version of the [کلیک مشتری](../interfaces/cli.md) یا یکی دیگر از مشتری تی پی. -- `client_version_patch` (UInt32) — Patch component of the [کلیک مشتری](../interfaces/cli.md) یا یکی دیگر از نسخه مشتری تی سی پی. -- `http_method` (UInt8) — HTTP method that initiated the query. Possible values: - - 0 — The query was launched from the TCP interface. - - 1 — `GET` روش مورد استفاده قرار گرفت. - - 2 — `POST` روش مورد استفاده قرار گرفت. -- `http_user_agent` (String) — The `UserAgent` هدر در درخواست قام منتقل می شود. -- `quota_key` (String) — The “quota key” مشخص شده در [سهمیه](quotas.md) تنظیم (دیدن `keyed`). -- `revision` (UInt32) — ClickHouse revision. -- `thread_numbers` (Array(UInt32)) — Number of threads that are participating in query execution. -- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics. The description of them could be found in the table [سیستم.رویدادها](#system_tables-events) -- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics that are listed in the `ProfileEvents.Names` ستون. -- `Settings.Names` (Array(String)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` پارامتر به 1. -- `Settings.Values` (Array(String)) — Values of settings that are listed in the `Settings.Names` ستون. - -هر پرس و جو ایجاد یک یا دو ردیف در `query_log` جدول بسته به وضعیت پرس و جو: - -1. اگر اجرای پرس و جو موفق است, دو رویداد با انواع 1 و 2 ایجاد می شوند (دیدن `type` ستون). -2. اگر یک خطا در طول پردازش پرس و جو رخ داده است, دو رویداد با انواع 1 و 4 ایجاد می شوند. -3. اگر یک خطا قبل از راه اندازی پرس و جو رخ داده است, یک رویداد واحد با نوع 3 ایجاد شده است. - -به طور پیش فرض, سیاهههای مربوط به جدول در فواصل 7.5 ثانیه اضافه. شما می توانید این فاصله در مجموعه [_خروج](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) تنظیم سرور (نگاه کنید به `flush_interval_milliseconds` پارامتر). به خیط و پیت کردن سیاهههای مربوط به زور از بافر حافظه را به جدول, استفاده از `SYSTEM FLUSH LOGS` پرس و جو. - -هنگامی که جدول به صورت دستی حذف, به طور خودکار در پرواز ایجاد. توجه داشته باشید که تمام سیاهههای مربوط قبلی حذف خواهد شد. - -!!! note "یادداشت" - دوره ذخیره سازی برای سیاهههای مربوط نامحدود است. سیاهههای مربوط به طور خودکار از جدول حذف نمی شود. شما نیاز به سازماندهی حذف سیاهههای مربوط منسوخ شده خود را. - -شما می توانید یک کلید پارتیشن بندی دلخواه برای مشخص `system.query_log` جدول در [_خروج](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) تنظیم سرور (نگاه کنید به `partition_by` پارامتر). - -## سیستم._ر_خروج {#system_tables-query-thread-log} - -جدول شامل اطلاعات در مورد هر موضوع اجرای پرس و جو. - -تاتر این جدول را فقط در صورتی ایجاد می کند [_ر_خروج](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) پارامتر سرور مشخص شده است. این پارامتر مجموعه قوانین ورود به سیستم, مانند فاصله ورود به سیستم و یا نام جدول نمایش داده شد خواهد شد وارد سایت شوید. - -برای فعال کردن ورود به سیستم پرس و جو, تنظیم [باز کردن](settings/settings.md#settings-log-query-threads) پارامتر به 1. برای اطلاعات بیشتر [تنظیمات](settings/settings.md) بخش. - -ستونها: - -- `event_date` (Date) — the date when the thread has finished execution of the query. -- `event_time` (DateTime) — the date and time when the thread has finished execution of the query. -- `query_start_time` (DateTime) — Start time of query execution. -- `query_duration_ms` (UInt64) — Duration of query execution. -- `read_rows` (UInt64) — Number of read rows. -- `read_bytes` (UInt64) — Number of read bytes. -- `written_rows` (UInt64) — For `INSERT` نمایش داده شد, تعداد ردیف نوشته شده. برای نمایش داده شد دیگر مقدار ستون 0 است. -- `written_bytes` (UInt64) — For `INSERT` نمایش داده شد, تعداد بایت نوشته شده. برای نمایش داده شد دیگر مقدار ستون 0 است. -- `memory_usage` (Int64) — The difference between the amount of allocated and freed memory in context of this thread. -- `peak_memory_usage` (Int64) — The maximum difference between the amount of allocated and freed memory in context of this thread. -- `thread_name` (String) — Name of the thread. -- `thread_number` (UInt32) — Internal thread ID. -- `os_thread_id` (Int32) — OS thread ID. -- `master_thread_id` (UInt64) — OS initial ID of initial thread. -- `query` (String) — Query string. -- `is_initial_query` (UInt8) — Query type. Possible values: - - 1 — Query was initiated by the client. - - 0 — Query was initiated by another query for distributed query execution. -- `user` (String) — Name of the user who initiated the current query. -- `query_id` (String) — ID of the query. -- `address` (IPv6) — IP address that was used to make the query. -- `port` (UInt16) — The client port that was used to make the query. -- `initial_user` (String) — Name of the user who ran the initial query (for distributed query execution). -- `initial_query_id` (String) — ID of the initial query (for distributed query execution). -- `initial_address` (IPv6) — IP address that the parent query was launched from. -- `initial_port` (UInt16) — The client port that was used to make the parent query. -- `interface` (UInt8) — Interface that the query was initiated from. Possible values: - - 1 — TCP. - - 2 — HTTP. -- `os_user` (String) — OS's username who runs [کلیک مشتری](../interfaces/cli.md). -- `client_hostname` (String) — Hostname of the client machine where the [کلیک مشتری](../interfaces/cli.md) یا یکی دیگر از مشتری تی پی اجرا می شود. -- `client_name` (String) — The [کلیک مشتری](../interfaces/cli.md) یا یکی دیگر از نام مشتری تی پی. -- `client_revision` (UInt32) — Revision of the [کلیک مشتری](../interfaces/cli.md) یا یکی دیگر از مشتری تی پی. -- `client_version_major` (UInt32) — Major version of the [کلیک مشتری](../interfaces/cli.md) یا یکی دیگر از مشتری تی پی. -- `client_version_minor` (UInt32) — Minor version of the [کلیک مشتری](../interfaces/cli.md) یا یکی دیگر از مشتری تی پی. -- `client_version_patch` (UInt32) — Patch component of the [کلیک مشتری](../interfaces/cli.md) یا یکی دیگر از نسخه مشتری تی سی پی. -- `http_method` (UInt8) — HTTP method that initiated the query. Possible values: - - 0 — The query was launched from the TCP interface. - - 1 — `GET` روش مورد استفاده قرار گرفت. - - 2 — `POST` روش مورد استفاده قرار گرفت. -- `http_user_agent` (String) — The `UserAgent` هدر در درخواست قام منتقل می شود. -- `quota_key` (String) — The “quota key” مشخص شده در [سهمیه](quotas.md) تنظیم (دیدن `keyed`). -- `revision` (UInt32) — ClickHouse revision. -- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics for this thread. The description of them could be found in the table [سیستم.رویدادها](#system_tables-events) -- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` ستون. - -به طور پیش فرض, سیاهههای مربوط به جدول در فواصل 7.5 ثانیه اضافه. شما می توانید این فاصله در مجموعه [_ر_خروج](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) تنظیم سرور (نگاه کنید به `flush_interval_milliseconds` پارامتر). به خیط و پیت کردن سیاهههای مربوط به زور از بافر حافظه را به جدول, استفاده از `SYSTEM FLUSH LOGS` پرس و جو. - -هنگامی که جدول به صورت دستی حذف, به طور خودکار در پرواز ایجاد. توجه داشته باشید که تمام سیاهههای مربوط قبلی حذف خواهد شد. - -!!! note "یادداشت" - دوره ذخیره سازی برای سیاهههای مربوط نامحدود است. سیاهههای مربوط به طور خودکار از جدول حذف نمی شود. شما نیاز به سازماندهی حذف سیاهههای مربوط منسوخ شده خود را. - -شما می توانید یک کلید پارتیشن بندی دلخواه برای مشخص `system.query_thread_log` جدول در [_ر_خروج](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) تنظیم سرور (نگاه کنید به `partition_by` پارامتر). - -## سیستم._قطع {#system_tables-trace_log} - -حاوی ردیاب های پشته ای است که توسط پروفایل پرس و جو نمونه گیری می شود. - -تاتر این جدول زمانی ایجاد می کند [_قطع](server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) بخش پیکربندی سرور تنظیم شده است. همچنین [جستجو](settings/settings.md#query_profiler_real_time_period_ns) و [ایران در تهران](settings/settings.md#query_profiler_cpu_time_period_ns) تنظیمات باید تنظیم شود. - -برای تجزیه و تحلیل سیاهههای مربوط, استفاده از `addressToLine`, `addressToSymbol` و `demangle` توابع درون گرایی. - -ستونها: - -- `event_date` ([تاریخ](../sql-reference/data-types/date.md)) — Date of sampling moment. - -- `event_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Timestamp of the sampling moment. - -- `timestamp_ns` ([UInt64](../sql-reference/data-types/int-uint.md)) — Timestamp of the sampling moment in nanoseconds. - -- `revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ClickHouse server build revision. - - هنگام اتصال به سرور توسط `clickhouse-client`, شما رشته شبیه به دیدن `Connected to ClickHouse server version 19.18.1 revision 54429.`. این فیلد شامل `revision` اما نه `version` از یک سرور. - -- `timer_type` ([شمار8](../sql-reference/data-types/enum.md)) — Timer type: - - - `Real` نشان دهنده زمان دیوار ساعت. - - `CPU` نشان دهنده زمان پردازنده. - -- `thread_number` ([UInt32](../sql-reference/data-types/int-uint.md)) — Thread identifier. - -- `query_id` ([رشته](../sql-reference/data-types/string.md)) — Query identifier that can be used to get details about a query that was running from the [_خروج](#system_tables-query_log) جدول سیستم. - -- `trace` ([Array(UInt64)](../sql-reference/data-types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. - -**مثال** - -``` sql -SELECT * FROM system.trace_log LIMIT 1 \G -``` - -``` text -Row 1: -────── -event_date: 2019-11-15 -event_time: 2019-11-15 15:09:38 -revision: 54428 -timer_type: Real -thread_number: 48 -query_id: acc4d61f-5bd1-4a3e-bc91-2180be37c915 -trace: [94222141367858,94222152240175,94222152325351,94222152329944,94222152330796,94222151449980,94222144088167,94222151682763,94222144088167,94222151682763,94222144088167,94222144058283,94222144059248,94222091840750,94222091842302,94222091831228,94222189631488,140509950166747,140509942945935] -``` - -## سیستم.تکرار {#system_tables-replicas} - -شامل اطلاعات و وضعیت برای جداول تکرار ساکن بر روی سرور محلی. -این جدول را می توان برای نظارت استفاده می شود. جدول شامل یک ردیف برای هر تکرار \* جدول. - -مثال: - -``` sql -SELECT * -FROM system.replicas -WHERE table = 'visits' -FORMAT Vertical -``` - -``` text -Row 1: -────── -database: merge -table: visits -engine: ReplicatedCollapsingMergeTree -is_leader: 1 -can_become_leader: 1 -is_readonly: 0 -is_session_expired: 0 -future_parts: 1 -parts_to_check: 0 -zookeeper_path: /clickhouse/tables/01-06/visits -replica_name: example01-06-1.yandex.ru -replica_path: /clickhouse/tables/01-06/visits/replicas/example01-06-1.yandex.ru -columns_version: 9 -queue_size: 1 -inserts_in_queue: 0 -merges_in_queue: 1 -part_mutations_in_queue: 0 -queue_oldest_time: 2020-02-20 08:34:30 -inserts_oldest_time: 1970-01-01 00:00:00 -merges_oldest_time: 2020-02-20 08:34:30 -part_mutations_oldest_time: 1970-01-01 00:00:00 -oldest_part_to_get: -oldest_part_to_merge_to: 20200220_20284_20840_7 -oldest_part_to_mutate_to: -log_max_index: 596273 -log_pointer: 596274 -last_queue_update: 2020-02-20 08:34:32 -absolute_delay: 0 -total_replicas: 2 -active_replicas: 2 -``` - -ستونها: - -- `database` (`String`)- نام پایگاه داده -- `table` (`String`)- نام جدول -- `engine` (`String`)- نام موتور جدول -- `is_leader` (`UInt8`)- چه ماکت رهبر است. - فقط یک ماکت در یک زمان می تواند رهبر باشد. رهبر برای انتخاب پس زمینه ادغام به انجام است. - توجه داشته باشید که می نویسد را می توان به هر ماکت است که در دسترس است و یک جلسه در زک انجام, صرف نظر از اینکه این یک رهبر است. -- `can_become_leader` (`UInt8`)- چه ماکت می تواند به عنوان یک رهبر انتخاب می شوند. -- `is_readonly` (`UInt8`)- چه ماکت در حالت فقط خواندنی است. - در این حالت روشن است اگر پیکربندی ندارد بخش با باغ وحش اگر یک خطای ناشناخته رخ داده است که reinitializing جلسات در باغ وحش و در طول جلسه reinitialization در باغ وحش. -- `is_session_expired` (`UInt8`)- جلسه با باغ وحش منقضی شده است. در واقع همان `is_readonly`. -- `future_parts` (`UInt32`)- تعداد قطعات داده است که به عنوان نتیجه درج و یا ادغام که هنوز انجام نشده است ظاهر می شود. -- `parts_to_check` (`UInt32`)- تعداد قطعات داده در صف برای تایید. اگر شک وجود دارد که ممکن است صدمه دیده است بخشی در صف تایید قرار داده است. -- `zookeeper_path` (`String`)- مسیر به داده های جدول در باغ وحش. -- `replica_name` (`String`)- نام ماکت در باغ وحش. کپی های مختلف از همان جدول نام های مختلف. -- `replica_path` (`String`)- مسیر به داده های ماکت در باغ وحش. همان الحاق ‘zookeeper_path/replicas/replica_path’. -- `columns_version` (`Int32`)- تعداد نسخه از ساختار جدول . نشان می دهد که چند بار تغییر انجام شد. اگر کپی نسخه های مختلف, به این معنی برخی از کپی ساخته شده است همه از تغییر نکرده است. -- `queue_size` (`UInt32`)- اندازه صف برای عملیات در حال انتظار برای انجام شود . عملیات شامل قرار دادن بلوک های داده ادغام و برخی اقدامات دیگر. معمولا همزمان با `future_parts`. -- `inserts_in_queue` (`UInt32`)- تعداد درج بلوک از داده ها که نیاز به ساخته شده است . درج معمولا نسبتا به سرعت تکرار. اگر این تعداد بزرگ است, به این معنی چیزی اشتباه است. -- `merges_in_queue` (`UInt32`)- تعداد ادغام انتظار ساخته شود. گاهی اوقات ادغام طولانی هستند, بنابراین این مقدار ممکن است بیشتر از صفر برای یک مدت طولانی. -- `part_mutations_in_queue` (`UInt32`)- تعداد جهش در انتظار ساخته شده است. -- `queue_oldest_time` (`DateTime`)- اگر `queue_size` بیشتر از 0, نشان می دهد که قدیمی ترین عملیات به صف اضافه شد. -- `inserts_oldest_time` (`DateTime` دیدن وضعیت شبکه `queue_oldest_time` -- `merges_oldest_time` (`DateTime` دیدن وضعیت شبکه `queue_oldest_time` -- `part_mutations_oldest_time` (`DateTime` دیدن وضعیت شبکه `queue_oldest_time` - -4 ستون بعدی یک مقدار غیر صفر تنها جایی که یک جلسه فعال با زک وجود دارد. - -- `log_max_index` (`UInt64`)- حداکثر تعداد ورودی در ورود به سیستم از فعالیت های عمومی. -- `log_pointer` (`UInt64`) - حداکثر تعداد ورودی در ورود به سیستم از فعالیت های عمومی که ماکت کپی شده به صف اعدام خود را, به علاوه یک. اگر `log_pointer` بسیار کوچکتر از `log_max_index`, چیزی اشتباه است. -- `last_queue_update` (`DateTime`)- هنگامی که صف در زمان گذشته به روز شد. -- `absolute_delay` (`UInt64`)- تاخیر چقدر بزرگ در ثانیه ماکت فعلی است. -- `total_replicas` (`UInt8`)- تعداد کل کپی شناخته شده از این جدول. -- `active_replicas` (`UInt8`)- تعداد کپی از این جدول که یک جلسه در باغ وحش (یعنی تعداد تکرار عملکرد). - -اگر شما درخواست تمام ستون, جدول ممکن است کمی کند کار, از چند بار خوانده شده از باغ وحش برای هر سطر ساخته شده. -اگر شما درخواست آخرین 4 ستون (log_max_index, log_pointer, total_replicas, active_replicas) جدول با این نسخهها کار به سرعت. - -مثلا, شما می توانید بررسی کنید که همه چیز به درستی کار مثل این: - -``` sql -SELECT - database, - table, - is_leader, - is_readonly, - is_session_expired, - future_parts, - parts_to_check, - columns_version, - queue_size, - inserts_in_queue, - merges_in_queue, - log_max_index, - log_pointer, - total_replicas, - active_replicas -FROM system.replicas -WHERE - is_readonly - OR is_session_expired - OR future_parts > 20 - OR parts_to_check > 10 - OR queue_size > 20 - OR inserts_in_queue > 10 - OR log_max_index - log_pointer > 10 - OR total_replicas < 2 - OR active_replicas < total_replicas -``` - -اگر این پرس و جو چیزی نمی گرداند, به این معنی که همه چیز خوب است. - -## سیستم.تنظیمات {#system-tables-system-settings} - -شامل اطلاعات در مورد تنظیمات جلسه برای کاربر فعلی. - -ستونها: - -- `name` ([رشته](../sql-reference/data-types/string.md)) — Setting name. -- `value` ([رشته](../sql-reference/data-types/string.md)) — Setting value. -- `changed` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether a setting is changed from its default value. -- `description` ([رشته](../sql-reference/data-types/string.md)) — Short setting description. -- `min` ([Nullable](../sql-reference/data-types/nullable.md)([رشته](../sql-reference/data-types/string.md))) — Minimum value of the setting, if any is set via [قیدها](settings/constraints-on-settings.md#constraints-on-settings). اگر تنظیمات دارای حداقل مقدار, شامل [NULL](../sql-reference/syntax.md#null-literal). -- `max` ([Nullable](../sql-reference/data-types/nullable.md)([رشته](../sql-reference/data-types/string.md))) — Maximum value of the setting, if any is set via [قیدها](settings/constraints-on-settings.md#constraints-on-settings). اگر تنظیمات دارای حداکثر مقدار, شامل [NULL](../sql-reference/syntax.md#null-literal). -- `readonly` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether the current user can change the setting: - - `0` — Current user can change the setting. - - `1` — Current user can't change the setting. - -**مثال** - -مثال زیر نشان می دهد که چگونه برای دریافت اطلاعات در مورد تنظیمات که شامل نام `min_i`. - -``` sql -SELECT * -FROM system.settings -WHERE name LIKE '%min_i%' -``` - -``` text -┌─name────────────────────────────────────────┬─value─────┬─changed─┬─description───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─min──┬─max──┬─readonly─┐ -│ min_insert_block_size_rows │ 1048576 │ 0 │ Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ -│ min_insert_block_size_bytes │ 268435456 │ 0 │ Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ -│ read_backoff_min_interval_between_events_ms │ 1000 │ 0 │ Settings to reduce the number of threads in case of slow reads. Do not pay attention to the event, if the previous one has passed less than a certain amount of time. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ -└─────────────────────────────────────────────┴───────────┴─────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴──────┴──────┴──────────┘ -``` - -استفاده از `WHERE changed` می تواند مفید باشد, مثلا, زمانی که شما می خواهید برای بررسی: - -- اینکه تنظیمات در پروندههای پیکربندی به درستی بارگذاری شوند یا در حال استفاده باشند. -- تنظیماتی که در جلسه فعلی تغییر کرده است. - - - -``` sql -SELECT * FROM system.settings WHERE changed AND name='load_balancing' -``` - -**همچنین نگاه کنید به** - -- [تنظیمات](settings/index.md#session-settings-intro) -- [مجوز برای نمایش داده شد](settings/permissions-for-queries.md#settings_readonly) -- [محدودیت در تنظیمات](settings/constraints-on-settings.md) - -## سیستم._زبانهها {#system.table_engines} - -``` text -┌─name───────────────────┬─value───────┐ -│ max_threads │ 8 │ -│ use_uncompressed_cache │ 0 │ -│ load_balancing │ random │ -│ max_memory_usage │ 10000000000 │ -└────────────────────────┴─────────────┘ -``` - -## سیستم.خرابی در حذف گواهینامهها {#system-merge_tree_settings} - -حاوی اطلاعات در مورد تنظیمات برای `MergeTree` میز - -ستونها: - -- `name` (String) — Setting name. -- `value` (String) — Setting value. -- `description` (String) — Setting description. -- `type` (String) — Setting type (implementation specific string value). -- `changed` (UInt8) — Whether the setting was explicitly defined in the config or explicitly changed. - -## سیستم._زبانهها {#system-table-engines} - -شامل شرح موتورهای جدول پشتیبانی شده توسط سرور و اطلاعات پشتیبانی از ویژگی های خود را. - -این جدول شامل ستون های زیر (نوع ستون در براکت نشان داده شده است): - -- `name` (String) — The name of table engine. -- `supports_settings` (UInt8) — Flag that indicates if table engine supports `SETTINGS` بند بند. -- `supports_skipping_indices` (UInt8) — Flag that indicates if table engine supports [پرش شاخص](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-data_skipping-indexes). -- `supports_ttl` (UInt8) — Flag that indicates if table engine supports [TTL](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl). -- `supports_sort_order` (UInt8) — Flag that indicates if table engine supports clauses `PARTITION_BY`, `PRIMARY_KEY`, `ORDER_BY` و `SAMPLE_BY`. -- `supports_replication` (UInt8) — Flag that indicates if table engine supports [تکرار داده ها](../engines/table-engines/mergetree-family/replication.md). -- `supports_duduplication` (UInt8) — Flag that indicates if table engine supports data deduplication. - -مثال: - -``` sql -SELECT * -FROM system.table_engines -WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') -``` - -``` text -┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┐ -│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ -│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ -│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ -└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┘ -``` - -**همچنین نگاه کنید به** - -- ادغام خانواده [بندهای پرسوجو](../engines/table-engines/mergetree-family/mergetree.md#mergetree-query-clauses) -- کافکا [تنظیمات](../engines/table-engines/integrations/kafka.md#table_engine-kafka-creating-a-table) -- پیوستن [تنظیمات](../engines/table-engines/special/join.md#join-limitations-and-settings) - -## سیستم.جداول {#system-tables} - -حاوی ابرداده از هر جدول که سرور می داند در مورد. جداول جداگانه در نشان داده نمی شود `system.tables`. - -این جدول شامل ستون های زیر (نوع ستون در براکت نشان داده شده است): - -- `database` (String) — The name of the database the table is in. - -- `name` (String) — Table name. - -- `engine` (String) — Table engine name (without parameters). - -- `is_temporary` (زیر8) - پرچم که نشان می دهد که جدول موقت است. - -- `data_path` (رشته) - مسیر به داده های جدول در سیستم فایل. - -- `metadata_path` (رشته) - مسیر به ابرداده جدول در سیستم فایل. - -- `metadata_modification_time` (تاریخ ساعت) - زمان شدن اصلاح ابرداده جدول. - -- `dependencies_database` - وابستگی پایگاه داده . - -- `dependencies_table` (رشته)) - وابستگی های جدول ([ماده بینی](../engines/table-engines/special/materializedview.md) جداول بر اساس جدول فعلی). - -- `create_table_query` (رشته) - پرس و جو که برای ایجاد جدول مورد استفاده قرار گرفت. - -- `engine_full` (رشته) - پارامترهای موتور جدول. - -- `partition_key` (رشته) - بیان کلید پارتیشن مشخص شده در جدول. - -- `sorting_key` (رشته) - عبارت کلیدی مرتب سازی مشخص شده در جدول. - -- `primary_key` (رشته) - عبارت کلیدی اولیه مشخص شده در جدول. - -- `sampling_key` (رشته) - نمونه عبارت کلیدی مشخص شده در جدول. - -- `storage_policy` (رشته) - سیاست ذخیره سازی: - - - [ادغام](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) - - [توزیع شده](../engines/table-engines/special/distributed.md#distributed) - -- `total_rows` (Nullable(UInt64)) - تعداد کل ردیف آن است که اگر ممکن است به سرعت تعیین دقیق تعداد ردیف در جدول در غیر این صورت `Null` (از جمله زیرینگ `Buffer` جدول). - -- `total_bytes` (Nullable(UInt64)) - مجموع تعداد بایت, اگر آن را ممکن است به سرعت تعیین دقیق تعداد بایت به صورت جدول ذخیره در غیر این صورت `Null` (**نه** شامل هر ذخیره سازی زمینه ای). - - - If the table stores data on disk, returns used space on disk (i.e. compressed). - - اگر جدول ذخیره داده ها در حافظه, بازده تعداد تقریبی بایت مورد استفاده در حافظه. - -این `system.tables` جدول در استفاده می شود `SHOW TABLES` اجرای پرس و جو. - -## سیستم.باغ وحش {#system-zookeeper} - -جدول وجود ندارد اگر باغ وحش پیکربندی نشده است. اجازه می دهد تا خواندن داده ها از خوشه باغ وحش تعریف شده در پیکربندی. -پرس و جو باید یک ‘path’ شرایط برابری در بند جایی که. این مسیر در باغ وحش برای کودکان که شما می خواهید برای دریافت اطلاعات برای است. - -پرسوجو `SELECT * FROM system.zookeeper WHERE path = '/clickhouse'` خروجی داده ها برای همه کودکان در `/clickhouse` گره. -به داده های خروجی برای تمام گره های ریشه, نوشتن مسیر = ‘/’. -اگر مسیر مشخص شده در ‘path’ وجود ندارد, یک استثنا پرتاب خواهد شد. - -ستونها: - -- `name` (String) — The name of the node. -- `path` (String) — The path to the node. -- `value` (String) — Node value. -- `dataLength` (Int32) — Size of the value. -- `numChildren` (Int32) — Number of descendants. -- `czxid` (Int64) — ID of the transaction that created the node. -- `mzxid` (Int64) — ID of the transaction that last changed the node. -- `pzxid` (Int64) — ID of the transaction that last deleted or added descendants. -- `ctime` (DateTime) — Time of node creation. -- `mtime` (DateTime) — Time of the last modification of the node. -- `version` (Int32) — Node version: the number of times the node was changed. -- `cversion` (Int32) — Number of added or removed descendants. -- `aversion` (Int32) — Number of changes to the ACL. -- `ephemeralOwner` (Int64) — For ephemeral nodes, the ID of the session that owns this node. - -مثال: - -``` sql -SELECT * -FROM system.zookeeper -WHERE path = '/clickhouse/tables/01-08/visits/replicas' -FORMAT Vertical -``` - -``` text -Row 1: -────── -name: example01-08-1.yandex.ru -value: -czxid: 932998691229 -mzxid: 932998691229 -ctime: 2015-03-27 16:49:51 -mtime: 2015-03-27 16:49:51 -version: 0 -cversion: 47 -aversion: 0 -ephemeralOwner: 0 -dataLength: 0 -numChildren: 7 -pzxid: 987021031383 -path: /clickhouse/tables/01-08/visits/replicas - -Row 2: -────── -name: example01-08-2.yandex.ru -value: -czxid: 933002738135 -mzxid: 933002738135 -ctime: 2015-03-27 16:57:01 -mtime: 2015-03-27 16:57:01 -version: 0 -cversion: 37 -aversion: 0 -ephemeralOwner: 0 -dataLength: 0 -numChildren: 7 -pzxid: 987021252247 -path: /clickhouse/tables/01-08/visits/replicas -``` - -## سیستم.جهشها {#system_tables-mutations} - -جدول حاوی اطلاعات در مورد [جهشها](../sql-reference/statements/alter.md#alter-mutations) از جداول ادغام و پیشرفت خود را. هر دستور جهش توسط یک ردیف نشان داده شده است. جدول دارای ستون های زیر است: - -**دادگان**, **جدول** - نام پایگاه داده و جدول که جهش استفاده شد . - -**قطع عضو** - شناسه جهش. برای جداول تکرار این شناسه به نام زنود در مطابقت `/mutations/` راهنمای در باغ وحش. برای جداول سه برابر شناسه مربوط به فایل نام در دایرکتوری داده ها از جدول. - -**فرمان** - رشته فرمان جهش (بخشی از پرس و جو پس از `ALTER TABLE [db.]table`). - -**_بروزرسانی** - هنگامی که این دستور جهش برای اجرای ارسال شد . - -**_شمارهی بلوک.ا_ضافه کردن**, **_شمارهی بلوک.شماره** - ستون تو در تو . برای جهش از جداول تکرار, این شامل یک رکورد برای هر پارتیشن: شناسه پارتیشن و شماره بلوک که توسط جهش خریداری شد (در هر پارتیشن, تنها بخش هایی که حاوی بلوک با اعداد کمتر از تعداد بلوک های خریداری شده توسط جهش در پارتیشن که جهش خواهد شد). در جداول غیر تکرار, تعداد بلوک در تمام پارتیشن به صورت یک توالی واحد. این به این معنی است که برای جهش از جداول غیر تکرار, ستون یک رکورد با یک عدد بلوک واحد خریداری شده توسط جهش شامل. - -**_کوچکنمایی** - تعدادی از قطعات داده است که نیاز به جهش را به پایان برساند جهش یافته است . - -**_مخفی کردن** - توجه داشته باشید که حتی اگر `parts_to_do = 0` ممکن است که جهش جدول تکرار هنوز به دلیل درج طولانی در حال اجرا است که ایجاد بخش داده های جدید است که نیاز به جهش انجام می شود است. - -اگر مشکلی با جهش برخی از قطعات وجود دارد, ستون های زیر حاوی اطلاعات اضافی: - -**_شروع مجدد** - نام جدید ترین بخش است که نمی تواند جهش یافته است. - -**زمان _رشته** - زمان جدید ترین شکست جهش بخشی . - -**_شروع مجدد** - پیام استثنا که باعث شکست جهش بخشی اخیر. - -## سیستم.دیسکها {#system_tables-disks} - -حاوی اطلاعات در مورد دیسک های تعریف شده در [پیکربندی کارساز](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes_configure). - -ستونها: - -- `name` ([رشته](../sql-reference/data-types/string.md)) — Name of a disk in the server configuration. -- `path` ([رشته](../sql-reference/data-types/string.md)) — Path to the mount point in the file system. -- `free_space` ([UInt64](../sql-reference/data-types/int-uint.md)) — Free space on disk in bytes. -- `total_space` ([UInt64](../sql-reference/data-types/int-uint.md)) — Disk volume in bytes. -- `keep_free_space` ([UInt64](../sql-reference/data-types/int-uint.md)) — Amount of disk space that should stay free on disk in bytes. Defined in the `keep_free_space_bytes` پارامتر پیکربندی دیسک. - -## سیستم.داستان_یابی {#system_tables-storage_policies} - -حاوی اطلاعات در مورد سیاست های ذخیره سازی و حجم تعریف شده در [پیکربندی کارساز](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes_configure). - -ستونها: - -- `policy_name` ([رشته](../sql-reference/data-types/string.md)) — Name of the storage policy. -- `volume_name` ([رشته](../sql-reference/data-types/string.md)) — Volume name defined in the storage policy. -- `volume_priority` ([UInt64](../sql-reference/data-types/int-uint.md)) — Volume order number in the configuration. -- `disks` ([رشته)](../sql-reference/data-types/array.md)) — Disk names, defined in the storage policy. -- `max_data_part_size` ([UInt64](../sql-reference/data-types/int-uint.md)) — Maximum size of a data part that can be stored on volume disks (0 — no limit). -- `move_factor` ([جسم شناور64](../sql-reference/data-types/float.md)) — Ratio of free disk space. When the ratio exceeds the value of configuration parameter, ClickHouse start to move data to the next volume in order. - -اگر سیاست ذخیره سازی شامل بیش از یک حجم, سپس اطلاعات برای هر حجم در ردیف فرد از جدول ذخیره می شود. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/system_tables/) diff --git a/docs/fa/operations/tips.md b/docs/fa/operations/tips.md deleted file mode 100644 index cf6e5660ac9..00000000000 --- a/docs/fa/operations/tips.md +++ /dev/null @@ -1,252 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 58 -toc_title: "\u062A\u0648\u0635\u06CC\u0647 \u0647\u0627\u06CC \u0627\u0633\u062A\u0641\ - \u0627\u062F\u0647" ---- - -# توصیه های استفاده {#usage-recommendations} - -## فرماندار پوسته پوسته شدن پردازنده {#cpu-scaling-governor} - -همیشه استفاده از `performance` پوسته پوسته شدن فرماندار. این `on-demand` پوسته پوسته شدن فرماندار کار می کند بسیار بدتر با تقاضای به طور مداوم بالا. - -``` bash -$ echo 'performance' | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -``` - -## محدودیت های پردازنده {#cpu-limitations} - -پردازنده می تواند بیش از حد گرم. استفاده `dmesg` برای دیدن اگر نرخ ساعت پردازنده به دلیل گرمای بیش از حد محدود بود. -محدودیت همچنین می توانید خارجی در سطح مرکز داده تنظیم شود. شما می توانید استفاده کنید `turbostat` تحت نظر داشته باشمش - -## RAM {#ram} - -برای مقدار کمی از داده ها (تا ~ 200 گیگابایت فشرده), بهتر است به استفاده از حافظه به همان اندازه که حجم داده ها. -برای مقادیر زیادی از داده ها و در هنگام پردازش تعاملی (اینترنتی) نمایش داده شد, شما باید یک مقدار مناسب از رم استفاده (128 گیگابایت یا بیشتر) بنابراین زیر مجموعه داده های داغ در کش صفحات مناسب خواهد شد. -حتی برای حجم داده ها از ~50 سل در هر سرور, با استفاده از 128 گیگابایت رم به طور قابل توجهی بهبود می بخشد عملکرد پرس و جو در مقایسه با 64 گیگابایت. - -هنوز بیش از حد غیر فعال کردن نیست. مقدار `cat /proc/sys/vm/overcommit_memory` باید 0 یا 1. بدو - -``` bash -$ echo 0 | sudo tee /proc/sys/vm/overcommit_memory -``` - -## صفحات بزرگ {#huge-pages} - -همیشه صفحات بزرگ شفاف غیر فعال کنید. این با تخصیص حافظه تداخل, که منجر به تخریب عملکرد قابل توجهی. - -``` bash -$ echo 'never' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled -``` - -استفاده `perf top` برای تماشای زمان صرف شده در هسته برای مدیریت حافظه. -صفحات بزرگ ثابت نیز لازم نیست اختصاص داده شود. - -## زیرسیستم ذخیره سازی {#storage-subsystem} - -اگر بودجه شما اجازه می دهد تا شما را به استفاده از اس اس دی, استفاده از اس اس دی. -اگر نه, استفاده از هارد. ساعت 7200 دور در دقیقه انجام خواهد شد. - -دادن اولویت به بسیاری از سرور با دیسک های سخت محلی بیش از تعداد کمتری از سرور با قفسه های دیسک متصل. -اما برای ذخیره سازی بایگانی با نمایش داده شد نادر, قفسه کار خواهد کرد. - -## RAID {#raid} - -هنگام استفاده از هارد, شما می توانید حمله خود را ترکیب-10, حمله-5, حمله-6 و یا حمله-50. -برای لینوکس, حمله نرم افزار بهتر است (با `mdadm`). ما توصیه نمی کنیم با استفاده از سطح. -هنگام ایجاد حمله-10, را انتخاب کنید `far` طرح بندی. -اگر بودجه شما اجازه می دهد تا, را انتخاب کنید حمله-10. - -اگر شما بیش از 4 دیسک, استفاده از حمله-6 (ترجیحا) و یا حمله-50, به جای حمله-5. -هنگام استفاده از حمله-5, حمله-6 و یا حمله-50, همیشه افزایش نزاع, از مقدار پیش فرض است که معمولا بهترین انتخاب نیست. - -``` bash -$ echo 4096 | sudo tee /sys/block/md2/md/stripe_cache_size -``` - -محاسبه تعداد دقیق از تعداد دستگاه ها و اندازه بلوک با استفاده از فرمول: `2 * num_devices * chunk_size_in_bytes / 4096`. - -اندازه بلوک 1024 کیلوبایت برای تمام تنظیمات حمله کافی است. -هرگز اندازه بلوک بیش از حد کوچک یا بیش از حد بزرگ تنظیم شده است. - -شما می توانید حمله استفاده-0 در اس اس دی. -صرف نظر از استفاده از حمله, همیشه تکرار برای امنیت داده ها استفاده. - -فعال کردن دفتر مرکزی اروپا با یک صف طولانی. برای HDD را انتخاب کنید CFQ زمانبندی و برای SSD را انتخاب کنید noop. کاهش نمی دهد ‘readahead’ تنظیمات. -برای هارد, فعال کردن کش نوشتن. - -## سیستم پرونده {#file-system} - -موجود 4 قابل اطمینان ترین گزینه است. تنظیم گزینههای سوارکردن `noatime, nobarrier`. -XFS نیز مناسب است اما از آن شده است به طور کامل تست شده با ClickHouse. -اکثر سیستم های فایل های دیگر نیز باید خوب کار می کنند. سیستم های فایل با تاخیر تخصیص کار بهتر است. - -## هسته لینوکس {#linux-kernel} - -هنوز یک هسته لینوکس منسوخ شده استفاده کنید. - -## شبکه {#network} - -اگر شما با استفاده از ایپو6, افزایش اندازه کش مسیر. -هسته لینوکس قبل از 3.2 بسیاری از مشکلات با اجرای قانون مجازات اسلامی بود. - -استفاده از حداقل یک 10 شبکه گیگابایت, در صورت امکان. 1 گیگابایت نیز کار خواهد کرد, اما برای وصله کپی با ده ها ترابایت داده بسیار بدتر خواهد بود, و یا برای پردازش نمایش داده شد توزیع با مقدار زیادی از داده های متوسط. - -## باغ وحش {#zookeeper} - -شما احتمالا در حال حاضر با استفاده از باغ وحش برای مقاصد دیگر. شما می توانید نصب و راه اندازی همان باغ وحش استفاده, اگر در حال حاضر بیش از حد نیست. - -It's best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated. - -شما هرگز نباید از اسکریپت های دستی نوشته شده برای انتقال داده ها بین خوشه های مختلف باغ وحش استفاده کنید زیرا نتیجه برای گره های متوالی نادرست خواهد بود. هرگز استفاده از “zkcopy” ابزار به همین دلیل: https://github.com/ksprojects/zkcopy/issues/15 - -اگر میخواهید یک خوشه باغ وحش موجود را به دو قسمت تقسیم کنید راه درست این است که تعداد تکرار های خود را افزایش دهید و سپس به عنوان دو خوشه مستقل پیکربندی کنید. - -باغ وحش را بر روی سرورهای مشابه کلیک کنید. چرا که باغ وحش برای تاخیر بسیار حساس است و خانه رعیتی ممکن است تمام منابع سیستم در دسترس استفاده کنند. - -با تنظیمات پیش فرض, باغ وحش یک بمب زمان است: - -> سرور باغ وحش فایل ها را از عکس های فوری و سیاهههای مربوط قدیمی هنگام استفاده از پیکربندی پیش فرض حذف نمی کند (نگاه کنید به کالبد شکافی), و این به عهده اپراتور است. - -این بمب باید خنثی شود. - -باغ وحش (3.5.1) پیکربندی زیر در یاندکس استفاده می شود.محیط تولید متریکا تا 20 مه 2017: - -باغ وحش.cfg: - -``` bash -# http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html - -# The number of milliseconds of each tick -tickTime=2000 -# The number of ticks that the initial -# synchronization phase can take -initLimit=30000 -# The number of ticks that can pass between -# sending a request and getting an acknowledgement -syncLimit=10 - -maxClientCnxns=2000 - -maxSessionTimeout=60000000 -# the directory where the snapshot is stored. -dataDir=/opt/zookeeper/{{ '{{' }} cluster['name'] {{ '}}' }}/data -# Place the dataLogDir to a separate physical disc for better performance -dataLogDir=/opt/zookeeper/{{ '{{' }} cluster['name'] {{ '}}' }}/logs - -autopurge.snapRetainCount=10 -autopurge.purgeInterval=1 - - -# To avoid seeks ZooKeeper allocates space in the transaction log file in -# blocks of preAllocSize kilobytes. The default block size is 64M. One reason -# for changing the size of the blocks is to reduce the block size if snapshots -# are taken more often. (Also, see snapCount). -preAllocSize=131072 - -# Clients can submit requests faster than ZooKeeper can process them, -# especially if there are a lot of clients. To prevent ZooKeeper from running -# out of memory due to queued requests, ZooKeeper will throttle clients so that -# there is no more than globalOutstandingLimit outstanding requests in the -# system. The default limit is 1,000.ZooKeeper logs transactions to a -# transaction log. After snapCount transactions are written to a log file a -# snapshot is started and a new transaction log file is started. The default -# snapCount is 10,000. -snapCount=3000000 - -# If this option is defined, requests will be will logged to a trace file named -# traceFile.year.month.day. -#traceFile= - -# Leader accepts client connections. Default value is "yes". The leader machine -# coordinates updates. For higher update throughput at thes slight expense of -# read throughput the leader can be configured to not accept clients and focus -# on coordination. -leaderServes=yes - -standaloneEnabled=false -dynamicConfigFile=/etc/zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }}/conf/zoo.cfg.dynamic -``` - -نسخه جاوا: - -``` text -Java(TM) SE Runtime Environment (build 1.8.0_25-b17) -Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode) -``` - -پارامترهای جی ام: - -``` bash -NAME=zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }} -ZOOCFGDIR=/etc/$NAME/conf - -# TODO this is really ugly -# How to find out, which jars are needed? -# seems, that log4j requires the log4j.properties file to be in the classpath -CLASSPATH="$ZOOCFGDIR:/usr/build/classes:/usr/build/lib/*.jar:/usr/share/zookeeper/zookeeper-3.5.1-metrika.jar:/usr/share/zookeeper/slf4j-log4j12-1.7.5.jar:/usr/share/zookeeper/slf4j-api-1.7.5.jar:/usr/share/zookeeper/servlet-api-2.5-20081211.jar:/usr/share/zookeeper/netty-3.7.0.Final.jar:/usr/share/zookeeper/log4j-1.2.16.jar:/usr/share/zookeeper/jline-2.11.jar:/usr/share/zookeeper/jetty-util-6.1.26.jar:/usr/share/zookeeper/jetty-6.1.26.jar:/usr/share/zookeeper/javacc.jar:/usr/share/zookeeper/jackson-mapper-asl-1.9.11.jar:/usr/share/zookeeper/jackson-core-asl-1.9.11.jar:/usr/share/zookeeper/commons-cli-1.2.jar:/usr/src/java/lib/*.jar:/usr/etc/zookeeper" - -ZOOCFG="$ZOOCFGDIR/zoo.cfg" -ZOO_LOG_DIR=/var/log/$NAME -USER=zookeeper -GROUP=zookeeper -PIDDIR=/var/run/$NAME -PIDFILE=$PIDDIR/$NAME.pid -SCRIPTNAME=/etc/init.d/$NAME -JAVA=/usr/bin/java -ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain" -ZOO_LOG4J_PROP="INFO,ROLLINGFILE" -JMXLOCALONLY=false -JAVA_OPTS="-Xms{{ '{{' }} cluster.get('xms','128M') {{ '}}' }} \ - -Xmx{{ '{{' }} cluster.get('xmx','1G') {{ '}}' }} \ - -Xloggc:/var/log/$NAME/zookeeper-gc.log \ - -XX:+UseGCLogFileRotation \ - -XX:NumberOfGCLogFiles=16 \ - -XX:GCLogFileSize=16M \ - -verbose:gc \ - -XX:+PrintGCTimeStamps \ - -XX:+PrintGCDateStamps \ - -XX:+PrintGCDetails - -XX:+PrintTenuringDistribution \ - -XX:+PrintGCApplicationStoppedTime \ - -XX:+PrintGCApplicationConcurrentTime \ - -XX:+PrintSafepointStatistics \ - -XX:+UseParNewGC \ - -XX:+UseConcMarkSweepGC \ --XX:+CMSParallelRemarkEnabled" -``` - -نمک درون: - -``` text -description "zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }} centralized coordination service" - -start on runlevel [2345] -stop on runlevel [!2345] - -respawn - -limit nofile 8192 8192 - -pre-start script - [ -r "/etc/zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }}/conf/environment" ] || exit 0 - . /etc/zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }}/conf/environment - [ -d $ZOO_LOG_DIR ] || mkdir -p $ZOO_LOG_DIR - chown $USER:$GROUP $ZOO_LOG_DIR -end script - -script - . /etc/zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }}/conf/environment - [ -r /etc/default/zookeeper ] && . /etc/default/zookeeper - if [ -z "$JMXDISABLE" ]; then - JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY" - fi - exec start-stop-daemon --start -c $USER --exec $JAVA --name zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }} \ - -- -cp $CLASSPATH $JAVA_OPTS -Dzookeeper.log.dir=${ZOO_LOG_DIR} \ - -Dzookeeper.root.logger=${ZOO_LOG4J_PROP} $ZOOMAIN $ZOOCFG -end script -``` - -{## [مقاله اصلی](https://clickhouse.tech/docs/en/operations/tips/) ##} diff --git a/docs/fa/operations/troubleshooting.md b/docs/fa/operations/troubleshooting.md deleted file mode 100644 index f2dc276dc89..00000000000 --- a/docs/fa/operations/troubleshooting.md +++ /dev/null @@ -1,146 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 46 -toc_title: "\u0639\u06CC\u0628 \u06CC\u0627\u0628\u06CC" ---- - -# عیب یابی {#troubleshooting} - -- [نصب و راه اندازی](#troubleshooting-installation-errors) -- [اتصال به سرور](#troubleshooting-accepts-no-connections) -- [پردازش پرس و جو](#troubleshooting-does-not-process-queries) -- [کارایی پردازش پرس و جو](#troubleshooting-too-slow) - -## نصب و راه اندازی {#troubleshooting-installation-errors} - -### شما می توانید بسته های دب از مخزن کلیک با مناسب دریافت کنید {#you-cannot-get-deb-packages-from-clickhouse-repository-with-apt-get} - -- بررسی تنظیمات فایروال. -- اگر شما می توانید مخزن به هر دلیلی دسترسی پیدا کنید, دانلود بسته همانطور که در توصیف [شروع کار](../getting-started/index.md) مقاله و نصب دستی با استفاده از `sudo dpkg -i ` فرمان. همچنین شما می خواهد نیاز `tzdata` بسته - -## اتصال به سرور {#troubleshooting-accepts-no-connections} - -مشکلات احتمالی: - -- سرور در حال اجرا نیست. -- پارامترهای پیکربندی غیر منتظره و یا اشتباه. - -### کارساز در حال اجرا نیست {#server-is-not-running} - -**بررسی کنید که کارگزار روننیگ باشد** - -فرمان: - -``` bash -$ sudo service clickhouse-server status -``` - -اگر سرور در حال اجرا نیست, شروع با فرمان: - -``` bash -$ sudo service clickhouse-server start -``` - -**بررسی سیاههها** - -ورود اصلی `clickhouse-server` در `/var/log/clickhouse-server/clickhouse-server.log` به طور پیش فرض. - -اگر سرور با موفقیت شروع, شما باید رشته ها را ببینید: - -- ` Application: starting up.` — Server started. -- ` Application: Ready for connections.` — Server is running and ready for connections. - -اگر `clickhouse-server` شروع با یک خطای پیکربندی شکست خورده, شما باید ببینید `` رشته با شرح خطا. به عنوان مثال: - -``` text -2019.01.11 15:23:25.549505 [ 45 ] {} ExternalDictionaries: Failed reloading 'event2id' external dictionary: Poco::Exception. Code: 1000, e.code() = 111, e.displayText() = Connection refused, e.what() = Connection refused -``` - -اگر شما یک خطا در انتهای فایل را نمی بینم, از طریق تمام فایل با شروع از رشته نگاه: - -``` text - Application: starting up. -``` - -اگر شما سعی می کنید برای شروع یک نمونه دوم از `clickhouse-server` بر روی سرور, شما ورود به سیستم زیر را ببینید: - -``` text -2019.01.11 15:25:11.151730 [ 1 ] {} : Starting ClickHouse 19.1.0 with revision 54413 -2019.01.11 15:25:11.154578 [ 1 ] {} Application: starting up -2019.01.11 15:25:11.156361 [ 1 ] {} StatusFile: Status file ./status already exists - unclean restart. Contents: -PID: 8510 -Started at: 2019-01-11 15:24:23 -Revision: 54413 - -2019.01.11 15:25:11.156673 [ 1 ] {} Application: DB::Exception: Cannot lock file ./status. Another server instance in same directory is already running. -2019.01.11 15:25:11.156682 [ 1 ] {} Application: shutting down -2019.01.11 15:25:11.156686 [ 1 ] {} Application: Uninitializing subsystem: Logging Subsystem -2019.01.11 15:25:11.156716 [ 2 ] {} BaseDaemon: Stop SignalListener thread -``` - -**مشاهده سیستم.د سیاهههای مربوط** - -اگر شما هر گونه اطلاعات مفید در پیدا کنید `clickhouse-server` سیاهههای مربوط و یا هر گونه سیاهههای مربوط وجود ندارد, شما می توانید مشاهده `system.d` سیاهههای مربوط با استفاده از دستور: - -``` bash -$ sudo journalctl -u clickhouse-server -``` - -**شروع کلیک-سرور در حالت تعاملی** - -``` bash -$ sudo -u clickhouse /usr/bin/clickhouse-server --config-file /etc/clickhouse-server/config.xml -``` - -این دستور سرور را به عنوان یک برنامه تعاملی با پارامترهای استاندارد اسکریپت خودکار شروع می کند. در این حالت `clickhouse-server` چاپ تمام پیام های رویداد در کنسول. - -### پارامترهای پیکربندی {#configuration-parameters} - -بررسی: - -- تنظیمات کارگر بارانداز. - - اطمینان حاصل کنید که اگر شما اجرا خانه عروسکی در کارگر بارانداز در یک شبکه اینترنتی6 `network=host` قرار است. - -- تنظیمات نقطه پایانی. - - بررسی [_نوست فهرست](server-configuration-parameters/settings.md#server_configuration_parameters-listen_host) و [_صادر کردن](server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port) تنظیمات. - - سرور کلیک می پذیرد اتصالات مجنون تنها به طور پیش فرض. - -- تنظیمات پروتکل قام. - - بررسی تنظیمات پروتکل برای صفحه اصلی. - -- تنظیمات اتصال امن. - - بررسی: - - - این [_شروع مجدد](server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port_secure) تنظیمات. - - تنظیمات برای [SSL sertificates](server-configuration-parameters/settings.md#server_configuration_parameters-openssl). - - استفاده از پارامترهای مناسب در حالی که اتصال. برای مثال با استفاده از `port_secure` پارامتر با `clickhouse_client`. - -- تنظیمات کاربر. - - شما ممکن است با استفاده از نام کاربری اشتباه و یا رمز عبور. - -## پردازش پرس و جو {#troubleshooting-does-not-process-queries} - -اگر فاحشه خانه است که قادر به پردازش پرس و جو نمی, این شرح خطا به مشتری می فرستد. در `clickhouse-client` شما دریافت می کنید شرح خطا در کنسول. اگر شما با استفاده از HTTP رابط ClickHouse می فرستد خطا توضیحات در پاسخ بدن. به عنوان مثال: - -``` bash -$ curl 'http://localhost:8123/' --data-binary "SELECT a" -Code: 47, e.displayText() = DB::Exception: Unknown identifier: a. Note that there are no tables (FROM clause) in your query, context: required_names: 'a' source_tables: table_aliases: private_aliases: column_aliases: public_columns: 'a' masked_columns: array_join_columns: source_columns: , e.what() = DB::Exception -``` - -اگر شما شروع `clickhouse-client` با `stack-trace` پارامتر, خانه را برمی گرداند ردیابی پشته سرور با شرح خطا. - -شما ممکن است یک پیام در مورد یک اتصال شکسته را ببینید. در این مورد می توانید پرس و جو را تکرار کنید. اگر اتصال می شکند هر بار که شما انجام پرس و جو, بررسی سیاهههای مربوط به سرور برای اشتباهات. - -## کارایی پردازش پرس و جو {#troubleshooting-too-slow} - -اگر شما می بینید که تاتر در حال کار بیش از حد کند, شما نیاز به مشخصات بار بر روی منابع سرور و شبکه برای نمایش داده شد خود را. - -شما می توانید ابزار کلیک معیار به نمایش داده شد مشخصات استفاده کنید. این نشان می دهد تعداد نمایش داده شد پردازش در هر ثانیه, تعداد ردیف پردازش در هر ثانیه, و صدک از زمان پردازش پرس و جو. diff --git a/docs/fa/operations/update.md b/docs/fa/operations/update.md deleted file mode 100644 index f49a3d60444..00000000000 --- a/docs/fa/operations/update.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 47 -toc_title: "\u0628\u0647 \u0631\u0648\u0632 \u0631\u0633\u0627\u0646\u06CC \u06A9\u0644\ - \u06CC\u06A9" ---- - -# به روز رسانی کلیک {#clickhouse-update} - -اگر تاتر از بسته های دب نصب شد, اجرای دستورات زیر را بر روی سرور: - -``` bash -$ sudo apt-get update -$ sudo apt-get install clickhouse-client clickhouse-server -$ sudo service clickhouse-server restart -``` - -اگر شما نصب تاتر با استفاده از چیزی غیر از بسته های دب توصیه می شود, استفاده از روش به روز رسانی مناسب. - -کلیک می کند به روز رسانی توزیع را پشتیبانی نمی کند. این عملیات باید به صورت متوالی در هر سرور جداگانه انجام شود. هنوز تمام سرور بر روی یک خوشه به طور همزمان به روز رسانی نیست, یا خوشه برای برخی از زمان در دسترس نخواهد بود. diff --git a/docs/fa/operations/utilities/clickhouse-benchmark.md b/docs/fa/operations/utilities/clickhouse-benchmark.md deleted file mode 100644 index ca0be0f7860..00000000000 --- a/docs/fa/operations/utilities/clickhouse-benchmark.md +++ /dev/null @@ -1,156 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 61 -toc_title: "\u06A9\u0644\u06CC\u06A9-\u0645\u0639\u06CC\u0627\u0631" ---- - -# کلیک-معیار {#clickhouse-benchmark} - -قابلیت اتصال به یک سرور کلیک و بارها و بارها نمایش داده شد مشخص می فرستد. - -نحو: - -``` bash -$ echo "single query" | clickhouse-benchmark [keys] -``` - -یا - -``` bash -$ clickhouse-benchmark [keys] <<< "single query" -``` - -اگر شما می خواهید برای ارسال مجموعه ای از نمایش داده شد, ایجاد یک فایل متنی و قرار دادن هر پرس و جو در رشته های فردی در این فایل. به عنوان مثال: - -``` sql -SELECT * FROM system.numbers LIMIT 10000000 -SELECT 1 -``` - -سپس این فایل را به یک ورودی استاندارد منتقل می کند `clickhouse-benchmark`. - -``` bash -clickhouse-benchmark [keys] < queries_file -``` - -## کلید {#clickhouse-benchmark-keys} - -- `-c N`, `--concurrency=N` — Number of queries that `clickhouse-benchmark` می فرستد به طور همزمان. مقدار پیش فرض: 1. -- `-d N`, `--delay=N` — Interval in seconds between intermediate reports (set 0 to disable reports). Default value: 1. -- `-h WORD`, `--host=WORD` — Server host. Default value: `localhost`. برای [حالت مقایسه](#clickhouse-benchmark-comparison-mode) شما می توانید چند استفاده کنید `-h` کلیدا -- `-p N`, `--port=N` — Server port. Default value: 9000. For the [حالت مقایسه](#clickhouse-benchmark-comparison-mode) شما می توانید چند استفاده کنید `-p` کلیدا -- `-i N`, `--iterations=N` — Total number of queries. Default value: 0. -- `-r`, `--randomize` — Random order of queries execution if there is more then one input query. -- `-s`, `--secure` — Using TLS connection. -- `-t N`, `--timelimit=N` — Time limit in seconds. `clickhouse-benchmark` متوقف می شود ارسال نمایش داده شد زمانی که محدودیت زمانی مشخص رسیده است. مقدار پیش فرض: 0 (محدودیت زمانی غیر فعال). -- `--confidence=N` — Level of confidence for T-test. Possible values: 0 (80%), 1 (90%), 2 (95%), 3 (98%), 4 (99%), 5 (99.5%). Default value: 5. In the [حالت مقایسه](#clickhouse-benchmark-comparison-mode) `clickhouse-benchmark` انجام [تی تست دانشجویان مستقل دو نمونه](https://en.wikipedia.org/wiki/Student%27s_t-test#Independent_two-sample_t-test) تست برای تعیین اینکه دو توزیع با سطح انتخاب شده اعتماد به نفس متفاوت نیست. -- `--cumulative` — Printing cumulative data instead of data per interval. -- `--database=DATABASE_NAME` — ClickHouse database name. Default value: `default`. -- `--json=FILEPATH` — JSON output. When the key is set, `clickhouse-benchmark` خروجی یک گزارش به مشخص جانسون فایل. -- `--user=USERNAME` — ClickHouse user name. Default value: `default`. -- `--password=PSWD` — ClickHouse user password. Default value: empty string. -- `--stacktrace` — Stack traces output. When the key is set, `clickhouse-bencmark` خروجی پشته اثری از استثنا. -- `--stage=WORD` — Query processing stage at server. ClickHouse stops query processing and returns answer to `clickhouse-benchmark` در مرحله مشخص شده. مقادیر ممکن: `complete`, `fetch_columns`, `with_mergeable_state`. مقدار پیشفرض: `complete`. -- `--help` — Shows the help message. - -اگر شما می خواهید به درخواست برخی از [تنظیمات](../../operations/settings/index.md) برای پرس و جو, عبور خود را به عنوان یک کلید `--= SETTING_VALUE`. به عنوان مثال, `--max_memory_usage=1048576`. - -## خروجی {#clickhouse-benchmark-output} - -به طور پیش فرض, `clickhouse-benchmark` گزارش برای هر `--delay` فاصله. - -نمونه ای از گزارش: - -``` text -Queries executed: 10. - -localhost:9000, queries 10, QPS: 6.772, RPS: 67904487.440, MiB/s: 518.070, result RPS: 67721584.984, result MiB/s: 516.675. - -0.000% 0.145 sec. -10.000% 0.146 sec. -20.000% 0.146 sec. -30.000% 0.146 sec. -40.000% 0.147 sec. -50.000% 0.148 sec. -60.000% 0.148 sec. -70.000% 0.148 sec. -80.000% 0.149 sec. -90.000% 0.150 sec. -95.000% 0.150 sec. -99.000% 0.150 sec. -99.900% 0.150 sec. -99.990% 0.150 sec. -``` - -در این گزارش شما می توانید پیدا کنید: - -- تعداد نمایش داده شد در `Queries executed:` رشته. - -- رشته وضعیت حاوی) به ترتیب (): - - - نقطه پایانی از سرور کلیک. - - تعداد نمایش داده شد پردازش شده است. - - QPS: QPS: چگونه بسیاری از نمایش داده شد سرور انجام شده در هر ثانیه در طول یک دوره مشخص شده در `--delay` استدلال کردن. - - سرور مجازی: چند ردیف سرور در هر ثانیه در طول یک دوره مشخص شده در `--delay` استدلال کردن. - - مگابایت بر ثانیه: چگونه بسیاری از مگابایت سرور در ثانیه در طول یک دوره مشخص شده در خواندن `--delay` استدلال کردن. - - نتیجه ریسمانهای: چگونه بسیاری از ردیف توسط سرور به نتیجه یک پرس و جو در هر ثانیه در طول یک دوره مشخص شده در قرار داده شده `--delay` استدلال کردن. - - چگونه بسیاری از مگابایت توسط سرور به نتیجه یک پرس و جو در هر ثانیه در طول یک دوره مشخص شده در قرار داده شده `--delay` استدلال کردن. - -- صدک از نمایش داده شد زمان اجرای. - -## حالت مقایسه {#clickhouse-benchmark-comparison-mode} - -`clickhouse-benchmark` می توانید اجرای برای دو سرور در حال اجرا تاتر مقایسه. - -برای استفاده از حالت مقایسه, مشخص نقطه پایانی هر دو سرور توسط دو جفت از `--host`, `--port` کلیدا کلید با هم توسط موقعیت در لیست استدلال همسان, اولین `--host` با اولین همسان `--port` و به همین ترتیب. `clickhouse-benchmark` ایجاد ارتباط به هر دو سرور, سپس نمایش داده شد می فرستد. هر پرس و جو خطاب به یک سرور به طور تصادفی انتخاب شده است. نتایج برای هر سرور به طور جداگانه نشان داده شده است. - -## مثال {#clickhouse-benchmark-example} - -``` bash -$ echo "SELECT * FROM system.numbers LIMIT 10000000 OFFSET 10000000" | clickhouse-benchmark -i 10 -``` - -``` text -Loaded 1 queries. - -Queries executed: 6. - -localhost:9000, queries 6, QPS: 6.153, RPS: 123398340.957, MiB/s: 941.455, result RPS: 61532982.200, result MiB/s: 469.459. - -0.000% 0.159 sec. -10.000% 0.159 sec. -20.000% 0.159 sec. -30.000% 0.160 sec. -40.000% 0.160 sec. -50.000% 0.162 sec. -60.000% 0.164 sec. -70.000% 0.165 sec. -80.000% 0.166 sec. -90.000% 0.166 sec. -95.000% 0.167 sec. -99.000% 0.167 sec. -99.900% 0.167 sec. -99.990% 0.167 sec. - - - -Queries executed: 10. - -localhost:9000, queries 10, QPS: 6.082, RPS: 121959604.568, MiB/s: 930.478, result RPS: 60815551.642, result MiB/s: 463.986. - -0.000% 0.159 sec. -10.000% 0.159 sec. -20.000% 0.160 sec. -30.000% 0.163 sec. -40.000% 0.164 sec. -50.000% 0.165 sec. -60.000% 0.166 sec. -70.000% 0.166 sec. -80.000% 0.167 sec. -90.000% 0.167 sec. -95.000% 0.170 sec. -99.000% 0.172 sec. -99.900% 0.172 sec. -99.990% 0.172 sec. -``` diff --git a/docs/fa/operations/utilities/clickhouse-copier.md b/docs/fa/operations/utilities/clickhouse-copier.md deleted file mode 100644 index 698ed6565b0..00000000000 --- a/docs/fa/operations/utilities/clickhouse-copier.md +++ /dev/null @@ -1,176 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 59 -toc_title: "\u062A\u0627\u062A\u0631-\u06A9\u067E\u06CC" ---- - -# تاتر-کپی {#clickhouse-copier} - -کپی داده ها از جداول در یک خوشه به جداول در یکی دیگر از (یا همان) خوشه. - -شما می توانید چند اجرا `clickhouse-copier` نمونه بر روی سرور های مختلف برای انجام همان کار. باغ وحش برای همگام سازی فرایندها استفاده می شود. - -پس از شروع, `clickhouse-copier`: - -- قابلیت اتصال به باغ وحش و دریافت: - - - شغل کپی. - - دولت از مشاغل کپی. - -- این کار را انجام می دهد. - - هر فرایند در حال اجرا را انتخاب “closest” سفال از خوشه منبع و کپی داده ها را به خوشه مقصد, تغییر شکل داده ها در صورت لزوم. - -`clickhouse-copier` تغییرات باغ وحش را دنبال می کند و در پرواز اعمال می شود. - -برای کاهش ترافیک شبکه توصیه می کنیم در حال اجرا `clickhouse-copier` در همان سرور که داده های منبع واقع شده است. - -## در حال اجرا تاتر-کپی {#running-clickhouse-copier} - -ابزار باید به صورت دستی اجرا شود: - -``` bash -$ clickhouse-copier copier --daemon --config zookeeper.xml --task-path /task/path --base-dir /path/to/dir -``` - -پارامترها: - -- `daemon` — Starts `clickhouse-copier` در حالت شبح. -- `config` — The path to the `zookeeper.xml` فایل با پارامترهای اتصال به باغ وحش. -- `task-path` — The path to the ZooKeeper node. This node is used for syncing `clickhouse-copier` پردازش و ذخیره سازی وظایف. وظایف در ذخیره می شود `$task-path/description`. -- `task-file` — Optional path to file with task configuration for initial upload to ZooKeeper. -- `task-upload-force` — Force upload `task-file` حتی اگر گره در حال حاضر وجود دارد. -- `base-dir` — The path to logs and auxiliary files. When it starts, `clickhouse-copier` ایجاد `clickhouse-copier_YYYYMMHHSS_` زیرشاخه در `$base-dir`. اگر این پارامتر حذف شده است, دایرکتوری ها در دایرکتوری که ایجاد `clickhouse-copier` راه اندازی شد. - -## قالب باغ وحش.شمع {#format-of-zookeeper-xml} - -``` xml - - - trace - 100M - 3 - - - - - 127.0.0.1 - 2181 - - - -``` - -## پیکربندی وظایف کپی کردن {#configuration-of-copying-tasks} - -``` xml - - - - - - false - - 127.0.0.1 - 9000 - - - ... - - - - ... - - - - - 2 - - - - 1 - - - - - 0 - - - - - 3 - - 1 - - - - - - - - source_cluster - test - hits - - - destination_cluster - test - hits2 - - - - ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hits2', '{replica}') - PARTITION BY toMonday(date) - ORDER BY (CounterID, EventDate) - - - - jumpConsistentHash(intHash64(UserID), 2) - - - CounterID != 0 - - - - '2018-02-26' - '2018-03-05' - ... - - - - - - ... - - ... - - -``` - -`clickhouse-copier` پیگیری تغییرات در `/task/path/description` و اونا رو تو پرواز بکار میبره برای مثال, اگر شما ارزش تغییر `max_workers` تعداد فرایندهای در حال اجرا وظایف نیز تغییر خواهد کرد. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/utils/clickhouse-copier/) diff --git a/docs/fa/operations/utilities/clickhouse-local.md b/docs/fa/operations/utilities/clickhouse-local.md deleted file mode 100644 index ceea7471c2a..00000000000 --- a/docs/fa/operations/utilities/clickhouse-local.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 60 -toc_title: "\u06A9\u0644\u06CC\u06A9-\u0645\u062D\u0644\u06CC" ---- - -# کلیک-محلی {#clickhouse-local} - -این `clickhouse-local` برنامه شما را قادر به انجام پردازش سریع بر روی فایل های محلی, بدون نیاز به استقرار و پیکربندی سرور کلیک. - -داده هایی را می پذیرد که نشان دهنده جداول و نمایش داده شد با استفاده از [گویش کلیک مربعی](../../sql-reference/index.md). - -`clickhouse-local` بنابراین پشتیبانی از بسیاری از ویژگی های و همان مجموعه ای از فرمت ها و موتورهای جدول با استفاده از هسته همان سرور تاتر. - -به طور پیش فرض `clickhouse-local` دسترسی به داده ها در همان میزبان ندارد, اما پشتیبانی از پیکربندی سرور در حال بارگذاری با استفاده از `--config-file` استدلال کردن. - -!!! warning "اخطار" - توصیه نمی شود که پیکربندی سرور تولید را بارگیری کنید `clickhouse-local` زیرا داده ها می توانند در صورت خطای انسانی صدمه ببینند. - -## استفاده {#usage} - -استفاده عمومی: - -``` bash -$ clickhouse-local --structure "table_structure" --input-format "format_of_incoming_data" -q "query" -``` - -نشانوندها: - -- `-S`, `--structure` — table structure for input data. -- `-if`, `--input-format` — input format, `TSV` به طور پیش فرض. -- `-f`, `--file` — path to data, `stdin` به طور پیش فرض. -- `-q` `--query` — queries to execute with `;` به عنوان دسیمترتراپی. -- `-N`, `--table` — table name where to put output data, `table` به طور پیش فرض. -- `-of`, `--format`, `--output-format` — output format, `TSV` به طور پیش فرض. -- `--stacktrace` — whether to dump debug output in case of exception. -- `--verbose` — more details on query execution. -- `-s` — disables `stderr` ثبت. -- `--config-file` — path to configuration file in same format as for ClickHouse server, by default the configuration empty. -- `--help` — arguments references for `clickhouse-local`. - -همچنین استدلال برای هر متغیر پیکربندی کلیک که معمولا به جای استفاده می شود وجود دارد `--config-file`. - -## مثالها {#examples} - -``` bash -$ echo -e "1,2\n3,4" | clickhouse-local -S "a Int64, b Int64" -if "CSV" -q "SELECT * FROM table" -Read 2 rows, 32.00 B in 0.000 sec., 5182 rows/sec., 80.97 KiB/sec. -1 2 -3 4 -``` - -مثال قبلی همان است: - -``` bash -$ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64) ENGINE = File(CSV, stdin); SELECT a, b FROM table; DROP TABLE table" -Read 2 rows, 32.00 B in 0.000 sec., 4987 rows/sec., 77.93 KiB/sec. -1 2 -3 4 -``` - -حالا اجازه دهید خروجی کاربر حافظه برای هر کاربر یونیکس: - -``` bash -$ ps aux | tail -n +2 | awk '{ printf("%s\t%s\n", $1, $4) }' | clickhouse-local -S "user String, mem Float64" -q "SELECT user, round(sum(mem), 2) as memTotal FROM table GROUP BY user ORDER BY memTotal DESC FORMAT Pretty" -``` - -``` text -Read 186 rows, 4.15 KiB in 0.035 sec., 5302 rows/sec., 118.34 KiB/sec. -┏━━━━━━━━━━┳━━━━━━━━━━┓ -┃ user ┃ memTotal ┃ -┡━━━━━━━━━━╇━━━━━━━━━━┩ -│ bayonet │ 113.5 │ -├──────────┼──────────┤ -│ root │ 8.8 │ -├──────────┼──────────┤ -... -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/utils/clickhouse-local/) diff --git a/docs/fa/operations/utilities/index.md b/docs/fa/operations/utilities/index.md deleted file mode 100644 index ae0f1f724d8..00000000000 --- a/docs/fa/operations/utilities/index.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u062A\u0627\u0633\u06CC\u0633\u0627\u062A" -toc_priority: 56 -toc_title: "\u0628\u0631\u0631\u0633\u06CC \u0627\u062C\u0645\u0627\u0644\u06CC" ---- - -# ابزار کلیک {#clickhouse-utility} - -- [کلیک-محلی](clickhouse-local.md) — Allows running SQL queries on data without stopping the ClickHouse server, similar to how `awk` این کار را می کند. -- [تاتر-کپی](clickhouse-copier.md) — Copies (and reshards) data from one cluster to another cluster. -- [کلیک-معیار](clickhouse-benchmark.md) — Loads server with the custom queries and settings. - -[مقاله اصلی](https://clickhouse.tech/docs/en/operations/utils/) diff --git a/docs/fa/sql-reference/aggregate-functions/combinators.md b/docs/fa/sql-reference/aggregate-functions/combinators.md deleted file mode 100644 index af9361c5478..00000000000 --- a/docs/fa/sql-reference/aggregate-functions/combinators.md +++ /dev/null @@ -1,245 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 37 -toc_title: "\u062A\u0631\u06A9\u06CC\u0628 \u06A9\u0646\u0646\u062F\u0647\u0647\u0627" ---- - -# ترکیب کنندههای تابع جمع {#aggregate_functions_combinators} - -نام یک تابع جمع می تواند یک پسوند اضافه شده است. این تغییر راه تابع کلی کار می کند. - -## - اگر {#agg-functions-combinator-if} - -The suffix -If can be appended to the name of any aggregate function. In this case, the aggregate function accepts an extra argument – a condition (Uint8 type). The aggregate function processes only the rows that trigger the condition. If the condition was not triggered even once, it returns a default value (usually zeros or empty strings). - -مثالها: `sumIf(column, cond)`, `countIf(cond)`, `avgIf(x, cond)`, `quantilesTimingIf(level1, level2)(x, cond)`, `argMinIf(arg, val, cond)` و به همین ترتیب. - -با توابع مجموع شرطی, شما می توانید مصالح برای چندین شرایط در یک بار محاسبه, بدون استفاده از کارخانه های فرعی و `JOIN`برای مثال در یاندکس.متریکا, توابع مجموع مشروط استفاده می شود برای پیاده سازی قابلیت مقایسه بخش. - -## حداقل صفحه نمایش: {#agg-functions-combinator-array} - -پسوند مجموعه را می توان به هر تابع جمع اضافه شده است. در این مورد, تابع کل استدلال از طول می کشد ‘Array(T)’ نوع (ارریس) به جای ‘T’ استدلال نوع. اگر تابع جمع استدلال های متعدد را می پذیرد, این باید مجموعه ای از طول های برابر شود. هنگامی که پردازش ارریس, تابع کل کار می کند مانند تابع کل اصلی در تمام عناصر مجموعه. - -مثال 1: `sumArray(arr)` - مجموع تمام عناصر از همه ‘arr’ اراریس در این مثال می توانست بیشتر به سادگی نوشته شده است: `sum(arraySum(arr))`. - -مثال 2: `uniqArray(arr)` – Counts the number of unique elements in all ‘arr’ اراریس این می تواند انجام شود یک راه ساده تر: `uniq(arrayJoin(arr))` اما همیشه امکان اضافه کردن وجود ندارد ‘arrayJoin’ به پرس و جو. - -\- اگر و مجموعه ای می تواند ترکیب شود. هرچند, ‘Array’ پس اول باید بیای ‘If’. مثالها: `uniqArrayIf(arr, cond)`, `quantilesTimingArrayIf(level1, level2)(arr, cond)`. با توجه به این سفارش ‘cond’ برهان صف نیست. - -## - وضعیت {#agg-functions-combinator-state} - -اگر شما درخواست این ترکیب, تابع کل می کند مقدار حاصل بازگشت نیست (مانند تعدادی از ارزش های منحصر به فرد برای [دانشگاه](reference.md#agg_function-uniq) تابع) , اما یک دولت متوسط از تجمع (برای `uniq`, این جدول هش برای محاسبه تعداد ارزش های منحصر به فرد است). این یک `AggregateFunction(...)` که می تواند برای پردازش بیشتر استفاده می شود و یا ذخیره شده در یک جدول را به پایان برساند جمع بعد. - -برای کار با این کشورها, استفاده: - -- [ریزدانه](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) موتور جدول. -- [پلاکتی](../../sql-reference/functions/other-functions.md#function-finalizeaggregation) تابع. -- [خرابی اجرا](../../sql-reference/functions/other-functions.md#function-runningaccumulate) تابع. -- [- ادغام](#aggregate_functions_combinators-merge) ترکیب کننده. -- [اطلاعات دقیق](#aggregate_functions_combinators-mergestate) ترکیب کننده. - -## - ادغام {#aggregate_functions_combinators-merge} - -اگر شما درخواست این ترکیب, تابع کل طول می کشد حالت تجمع متوسط به عنوان یک استدلال, ترکیبی از کشورهای به پایان تجمع, و ارزش حاصل می گرداند. - -## اطلاعات دقیق {#aggregate_functions_combinators-mergestate} - -ادغام کشورهای تجمع متوسط در همان راه به عنوان ترکیب-ادغام. با این حال, این مقدار حاصل بازگشت نیست, اما یک دولت تجمع متوسط, شبیه به ترکیب دولت. - -## - فورچ {#agg-functions-combinator-foreach} - -تبدیل یک تابع جمع برای جداول به یک تابع کلی برای ارریس که جمع اقلام مجموعه مربوطه و مجموعه ای از نتایج را برمی گرداند. به عنوان مثال, `sumForEach` برای ارریس `[1, 2]`, `[3, 4, 5]`و`[6, 7]`نتیجه را برمی گرداند `[10, 13, 5]` پس از اضافه کردن با هم موارد مجموعه مربوطه. - -## شناسه بسته: {#agg-functions-combinator-ordefault} - -تغییر رفتار یک تابع جمع. - -اگر یک تابع جمع می کند ارزش های ورودی ندارد, با این ترکیب این مقدار پیش فرض برای نوع داده بازگشت خود را برمی گرداند. شامل توابع کل است که می تواند داده های ورودی خالی را. - -`-OrDefault` می توان با سایر ترکیب کننده ها استفاده کرد. - -**نحو** - -``` sql -OrDefault(x) -``` - -**پارامترها** - -- `x` — Aggregate function parameters. - -**مقادیر بازگشتی** - -بازگرداندن مقدار پیش فرض از نوع بازگشت یک تابع جمع است اگر چیزی برای جمع وجود دارد. - -نوع بستگی به عملکرد کلی مورد استفاده دارد. - -**مثال** - -پرسوجو: - -``` sql -SELECT avg(number), avgOrDefault(number) FROM numbers(0) -``` - -نتیجه: - -``` text -┌─avg(number)─┬─avgOrDefault(number)─┐ -│ nan │ 0 │ -└─────────────┴──────────────────────┘ -``` - -همچنین `-OrDefault` می توان با یک ترکیب کننده دیگر استفاده کرد. این زمانی مفید است که تابع جمع می کند ورودی خالی را قبول نمی کند. - -پرسوجو: - -``` sql -SELECT avgOrDefaultIf(x, x > 10) -FROM -( - SELECT toDecimal32(1.23, 2) AS x -) -``` - -نتیجه: - -``` text -┌─avgOrDefaultIf(x, greater(x, 10))─┐ -│ 0.00 │ -└───────────────────────────────────┘ -``` - -## اطلاعات دقیق {#agg-functions-combinator-ornull} - -تغییر رفتار یک تابع جمع. - -این ترکیب تبدیل یک نتیجه از یک تابع جمع به [Nullable](../data-types/nullable.md) نوع داده. اگر تابع جمع می کند ارزش برای محاسبه بازده ندارد [NULL](../syntax.md#null-literal). - -`-OrNull` می توان با سایر ترکیب کننده ها استفاده کرد. - -**نحو** - -``` sql -OrNull(x) -``` - -**پارامترها** - -- `x` — Aggregate function parameters. - -**مقادیر بازگشتی** - -- نتیجه عملکرد کل, تبدیل به `Nullable` نوع داده. -- `NULL`, اگر چیزی برای جمع وجود دارد. - -نوع: `Nullable(aggregate function return type)`. - -**مثال** - -افزودن `-orNull` به پایان تابع جمع. - -پرسوجو: - -``` sql -SELECT sumOrNull(number), toTypeName(sumOrNull(number)) FROM numbers(10) WHERE number > 10 -``` - -نتیجه: - -``` text -┌─sumOrNull(number)─┬─toTypeName(sumOrNull(number))─┐ -│ ᴺᵁᴸᴸ │ Nullable(UInt64) │ -└───────────────────┴───────────────────────────────┘ -``` - -همچنین `-OrNull` می توان با یک ترکیب کننده دیگر استفاده کرد. این زمانی مفید است که تابع جمع می کند ورودی خالی را قبول نمی کند. - -پرسوجو: - -``` sql -SELECT avgOrNullIf(x, x > 10) -FROM -( - SELECT toDecimal32(1.23, 2) AS x -) -``` - -نتیجه: - -``` text -┌─avgOrNullIf(x, greater(x, 10))─┐ -│ ᴺᵁᴸᴸ │ -└────────────────────────────────┘ -``` - -## - نمونه {#agg-functions-combinator-resample} - -به شما امکان می دهد داده ها را به گروه تقسیم کنید و سپس به طور جداگانه داده ها را در این گروه ها جمع کنید. گروه ها با تقسیم مقادیر از یک ستون به فواصل ایجاد شده است. - -``` sql -Resample(start, end, step)(, resampling_key) -``` - -**پارامترها** - -- `start` — Starting value of the whole required interval for `resampling_key` ارزشهای خبری عبارتند از: -- `stop` — Ending value of the whole required interval for `resampling_key` ارزشهای خبری عبارتند از: کل فاصله شامل نمی شود `stop` مقدار `[start, stop)`. -- `step` — Step for separating the whole interval into subintervals. The `aggFunction` بیش از هر یک از این زیرگروه اعدام به طور مستقل. -- `resampling_key` — Column whose values are used for separating data into intervals. -- `aggFunction_params` — `aggFunction` پارامترها - -**مقادیر بازگشتی** - -- مجموعه ای از `aggFunction` نتایج برای هر زیر خدمات. - -**مثال** - -در نظر بگیرید که `people` جدول با داده های زیر: - -``` text -┌─name───┬─age─┬─wage─┐ -│ John │ 16 │ 10 │ -│ Alice │ 30 │ 15 │ -│ Mary │ 35 │ 8 │ -│ Evelyn │ 48 │ 11.5 │ -│ David │ 62 │ 9.9 │ -│ Brian │ 60 │ 16 │ -└────────┴─────┴──────┘ -``` - -بیایید نام افرادی که سن نهفته در فواصل `[30,60)` و `[60,75)`. پس ما با استفاده از نمایندگی عدد صحیح برای سن, ما سنین در `[30, 59]` و `[60,74]` فواصل زمانی. - -به نام کلی در مجموعه, ما با استفاده از [گرامری](reference.md#agg_function-grouparray) تابع جمع. طول می کشد تا یک استدلال. در مورد ما این است `name` ستون. این `groupArrayResample` تابع باید از `age` ستون به نام دانه های سن. برای تعریف فواصل مورد نیاز ما `30, 75, 30` نشانوندها به `groupArrayResample` تابع. - -``` sql -SELECT groupArrayResample(30, 75, 30)(name, age) FROM people -``` - -``` text -┌─groupArrayResample(30, 75, 30)(name, age)─────┐ -│ [['Alice','Mary','Evelyn'],['David','Brian']] │ -└───────────────────────────────────────────────┘ -``` - -نتایج را در نظر بگیرید. - -`Jonh` خارج از نمونه است چرا که او بیش از حد جوان است. افراد دیگر با توجه به فواصل زمانی مشخص شده توزیع می شوند. - -حالا اجازه دهید تعداد کل مردم و متوسط دستمزد خود را در فواصل سنی مشخص شده است. - -``` sql -SELECT - countResample(30, 75, 30)(name, age) AS amount, - avgResample(30, 75, 30)(wage, age) AS avg_wage -FROM people -``` - -``` text -┌─amount─┬─avg_wage──────────────────┐ -│ [3,2] │ [11.5,12.949999809265137] │ -└────────┴───────────────────────────┘ -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/query_language/agg_functions/combinators/) diff --git a/docs/fa/sql-reference/aggregate-functions/index.md b/docs/fa/sql-reference/aggregate-functions/index.md deleted file mode 100644 index 11d23a0c04d..00000000000 --- a/docs/fa/sql-reference/aggregate-functions/index.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u062A\u0648\u0627\u0628\u0639 \u0645\u062C\u0645\u0648\u0639" -toc_priority: 33 -toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" ---- - -# توابع مجموع {#aggregate-functions} - -توابع مجموع در کار [عادی](http://www.sql-tutorial.com/sql-aggregate-functions-sql-tutorial) راه به عنوان کارشناسان پایگاه داده انتظار می رود. - -فاحشه خانه نیز پشتیبانی می کند: - -- [توابع مجموع پارامتری](parametric-functions.md#aggregate_functions_parametric), که قبول پارامترهای دیگر علاوه بر ستون. -- [ترکیب کنندهها](combinators.md#aggregate_functions_combinators) که تغییر رفتار مجموع توابع. - -## پردازش پوچ {#null-processing} - -در طول تجمع همه `NULL`بازدید کنندگان قلم می. - -**مثالها:** - -این جدول را در نظر بگیرید: - -``` text -┌─x─┬────y─┐ -│ 1 │ 2 │ -│ 2 │ ᴺᵁᴸᴸ │ -│ 3 │ 2 │ -│ 3 │ 3 │ -│ 3 │ ᴺᵁᴸᴸ │ -└───┴──────┘ -``` - -بیایید می گویند شما نیاز به کل ارزش ها در `y` ستون: - -``` sql -SELECT sum(y) FROM t_null_big -``` - - ┌─sum(y)─┐ - │ 7 │ - └────────┘ - -این `sum` تابع تفسیر می کند `NULL` به عنوان `0`. به خصوص, این بدان معنی است که اگر تابع ورودی از یک انتخاب که تمام مقادیر دریافت `NULL` سپس نتیجه خواهد بود `0` نه `NULL`. - -حالا شما می توانید استفاده کنید `groupArray` تابع برای ایجاد مجموعه ای از `y` ستون: - -``` sql -SELECT groupArray(y) FROM t_null_big -``` - -``` text -┌─groupArray(y)─┐ -│ [2,2,3] │ -└───────────────┘ -``` - -`groupArray` شامل نمی شود `NULL` در مجموعه ای نتیجه. - -[مقاله اصلی](https://clickhouse.tech/docs/en/query_language/agg_functions/) diff --git a/docs/fa/sql-reference/aggregate-functions/parametric-functions.md b/docs/fa/sql-reference/aggregate-functions/parametric-functions.md deleted file mode 100644 index c034720e485..00000000000 --- a/docs/fa/sql-reference/aggregate-functions/parametric-functions.md +++ /dev/null @@ -1,499 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 38 -toc_title: "\u067E\u0627\u0631\u0627\u0645\u062A\u0631\u06CC" ---- - -# توابع مجموع پارامتری {#aggregate_functions_parametric} - -Some aggregate functions can accept not only argument columns (used for compression), but a set of parameters – constants for initialization. The syntax is two pairs of brackets instead of one. The first is for parameters, and the second is for arguments. - -## سابقهنما {#histogram} - -محاسبه هیستوگرام تطبیقی. این نتایج دقیق را تضمین نمی کند. - -``` sql -histogram(number_of_bins)(values) -``` - -توابع استفاده می کند [جریان الگوریتم درخت تصمیم موازی](http://jmlr.org/papers/volume11/ben-haim10a/ben-haim10a.pdf). مرزهای سطل هیستوگرام تنظیم به عنوان داده های جدید وارد یک تابع. در مورد مشترک عرض سطل برابر نیست. - -**پارامترها** - -`number_of_bins` — Upper limit for the number of bins in the histogram. The function automatically calculates the number of bins. It tries to reach the specified number of bins, but if it fails, it uses fewer bins. -`values` — [عبارت](../syntax.md#syntax-expressions) در نتیجه مقادیر ورودی. - -**مقادیر بازگشتی** - -- [& حذف](../../sql-reference/data-types/array.md) از [توپلس](../../sql-reference/data-types/tuple.md) از قالب زیر: - - ``` - [(lower_1, upper_1, height_1), ... (lower_N, upper_N, height_N)] - ``` - - - `lower` — Lower bound of the bin. - - `upper` — Upper bound of the bin. - - `height` — Calculated height of the bin. - -**مثال** - -``` sql -SELECT histogram(5)(number + 1) -FROM ( - SELECT * - FROM system.numbers - LIMIT 20 -) -``` - -``` text -┌─histogram(5)(plus(number, 1))───────────────────────────────────────────┐ -│ [(1,4.5,4),(4.5,8.5,4),(8.5,12.75,4.125),(12.75,17,4.625),(17,20,3.25)] │ -└─────────────────────────────────────────────────────────────────────────┘ -``` - -شما می توانید یک هیستوگرام با تجسم [بار](../../sql-reference/functions/other-functions.md#function-bar) تابع, مثلا: - -``` sql -WITH histogram(5)(rand() % 100) AS hist -SELECT - arrayJoin(hist).3 AS height, - bar(height, 0, 6, 5) AS bar -FROM -( - SELECT * - FROM system.numbers - LIMIT 20 -) -``` - -``` text -┌─height─┬─bar───┐ -│ 2.125 │ █▋ │ -│ 3.25 │ ██▌ │ -│ 5.625 │ ████▏ │ -│ 5.625 │ ████▏ │ -│ 3.375 │ ██▌ │ -└────────┴───────┘ -``` - -در این مورد, شما باید به یاد داشته باشید که شما مرزهای هیستوگرام بن نمی دانند. - -## sequenceMatch(pattern)(timestamp, cond1, cond2, …) {#function-sequencematch} - -بررسی اینکه دنباله شامل یک زنجیره رویداد که منطبق بر الگوی. - -``` sql -sequenceMatch(pattern)(timestamp, cond1, cond2, ...) -``` - -!!! warning "اخطار" - رویدادهایی که در همان دوم رخ می دهد ممکن است در دنباله در سفارش تعریف نشده موثر بر نتیجه دراز. - -**پارامترها** - -- `pattern` — Pattern string. See [نحو الگو](#sequence-function-pattern-syntax). - -- `timestamp` — Column considered to contain time data. Typical data types are `Date` و `DateTime`. شما همچنین می توانید هر یک از پشتیبانی استفاده کنید [اینترنت](../../sql-reference/data-types/int-uint.md) انواع داده ها. - -- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. شما می توانید به تصویب تا 32 استدلال شرط. تابع طول می کشد تنها حوادث شرح داده شده در این شرایط به حساب. اگر دنباله حاوی اطلاعاتی است که در شرایط توصیف نشده, تابع پرش. - -**مقادیر بازگشتی** - -- 1, اگر الگوی همسان است. -- 0, اگر الگوی همسان نیست. - -نوع: `UInt8`. - - -**نحو الگو** - -- `(?N)` — Matches the condition argument at position `N`. شرایط در شماره `[1, 32]` محدوده. به عنوان مثال, `(?1)` با استدلال به تصویب رسید `cond1` پارامتر. - -- `.*` — Matches any number of events. You don't need conditional arguments to match this element of the pattern. - -- `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` مسابقات رویدادهایی که رخ می دهد بیش از 1800 ثانیه از یکدیگر. تعداد دلخواه از هر رویدادی می تواند بین این حوادث دراز. شما می توانید از `>=`, `>`, `<`, `<=` اپراتورها. - -**مثالها** - -داده ها را در نظر بگیرید `t` جدول: - -``` text -┌─time─┬─number─┐ -│ 1 │ 1 │ -│ 2 │ 3 │ -│ 3 │ 2 │ -└──────┴────────┘ -``` - -انجام پرس و جو: - -``` sql -SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2) FROM t -``` - -``` text -┌─sequenceMatch('(?1)(?2)')(time, equals(number, 1), equals(number, 2))─┐ -│ 1 │ -└───────────────────────────────────────────────────────────────────────┘ -``` - -تابع زنجیره رویداد که تعداد پیدا شده است 2 زیر شماره 1. این قلم شماره 3 بین, چرا که تعداد به عنوان یک رویداد توصیف نشده. اگر ما می خواهیم این شماره را در نظر بگیریم هنگام جستجو برای زنجیره رویداد داده شده در مثال باید شرایط را ایجاد کنیم. - -``` sql -SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 3) FROM t -``` - -``` text -┌─sequenceMatch('(?1)(?2)')(time, equals(number, 1), equals(number, 2), equals(number, 3))─┐ -│ 0 │ -└──────────────────────────────────────────────────────────────────────────────────────────┘ -``` - -در این مورد, تابع می تواند زنجیره رویداد تطبیق الگوی پیدا کنید, چرا که این رویداد برای شماره 3 رخ داده است بین 1 و 2. اگر در همان مورد ما شرایط را برای شماره بررسی 4, دنباله الگوی مطابقت. - -``` sql -SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM t -``` - -``` text -┌─sequenceMatch('(?1)(?2)')(time, equals(number, 1), equals(number, 2), equals(number, 4))─┐ -│ 1 │ -└──────────────────────────────────────────────────────────────────────────────────────────┘ -``` - -**همچنین نگاه کنید به** - -- [شمارش معکوس](#function-sequencecount) - -## sequenceCount(pattern)(time, cond1, cond2, …) {#function-sequencecount} - -شمارش تعداد زنجیره رویداد که الگوی همسان. تابع جستجو زنجیره رویداد که با هم همپوشانی دارند. این شروع به جستجو برای زنجیره بعدی پس از زنجیره فعلی همسان است. - -!!! warning "اخطار" - رویدادهایی که در همان دوم رخ می دهد ممکن است در دنباله در سفارش تعریف نشده موثر بر نتیجه دراز. - -``` sql -sequenceCount(pattern)(timestamp, cond1, cond2, ...) -``` - -**پارامترها** - -- `pattern` — Pattern string. See [نحو الگو](#sequence-function-pattern-syntax). - -- `timestamp` — Column considered to contain time data. Typical data types are `Date` و `DateTime`. شما همچنین می توانید هر یک از پشتیبانی استفاده کنید [اینترنت](../../sql-reference/data-types/int-uint.md) انواع داده ها. - -- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. شما می توانید به تصویب تا 32 استدلال شرط. تابع طول می کشد تنها حوادث شرح داده شده در این شرایط به حساب. اگر دنباله حاوی اطلاعاتی است که در شرایط توصیف نشده, تابع پرش. - -**مقادیر بازگشتی** - -- تعداد زنجیره رویداد غیر با هم تداخل دارند که همسان. - -نوع: `UInt64`. - -**مثال** - -داده ها را در نظر بگیرید `t` جدول: - -``` text -┌─time─┬─number─┐ -│ 1 │ 1 │ -│ 2 │ 3 │ -│ 3 │ 2 │ -│ 4 │ 1 │ -│ 5 │ 3 │ -│ 6 │ 2 │ -└──────┴────────┘ -``` - -تعداد چند بار تعداد 2 پس از شماره 1 با هر مقدار از شماره های دیگر بین رخ می دهد: - -``` sql -SELECT sequenceCount('(?1).*(?2)')(time, number = 1, number = 2) FROM t -``` - -``` text -┌─sequenceCount('(?1).*(?2)')(time, equals(number, 1), equals(number, 2))─┐ -│ 2 │ -└─────────────────────────────────────────────────────────────────────────┘ -``` - -**همچنین نگاه کنید به** - -- [ترتیب سنج](#function-sequencematch) - -## در پنجره {#windowfunnel} - -جستجو برای زنجیره رویداد در یک پنجره زمان کشویی و محاسبه حداکثر تعداد رویدادهایی که از زنجیره رخ داده است. - -تابع با توجه به الگوریتم کار می کند: - -- تابع جستجو برای داده هایی که باعث شرط اول در زنجیره و مجموعه ضد رویداد به 1. این لحظه ای است که پنجره کشویی شروع می شود. - -- اگر حوادث از زنجیره پی در پی در پنجره رخ می دهد, ضد افزایش است. اگر دنباله ای از حوادث مختل شده است, شمارنده است افزایش نمی. - -- اگر داده های زنجیره رویداد های متعدد در نقاط مختلف از اتمام, تابع تنها خروجی به اندازه طولانی ترین زنجیره ای. - -**نحو** - -``` sql -windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) -``` - -**پارامترها** - -- `window` — Length of the sliding window in seconds. -- `mode` - این یک استدلال اختیاری است . - - `'strict'` - وقتی که `'strict'` تنظیم شده است, پنجره() اعمال شرایط تنها برای ارزش های منحصر به فرد. -- `timestamp` — Name of the column containing the timestamp. Data types supported: [تاریخ](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) و دیگر انواع عدد صحیح بدون علامت (توجه داشته باشید که حتی اگر برچسب زمان پشتیبانی از `UInt64` نوع, این مقدار می تواند بین المللی تجاوز نمی64 بیشترین, که 2^63 - 1). -- `cond` — Conditions or data describing the chain of events. [UInt8](../../sql-reference/data-types/int-uint.md). - -**مقدار بازگشتی** - -حداکثر تعداد متوالی باعث شرایط از زنجیره ای در پنجره زمان کشویی. -تمام زنجیره ها در انتخاب تجزیه و تحلیل می شوند. - -نوع: `Integer`. - -**مثال** - -تعیین کنید که یک دوره زمانی معین برای کاربر کافی باشد تا گوشی را انتخاب کند و دو بار در فروشگاه اینترنتی خریداری کند. - -زنجیره ای از وقایع زیر را تنظیم کنید: - -1. کاربر وارد شده به حساب خود را در فروشگاه (`eventID = 1003`). -2. کاربر برای یک تلفن جستجو می کند (`eventID = 1007, product = 'phone'`). -3. کاربر سفارش داده شده (`eventID = 1009`). -4. کاربر دوباره سفارش داد (`eventID = 1010`). - -جدول ورودی: - -``` text -┌─event_date─┬─user_id─┬───────────timestamp─┬─eventID─┬─product─┐ -│ 2019-01-28 │ 1 │ 2019-01-29 10:00:00 │ 1003 │ phone │ -└────────────┴─────────┴─────────────────────┴─────────┴─────────┘ -┌─event_date─┬─user_id─┬───────────timestamp─┬─eventID─┬─product─┐ -│ 2019-01-31 │ 1 │ 2019-01-31 09:00:00 │ 1007 │ phone │ -└────────────┴─────────┴─────────────────────┴─────────┴─────────┘ -┌─event_date─┬─user_id─┬───────────timestamp─┬─eventID─┬─product─┐ -│ 2019-01-30 │ 1 │ 2019-01-30 08:00:00 │ 1009 │ phone │ -└────────────┴─────────┴─────────────────────┴─────────┴─────────┘ -┌─event_date─┬─user_id─┬───────────timestamp─┬─eventID─┬─product─┐ -│ 2019-02-01 │ 1 │ 2019-02-01 08:00:00 │ 1010 │ phone │ -└────────────┴─────────┴─────────────────────┴─────────┴─────────┘ -``` - -یافتن پست های تا چه حد کاربر `user_id` می تواند از طریق زنجیره ای در یک دوره در ژانویه و فوریه از 2019. - -پرسوجو: - -``` sql -SELECT - level, - count() AS c -FROM -( - SELECT - user_id, - windowFunnel(6048000000000000)(timestamp, eventID = 1003, eventID = 1009, eventID = 1007, eventID = 1010) AS level - FROM trend - WHERE (event_date >= '2019-01-01') AND (event_date <= '2019-02-02') - GROUP BY user_id -) -GROUP BY level -ORDER BY level ASC -``` - -نتیجه: - -``` text -┌─level─┬─c─┐ -│ 4 │ 1 │ -└───────┴───┘ -``` - -## نگهداری {#retention} - -تابع طول می کشد به عنوان استدلال مجموعه ای از شرایط از 1 به 32 استدلال از نوع `UInt8` که نشان می دهد که یک بیماری خاص برای این رویداد مواجه شد. -هر گونه شرایط را می توان به عنوان یک استدلال مشخص (همانطور که در [WHERE](../../sql-reference/statements/select/where.md#select-where)). - -شرایط, به جز اولین, درخواست در جفت: نتیجه دوم درست خواهد بود اگر اول و دوم درست باشد, از سوم اگر اولین و فیرد درست باشد, و غیره. - -**نحو** - -``` sql -retention(cond1, cond2, ..., cond32); -``` - -**پارامترها** - -- `cond` — an expression that returns a `UInt8` نتیجه (1 یا 0). - -**مقدار بازگشتی** - -مجموعه ای از 1 یا 0. - -- 1 — condition was met for the event. -- 0 — condition wasn't met for the event. - -نوع: `UInt8`. - -**مثال** - -بیایید یک نمونه از محاسبه را در نظر بگیریم `retention` تابع برای تعیین ترافیک سایت. - -**1.** Сreate a table to illustrate an example. - -``` sql -CREATE TABLE retention_test(date Date, uid Int32) ENGINE = Memory; - -INSERT INTO retention_test SELECT '2020-01-01', number FROM numbers(5); -INSERT INTO retention_test SELECT '2020-01-02', number FROM numbers(10); -INSERT INTO retention_test SELECT '2020-01-03', number FROM numbers(15); -``` - -جدول ورودی: - -پرسوجو: - -``` sql -SELECT * FROM retention_test -``` - -نتیجه: - -``` text -┌───────date─┬─uid─┐ -│ 2020-01-01 │ 0 │ -│ 2020-01-01 │ 1 │ -│ 2020-01-01 │ 2 │ -│ 2020-01-01 │ 3 │ -│ 2020-01-01 │ 4 │ -└────────────┴─────┘ -┌───────date─┬─uid─┐ -│ 2020-01-02 │ 0 │ -│ 2020-01-02 │ 1 │ -│ 2020-01-02 │ 2 │ -│ 2020-01-02 │ 3 │ -│ 2020-01-02 │ 4 │ -│ 2020-01-02 │ 5 │ -│ 2020-01-02 │ 6 │ -│ 2020-01-02 │ 7 │ -│ 2020-01-02 │ 8 │ -│ 2020-01-02 │ 9 │ -└────────────┴─────┘ -┌───────date─┬─uid─┐ -│ 2020-01-03 │ 0 │ -│ 2020-01-03 │ 1 │ -│ 2020-01-03 │ 2 │ -│ 2020-01-03 │ 3 │ -│ 2020-01-03 │ 4 │ -│ 2020-01-03 │ 5 │ -│ 2020-01-03 │ 6 │ -│ 2020-01-03 │ 7 │ -│ 2020-01-03 │ 8 │ -│ 2020-01-03 │ 9 │ -│ 2020-01-03 │ 10 │ -│ 2020-01-03 │ 11 │ -│ 2020-01-03 │ 12 │ -│ 2020-01-03 │ 13 │ -│ 2020-01-03 │ 14 │ -└────────────┴─────┘ -``` - -**2.** کاربران گروه با شناسه منحصر به فرد `uid` با استفاده از `retention` تابع. - -پرسوجو: - -``` sql -SELECT - uid, - retention(date = '2020-01-01', date = '2020-01-02', date = '2020-01-03') AS r -FROM retention_test -WHERE date IN ('2020-01-01', '2020-01-02', '2020-01-03') -GROUP BY uid -ORDER BY uid ASC -``` - -نتیجه: - -``` text -┌─uid─┬─r───────┐ -│ 0 │ [1,1,1] │ -│ 1 │ [1,1,1] │ -│ 2 │ [1,1,1] │ -│ 3 │ [1,1,1] │ -│ 4 │ [1,1,1] │ -│ 5 │ [0,0,0] │ -│ 6 │ [0,0,0] │ -│ 7 │ [0,0,0] │ -│ 8 │ [0,0,0] │ -│ 9 │ [0,0,0] │ -│ 10 │ [0,0,0] │ -│ 11 │ [0,0,0] │ -│ 12 │ [0,0,0] │ -│ 13 │ [0,0,0] │ -│ 14 │ [0,0,0] │ -└─────┴─────────┘ -``` - -**3.** محاسبه تعداد کل بازدیدکننده داشته است سایت در هر روز. - -پرسوجو: - -``` sql -SELECT - sum(r[1]) AS r1, - sum(r[2]) AS r2, - sum(r[3]) AS r3 -FROM -( - SELECT - uid, - retention(date = '2020-01-01', date = '2020-01-02', date = '2020-01-03') AS r - FROM retention_test - WHERE date IN ('2020-01-01', '2020-01-02', '2020-01-03') - GROUP BY uid -) -``` - -نتیجه: - -``` text -┌─r1─┬─r2─┬─r3─┐ -│ 5 │ 5 │ 5 │ -└────┴────┴────┘ -``` - -کجا: - -- `r1`- تعداد بازدید کنندگان منحصر به فرد که در طول 2020-01-01 (بازدید `cond1` شرط). -- `r2`- تعداد بازدید کنندگان منحصر به فرد که برای بازدید از سایت در طول یک دوره زمانی خاص بین 2020-01-01 و 2020-01-02 (`cond1` و `cond2` شرایط). -- `r3`- تعداد بازدید کنندگان منحصر به فرد که برای بازدید از سایت در طول یک دوره زمانی خاص بین 2020-01-01 و 2020-01-03 (`cond1` و `cond3` شرایط). - -## uniqUpTo(N)(x) {#uniquptonx} - -Calculates the number of different argument values ​​if it is less than or equal to N. If the number of different argument values is greater than N, it returns N + 1. - -توصیه می شود برای استفاده با شماره های کوچک, تا 10. حداکثر مقدار نفر است 100. - -برای دولت از یک تابع جمع, با استفاده از مقدار حافظه برابر با 1 + نفر \* اندازه یک مقدار بایت. -برای رشته, این فروشگاه یک هش غیر رمزنگاری 8 بایت. به این معنا که محاسبه برای رشته ها تقریبی است. - -این تابع همچنین برای چندین استدلال کار می کند. - -این کار به همان سرعتی که ممکن است, به جز برای موارد زمانی که یک مقدار نفر بزرگ استفاده می شود و تعدادی از ارزش های منحصر به فرد است کمی کمتر از ان. - -مثال طریقه استفاده: - -``` text -Problem: Generate a report that shows only keywords that produced at least 5 unique users. -Solution: Write in the GROUP BY query SearchPhrase HAVING uniqUpTo(4)(UserID) >= 5 -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/query_language/agg_functions/parametric_functions/) - -## sumMapFiltered(keys_to_keep)(کلید ارزش ها) {#summapfilteredkeys-to-keepkeys-values} - -رفتار مشابه [& سواپ](reference.md#agg_functions-summap) جز این که مجموعه ای از کلید به عنوان یک پارامتر منتقل می شود. این می تواند مفید باشد به خصوص در هنگام کار با یک کارت از کلید های بالا. diff --git a/docs/fa/sql-reference/aggregate-functions/reference.md b/docs/fa/sql-reference/aggregate-functions/reference.md deleted file mode 100644 index f18dc9e6bf6..00000000000 --- a/docs/fa/sql-reference/aggregate-functions/reference.md +++ /dev/null @@ -1,1914 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 36 -toc_title: "\u0645\u0631\u062C\u0639" ---- - -# مرجع عملکرد کامل {#aggregate-functions-reference} - -## شمارش {#agg_function-count} - -شمارش تعداد ردیف یا نه تهی ارزش. - -ClickHouse زیر پشتیبانی می کند syntaxes برای `count`: -- `count(expr)` یا `COUNT(DISTINCT expr)`. -- `count()` یا `COUNT(*)`. این `count()` نحو تاتر خاص است. - -**پارامترها** - -این تابع می تواند: - -- صفر پارامتر. -- یک [عبارت](../syntax.md#syntax-expressions). - -**مقدار بازگشتی** - -- اگر تابع بدون پارامتر نامیده می شود تعداد ردیف شمارش. -- اگر [عبارت](../syntax.md#syntax-expressions) به تصویب می رسد, سپس تابع شمارش چند بار این عبارت بازگشت تهی نیست. اگر بیان می گرداند [Nullable](../../sql-reference/data-types/nullable.md)- نوع ارزش و سپس نتیجه `count` باقی نمی ماند `Nullable`. تابع بازده 0 اگر بیان بازگشت `NULL` برای تمام ردیف. - -در هر دو مورد نوع مقدار بازگشتی است [UInt64](../../sql-reference/data-types/int-uint.md). - -**اطلاعات دقیق** - -تاتر از `COUNT(DISTINCT ...)` نحو. رفتار این ساخت و ساز بستگی به [ا_فزونهها](../../operations/settings/settings.md#settings-count_distinct_implementation) تنظیمات. این تعریف می کند که کدام یک از [دانشگاه\*](#agg_function-uniq) توابع برای انجام عملیات استفاده می شود. به طور پیش فرض است [قرارداد اتحادیه](#agg_function-uniqexact) تابع. - -این `SELECT count() FROM table` پرس و جو بهینه سازی شده نیست, چرا که تعداد ورودی در جدول به طور جداگانه ذخیره نمی. این ستون کوچک را از جدول انتخاب می کند و تعداد مقادیر موجود را شمارش می کند. - -**مثالها** - -مثال 1: - -``` sql -SELECT count() FROM t -``` - -``` text -┌─count()─┐ -│ 5 │ -└─────────┘ -``` - -مثال 2: - -``` sql -SELECT name, value FROM system.settings WHERE name = 'count_distinct_implementation' -``` - -``` text -┌─name──────────────────────────┬─value─────┐ -│ count_distinct_implementation │ uniqExact │ -└───────────────────────────────┴───────────┘ -``` - -``` sql -SELECT count(DISTINCT num) FROM t -``` - -``` text -┌─uniqExact(num)─┐ -│ 3 │ -└────────────────┘ -``` - -این مثال نشان می دهد که `count(DISTINCT num)` توسط `uniqExact` عملکرد با توجه به `count_distinct_implementation` مقدار تنظیم. - -## هر) {#agg_function-any} - -انتخاب اولین مقدار مواجه می شوند. -پرس و جو را می توان در هر سفارش و حتی در جهت های مختلف در هر زمان اجرا, بنابراین نتیجه این تابع نامشخص است. -برای دریافت یک نتیجه معین, شما می توانید با استفاده از ‘min’ یا ‘max’ تابع به جای ‘any’. - -در بعضی موارد, شما می توانید در جهت اعدام تکیه. این امر در مورد مواردی که انتخاب می شود از یک زیرخاکی است که از سفارش استفاده می کند. - -هنگامی که یک `SELECT` پرسوجو دارد `GROUP BY` بند و یا حداقل یک مجموع عملکرد ClickHouse (در مقایسه با MySQL) مستلزم آن است که تمام عبارات در `SELECT`, `HAVING` و `ORDER BY` بند از کلید و یا از توابع کل محاسبه می شود. به عبارت دیگر, هر ستون انتخاب شده از جدول باید یا در کلید و یا در داخل توابع دانه استفاده می شود. برای دریافت رفتار مانند خروجی زیر, شما می توانید ستون های دیگر در قرار `any` تابع جمع. - -## هشدار داده می شود) {#anyheavyx} - -انتخاب یک مقدار اغلب اتفاق می افتد با استفاده از [بزرگان سنگین](http://www.cs.umd.edu/~samir/498/karp.pdf) الگوریتم. در صورتی که یک مقدار که بیش از در نیمی از موارد در هر یک از موضوعات اعدام پرس و جو رخ می دهد وجود دارد, این مقدار بازگشته است. به طور معمول نتیجه nondeterministic. - -``` sql -anyHeavy(column) -``` - -**نشانوندها** - -- `column` – The column name. - -**مثال** - -نگاهی به [به موقع](../../getting-started/example-datasets/ontime.md) مجموعه داده ها و انتخاب هر مقدار اغلب اتفاق می افتد در `AirlineID` ستون. - -``` sql -SELECT anyHeavy(AirlineID) AS res -FROM ontime -``` - -``` text -┌───res─┐ -│ 19690 │ -└───────┘ -``` - -## حداقل صفحه نمایش:) {#anylastx} - -ارزش گذشته مواجه می شوند را انتخاب می کند. -نتیجه این است که فقط به عنوان نامشخص به عنوان برای `any` تابع. - -## گروه بیتاند {#groupbitand} - -اعمال بیتی `AND` برای مجموعه ای از اعداد. - -``` sql -groupBitAnd(expr) -``` - -**پارامترها** - -`expr` – An expression that results in `UInt*` نوع. - -**مقدار بازگشتی** - -ارزش `UInt*` نوع. - -**مثال** - -داده های تست: - -``` text -binary decimal -00101100 = 44 -00011100 = 28 -00001101 = 13 -01010101 = 85 -``` - -پرسوجو: - -``` sql -SELECT groupBitAnd(num) FROM t -``` - -کجا `num` ستون با داده های تست است. - -نتیجه: - -``` text -binary decimal -00000100 = 4 -``` - -## ویرایشگر گروه {#groupbitor} - -اعمال بیتی `OR` برای مجموعه ای از اعداد. - -``` sql -groupBitOr(expr) -``` - -**پارامترها** - -`expr` – An expression that results in `UInt*` نوع. - -**مقدار بازگشتی** - -ارزش `UInt*` نوع. - -**مثال** - -داده های تست: - -``` text -binary decimal -00101100 = 44 -00011100 = 28 -00001101 = 13 -01010101 = 85 -``` - -پرسوجو: - -``` sql -SELECT groupBitOr(num) FROM t -``` - -کجا `num` ستون با داده های تست است. - -نتیجه: - -``` text -binary decimal -01111101 = 125 -``` - -## گروهبیتکسور {#groupbitxor} - -اعمال بیتی `XOR` برای مجموعه ای از اعداد. - -``` sql -groupBitXor(expr) -``` - -**پارامترها** - -`expr` – An expression that results in `UInt*` نوع. - -**مقدار بازگشتی** - -ارزش `UInt*` نوع. - -**مثال** - -داده های تست: - -``` text -binary decimal -00101100 = 44 -00011100 = 28 -00001101 = 13 -01010101 = 85 -``` - -پرسوجو: - -``` sql -SELECT groupBitXor(num) FROM t -``` - -کجا `num` ستون با داده های تست است. - -نتیجه: - -``` text -binary decimal -01101000 = 104 -``` - -## نگاشت گروهی {#groupbitmap} - -بیت مپ و یا کل محاسبات از یک unsigned integer ستون بازگشت cardinality از نوع UInt64 اگر اضافه کردن پسوند -دولت بازگشت [شی نگاشت بیت](../../sql-reference/functions/bitmap-functions.md). - -``` sql -groupBitmap(expr) -``` - -**پارامترها** - -`expr` – An expression that results in `UInt*` نوع. - -**مقدار بازگشتی** - -ارزش `UInt64` نوع. - -**مثال** - -داده های تست: - -``` text -UserID -1 -1 -2 -3 -``` - -پرسوجو: - -``` sql -SELECT groupBitmap(UserID) as num FROM t -``` - -نتیجه: - -``` text -num -3 -``` - -## کمینه) {#agg_function-min} - -محاسبه حداقل. - -## بیشینه) {#agg_function-max} - -محاسبه حداکثر. - -## هشدار داده می شود) {#agg-function-argmin} - -محاسبه ‘arg’ ارزش برای حداقل ‘val’ ارزش. اگر چندین مقدار مختلف وجود دارد ‘arg’ برای مقادیر حداقل ‘val’ اولین بار از این مقادیر مواجه خروجی است. - -**مثال:** - -``` text -┌─user─────┬─salary─┐ -│ director │ 5000 │ -│ manager │ 3000 │ -│ worker │ 1000 │ -└──────────┴────────┘ -``` - -``` sql -SELECT argMin(user, salary) FROM salary -``` - -``` text -┌─argMin(user, salary)─┐ -│ worker │ -└──────────────────────┘ -``` - -## هشدار داده می شود) {#agg-function-argmax} - -محاسبه ‘arg’ مقدار برای حداکثر ‘val’ ارزش. اگر چندین مقدار مختلف وجود دارد ‘arg’ برای حداکثر مقادیر ‘val’ اولین بار از این مقادیر مواجه خروجی است. - -## جمع) {#agg_function-sum} - -محاسبه مجموع. -فقط برای اعداد کار می کند. - -## ورود به سیستم) {#sumwithoverflowx} - -محاسبه مجموع اعداد, با استفاده از همان نوع داده برای نتیجه به عنوان پارامترهای ورودی. اگر مجموع بیش از حداکثر مقدار برای این نوع داده, تابع یک خطا می گرداند. - -فقط برای اعداد کار می کند. - -## sumMap(key, value), sumMap(تاپل(key, value)) {#agg_functions-summap} - -مجموع ‘value’ تنظیم با توجه به کلید های مشخص شده در ‘key’ صف کردن. -عبور تاپل از کلید ها و ارزش های عرریس مترادف به عبور از دو مجموعه از کلید ها و ارزش است. -تعداد عناصر در ‘key’ و ‘value’ باید همین کار را برای هر سطر است که بالغ بر شود. -Returns a tuple of two arrays: keys in sorted order, and values ​​summed for the corresponding keys. - -مثال: - -``` sql -CREATE TABLE sum_map( - date Date, - timeslot DateTime, - statusMap Nested( - status UInt16, - requests UInt64 - ), - statusMapTuple Tuple(Array(Int32), Array(Int32)) -) ENGINE = Log; -INSERT INTO sum_map VALUES - ('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10], ([1, 2, 3], [10, 10, 10])), - ('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10], ([3, 4, 5], [10, 10, 10])), - ('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10], ([4, 5, 6], [10, 10, 10])), - ('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10], ([6, 7, 8], [10, 10, 10])); - -SELECT - timeslot, - sumMap(statusMap.status, statusMap.requests), - sumMap(statusMapTuple) -FROM sum_map -GROUP BY timeslot -``` - -``` text -┌────────────timeslot─┬─sumMap(statusMap.status, statusMap.requests)─┬─sumMap(statusMapTuple)─────────┐ -│ 2000-01-01 00:00:00 │ ([1,2,3,4,5],[10,10,20,10,10]) │ ([1,2,3,4,5],[10,10,20,10,10]) │ -│ 2000-01-01 00:01:00 │ ([4,5,6,7,8],[10,10,20,10,10]) │ ([4,5,6,7,8],[10,10,20,10,10]) │ -└─────────────────────┴──────────────────────────────────────────────┴────────────────────────────────┘ -``` - -## سیخ کباب {#skewpop} - -محاسبه [skewness](https://en.wikipedia.org/wiki/Skewness) از یک توالی. - -``` sql -skewPop(expr) -``` - -**پارامترها** - -`expr` — [عبارت](../syntax.md#syntax-expressions) بازگشت یک عدد. - -**مقدار بازگشتی** - -The skewness of the given distribution. Type — [جسم شناور64](../../sql-reference/data-types/float.md) - -**مثال** - -``` sql -SELECT skewPop(value) FROM series_with_value_column -``` - -## سیخ {#skewsamp} - -محاسبه [نمونه skewness](https://en.wikipedia.org/wiki/Skewness) از یک توالی. - -این نشان دهنده یک تخمین بی طرفانه از اریب یک متغیر تصادفی اگر ارزش گذشت نمونه خود را تشکیل می دهند. - -``` sql -skewSamp(expr) -``` - -**پارامترها** - -`expr` — [عبارت](../syntax.md#syntax-expressions) بازگشت یک عدد. - -**مقدار بازگشتی** - -The skewness of the given distribution. Type — [جسم شناور64](../../sql-reference/data-types/float.md). اگر `n <= 1` (`n` اندازه نمونه است), سپس بازده تابع `nan`. - -**مثال** - -``` sql -SELECT skewSamp(value) FROM series_with_value_column -``` - -## کورتپ {#kurtpop} - -محاسبه [kurtosis](https://en.wikipedia.org/wiki/Kurtosis) از یک توالی. - -``` sql -kurtPop(expr) -``` - -**پارامترها** - -`expr` — [عبارت](../syntax.md#syntax-expressions) بازگشت یک عدد. - -**مقدار بازگشتی** - -The kurtosis of the given distribution. Type — [جسم شناور64](../../sql-reference/data-types/float.md) - -**مثال** - -``` sql -SELECT kurtPop(value) FROM series_with_value_column -``` - -## کردها {#kurtsamp} - -محاسبه [نمونه kurtosis](https://en.wikipedia.org/wiki/Kurtosis) از یک توالی. - -این نشان دهنده یک تخمین بی طرفانه از کورتوز یک متغیر تصادفی اگر ارزش گذشت نمونه خود را تشکیل می دهند. - -``` sql -kurtSamp(expr) -``` - -**پارامترها** - -`expr` — [عبارت](../syntax.md#syntax-expressions) بازگشت یک عدد. - -**مقدار بازگشتی** - -The kurtosis of the given distribution. Type — [جسم شناور64](../../sql-reference/data-types/float.md). اگر `n <= 1` (`n` اندازه نمونه است) و سپس تابع بازده `nan`. - -**مثال** - -``` sql -SELECT kurtSamp(value) FROM series_with_value_column -``` - -## میانگین) {#agg_function-avg} - -محاسبه متوسط. -فقط برای اعداد کار می کند. -نتیجه این است که همیشه شناور64. - -## کشتی کج {#avgweighted} - -محاسبه [میانگین ریاضی وزنی](https://en.wikipedia.org/wiki/Weighted_arithmetic_mean). - -**نحو** - -``` sql -avgWeighted(x, weight) -``` - -**پارامترها** - -- `x` — Values. [عدد صحیح](../data-types/int-uint.md) یا [شناور نقطه](../data-types/float.md). -- `weight` — Weights of the values. [عدد صحیح](../data-types/int-uint.md) یا [شناور نقطه](../data-types/float.md). - -نوع `x` و `weight` باید مثل قبل باشه - -**مقدار بازگشتی** - -- وزن متوسط. -- `NaN`. اگر تمام وزن به برابر هستند 0. - -نوع: [جسم شناور64](../data-types/float.md). - -**مثال** - -پرسوجو: - -``` sql -SELECT avgWeighted(x, w) -FROM values('x Int8, w Int8', (4, 1), (1, 0), (10, 2)) -``` - -نتیجه: - -``` text -┌─avgWeighted(x, weight)─┐ -│ 8 │ -└────────────────────────┘ -``` - -## دانشگاه {#agg_function-uniq} - -محاسبه تعداد تقریبی مقادیر مختلف استدلال. - -``` sql -uniq(x[, ...]) -``` - -**پارامترها** - -تابع طول می کشد تعداد متغیر از پارامترهای. پارامترها می توانند باشند `Tuple`, `Array`, `Date`, `DateTime`, `String`, یا انواع عددی. - -**مقدار بازگشتی** - -- A [UInt64](../../sql-reference/data-types/int-uint.md)- نوع شماره . - -**پیاده سازی اطلاعات** - -تابع: - -- هش را برای تمام پارامترها در مجموع محاسبه می کند و سپس در محاسبات استفاده می شود. - -- با استفاده از یک الگوریتم نمونه تطبیقی. برای محاسبه دولت تابع با استفاده از یک نمونه از عناصر هش ارزش تا 65536. - - This algorithm is very accurate and very efficient on the CPU. When the query contains several of these functions, using `uniq` is almost as fast as using other aggregate functions. - -- نتیجه را تعیین می کند (به سفارش پردازش پرس و جو بستگی ندارد). - -ما توصیه می کنیم با استفاده از این تابع تقریبا در تمام حالات. - -**همچنین نگاه کنید به** - -- [مخلوط نشده](#agg_function-uniqcombined) -- [نیم قرن 64](#agg_function-uniqcombined64) -- [یونقلل12](#agg_function-uniqhll12) -- [قرارداد اتحادیه](#agg_function-uniqexact) - -## مخلوط نشده {#agg_function-uniqcombined} - -محاسبه تعداد تقریبی مقادیر استدلال های مختلف. - -``` sql -uniqCombined(HLL_precision)(x[, ...]) -``` - -این `uniqCombined` تابع یک انتخاب خوب برای محاسبه تعداد مقادیر مختلف است. - -**پارامترها** - -تابع طول می کشد تعداد متغیر از پارامترهای. پارامترها می توانند باشند `Tuple`, `Array`, `Date`, `DateTime`, `String`, یا انواع عددی. - -`HLL_precision` پایه 2 لگاریتم تعداد سلول ها در است [جمع شدن](https://en.wikipedia.org/wiki/HyperLogLog). اختیاری, شما می توانید تابع به عنوان استفاده `uniqCombined(x[, ...])`. مقدار پیش فرض برای `HLL_precision` است 17, که به طور موثر 96 کیلوبایت فضا (2^17 سلول ها, 6 بیت در هر). - -**مقدار بازگشتی** - -- یک عدد [UInt64](../../sql-reference/data-types/int-uint.md)- نوع شماره . - -**پیاده سازی اطلاعات** - -تابع: - -- محاسبه هش (هش 64 بیتی برای `String` و در غیر این صورت 32 بیتی) برای تمام پارامترها در مجموع و سپس در محاسبات استفاده می شود. - -- با استفاده از ترکیبی از سه الگوریتم: مجموعه, جدول هش, و جمع شدن با جدول تصحیح خطا. - - For a small number of distinct elements, an array is used. When the set size is larger, a hash table is used. For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory. - -- نتیجه را تعیین می کند (به سفارش پردازش پرس و جو بستگی ندارد). - -!!! note "یادداشت" - از هش 32 بیتی برای غیر استفاده می کند-`String` نوع, نتیجه خطا بسیار بالا برای کاریت به طور قابل توجهی بزرگتر از اند `UINT_MAX` (خطا به سرعت پس از چند ده میلیارد ارزش متمایز افزایش خواهد یافت), از این رو در این مورد شما باید استفاده کنید [نیم قرن 64](#agg_function-uniqcombined64) - -در مقایسه با [دانشگاه](#agg_function-uniq) عملکرد `uniqCombined`: - -- مصرف چندین بار حافظه کمتر. -- محاسبه با دقت چند بار بالاتر است. -- معمولا عملکرد کمی پایین تر است. در برخی از حالات, `uniqCombined` می توانید بهتر از انجام `uniq` برای مثال با توزیع نمایش داده شد که انتقال تعداد زیادی از جمع متحده بر روی شبکه. - -**همچنین نگاه کنید به** - -- [دانشگاه](#agg_function-uniq) -- [نیم قرن 64](#agg_function-uniqcombined64) -- [یونقلل12](#agg_function-uniqhll12) -- [قرارداد اتحادیه](#agg_function-uniqexact) - -## نیم قرن 64 {#agg_function-uniqcombined64} - -مثل [مخلوط نشده](#agg_function-uniqcombined), اما با استفاده از هش 64 بیتی برای تمام انواع داده ها. - -## یونقلل12 {#agg_function-uniqhll12} - -محاسبه تعداد تقریبی مقادیر استدلال های مختلف, با استفاده از [جمع شدن](https://en.wikipedia.org/wiki/HyperLogLog) الگوریتم. - -``` sql -uniqHLL12(x[, ...]) -``` - -**پارامترها** - -تابع طول می کشد تعداد متغیر از پارامترهای. پارامترها می توانند باشند `Tuple`, `Array`, `Date`, `DateTime`, `String`, یا انواع عددی. - -**مقدار بازگشتی** - -- A [UInt64](../../sql-reference/data-types/int-uint.md)- نوع شماره . - -**پیاده سازی اطلاعات** - -تابع: - -- هش را برای تمام پارامترها در مجموع محاسبه می کند و سپس در محاسبات استفاده می شود. - -- با استفاده از الگوریتم جمع شدن تقریبی تعداد مقادیر استدلال های مختلف. - - 212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. The result is not very accurate (up to ~10% error) for small data sets (<10K elements). However, the result is fairly accurate for high-cardinality data sets (10K-100M), with a maximum error of ~1.6%. Starting from 100M, the estimation error increases, and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements). - -- نتیجه تعیین شده را فراهم می کند (به سفارش پردازش پرس و جو بستگی ندارد). - -ما توصیه نمی کنیم با استفاده از این تابع. در اغلب موارد از [دانشگاه](#agg_function-uniq) یا [مخلوط نشده](#agg_function-uniqcombined) تابع. - -**همچنین نگاه کنید به** - -- [دانشگاه](#agg_function-uniq) -- [مخلوط نشده](#agg_function-uniqcombined) -- [قرارداد اتحادیه](#agg_function-uniqexact) - -## قرارداد اتحادیه {#agg_function-uniqexact} - -محاسبه تعداد دقیق ارزش استدلال های مختلف. - -``` sql -uniqExact(x[, ...]) -``` - -استفاده از `uniqExact` تابع اگر شما کاملا نیاز به یک نتیجه دقیق. در غیر این صورت استفاده از [دانشگاه](#agg_function-uniq) تابع. - -این `uniqExact` تابع با استفاده از حافظه بیش از `uniq`, چرا که اندازه دولت رشد گشوده است به عنوان تعدادی از ارزش های مختلف را افزایش می دهد. - -**پارامترها** - -تابع طول می کشد تعداد متغیر از پارامترهای. پارامترها می توانند باشند `Tuple`, `Array`, `Date`, `DateTime`, `String`, یا انواع عددی. - -**همچنین نگاه کنید به** - -- [دانشگاه](#agg_function-uniq) -- [مخلوط نشده](#agg_function-uniqcombined) -- [یونقلل12](#agg_function-uniqhll12) - -## groupArray(x) groupArray(max_size)(x) {#agg_function-grouparray} - -مجموعه ای از مقادیر استدلال را ایجاد می کند. -مقادیر را می توان به ترتیب در هر (نامعین) اضافه کرد. - -نسخه دوم (با `max_size` پارامتر) اندازه مجموعه حاصل را محدود می کند `max_size` عناصر. -به عنوان مثال, `groupArray (1) (x)` معادل است `[any (x)]`. - -در بعضی موارد, شما هنوز هم می توانید در جهت اعدام تکیه. این امر در مورد مواردی که `SELECT` همراه از یک خرده فروشی که با استفاده از `ORDER BY`. - -## هشدار داده می شود {#grouparrayinsertat} - -مقدار را به مجموعه ای در موقعیت مشخص شده وارد می کند. - -**نحو** - -``` sql -groupArrayInsertAt(default_x, size)(x, pos); -``` - -اگر در یک پرس و جو چند مقدار به همان موقعیت قرار داده, تابع رفتار در روش های زیر: - -- اگر پرس و جو در یک موضوع اجرا, یکی از اولین از مقادیر درج شده استفاده شده است. -- اگر یک پرس و جو در موضوعات مختلف اجرا, ارزش حاصل یکی نامشخص از مقادیر درج شده است. - -**پارامترها** - -- `x` — Value to be inserted. [عبارت](../syntax.md#syntax-expressions) در نتیجه یکی از [انواع داده های پشتیبانی شده](../../sql-reference/data-types/index.md). -- `pos` — Position at which the specified element `x` قرار داده می شود. شماره شاخص در مجموعه از صفر شروع می شود. [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). -- `default_x`— Default value for substituting in empty positions. Optional parameter. [عبارت](../syntax.md#syntax-expressions) در نتیجه نوع داده پیکربندی شده برای `x` پارامتر. اگر `default_x` تعریف نشده است [مقادیر پیشفرض](../../sql-reference/statements/create.md#create-default-values) استفاده می شود. -- `size`— Length of the resulting array. Optional parameter. When using this parameter, the default value `default_x` باید مشخص شود. [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). - -**مقدار بازگشتی** - -- مجموعه ای با مقادیر درج شده. - -نوع: [& حذف](../../sql-reference/data-types/array.md#data-type-array). - -**مثال** - -پرسوجو: - -``` sql -SELECT groupArrayInsertAt(toString(number), number * 2) FROM numbers(5); -``` - -نتیجه: - -``` text -┌─groupArrayInsertAt(toString(number), multiply(number, 2))─┐ -│ ['0','','1','','2','','3','','4'] │ -└───────────────────────────────────────────────────────────┘ -``` - -پرسوجو: - -``` sql -SELECT groupArrayInsertAt('-')(toString(number), number * 2) FROM numbers(5); -``` - -نتیجه: - -``` text -┌─groupArrayInsertAt('-')(toString(number), multiply(number, 2))─┐ -│ ['0','-','1','-','2','-','3','-','4'] │ -└────────────────────────────────────────────────────────────────┘ -``` - -پرسوجو: - -``` sql -SELECT groupArrayInsertAt('-', 5)(toString(number), number * 2) FROM numbers(5); -``` - -نتیجه: - -``` text -┌─groupArrayInsertAt('-', 5)(toString(number), multiply(number, 2))─┐ -│ ['0','-','1','-','2'] │ -└───────────────────────────────────────────────────────────────────┘ -``` - -درج چند رشته ای از عناصر را به یک موقعیت. - -پرسوجو: - -``` sql -SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; -``` - -به عنوان یک نتیجه از این پرس و جو شما عدد صحیح تصادفی در `[0,9]` محدوده. به عنوان مثال: - -``` text -┌─groupArrayInsertAt(number, 0)─┐ -│ [7] │ -└───────────────────────────────┘ -``` - -## هشدار داده می شود {#agg_function-grouparraymovingsum} - -محاسبه مجموع در حال حرکت از ارزش های ورودی. - -``` sql -groupArrayMovingSum(numbers_for_summing) -groupArrayMovingSum(window_size)(numbers_for_summing) -``` - -این تابع می تواند اندازه پنجره به عنوان یک پارامتر را. اگر سمت چپ نامشخص, تابع طول می کشد اندازه پنجره به تعداد ردیف در ستون برابر. - -**پارامترها** - -- `numbers_for_summing` — [عبارت](../syntax.md#syntax-expressions) در نتیجه یک مقدار نوع داده عددی. -- `window_size` — Size of the calculation window. - -**مقادیر بازگشتی** - -- مجموعه ای از همان اندازه و نوع به عنوان داده های ورودی. - -**مثال** - -جدول نمونه: - -``` sql -CREATE TABLE t -( - `int` UInt8, - `float` Float32, - `dec` Decimal32(2) -) -ENGINE = TinyLog -``` - -``` text -┌─int─┬─float─┬──dec─┐ -│ 1 │ 1.1 │ 1.10 │ -│ 2 │ 2.2 │ 2.20 │ -│ 4 │ 4.4 │ 4.40 │ -│ 7 │ 7.77 │ 7.77 │ -└─────┴───────┴──────┘ -``` - -نمایش داده شد: - -``` sql -SELECT - groupArrayMovingSum(int) AS I, - groupArrayMovingSum(float) AS F, - groupArrayMovingSum(dec) AS D -FROM t -``` - -``` text -┌─I──────────┬─F───────────────────────────────┬─D──────────────────────┐ -│ [1,3,7,14] │ [1.1,3.3000002,7.7000003,15.47] │ [1.10,3.30,7.70,15.47] │ -└────────────┴─────────────────────────────────┴────────────────────────┘ -``` - -``` sql -SELECT - groupArrayMovingSum(2)(int) AS I, - groupArrayMovingSum(2)(float) AS F, - groupArrayMovingSum(2)(dec) AS D -FROM t -``` - -``` text -┌─I──────────┬─F───────────────────────────────┬─D──────────────────────┐ -│ [1,3,6,11] │ [1.1,3.3000002,6.6000004,12.17] │ [1.10,3.30,6.60,12.17] │ -└────────────┴─────────────────────────────────┴────────────────────────┘ -``` - -## گروهاریموینگاوگ {#agg_function-grouparraymovingavg} - -محاسبه میانگین متحرک از ارزش های ورودی. - -``` sql -groupArrayMovingAvg(numbers_for_summing) -groupArrayMovingAvg(window_size)(numbers_for_summing) -``` - -این تابع می تواند اندازه پنجره به عنوان یک پارامتر را. اگر سمت چپ نامشخص, تابع طول می کشد اندازه پنجره به تعداد ردیف در ستون برابر. - -**پارامترها** - -- `numbers_for_summing` — [عبارت](../syntax.md#syntax-expressions) در نتیجه یک مقدار نوع داده عددی. -- `window_size` — Size of the calculation window. - -**مقادیر بازگشتی** - -- مجموعه ای از همان اندازه و نوع به عنوان داده های ورودی. - -تابع استفاده می کند [گرد کردن به سمت صفر](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero). این کوتاه رقم اعشار ناچیز برای نوع داده و در نتیجه. - -**مثال** - -جدول نمونه `b`: - -``` sql -CREATE TABLE t -( - `int` UInt8, - `float` Float32, - `dec` Decimal32(2) -) -ENGINE = TinyLog -``` - -``` text -┌─int─┬─float─┬──dec─┐ -│ 1 │ 1.1 │ 1.10 │ -│ 2 │ 2.2 │ 2.20 │ -│ 4 │ 4.4 │ 4.40 │ -│ 7 │ 7.77 │ 7.77 │ -└─────┴───────┴──────┘ -``` - -نمایش داده شد: - -``` sql -SELECT - groupArrayMovingAvg(int) AS I, - groupArrayMovingAvg(float) AS F, - groupArrayMovingAvg(dec) AS D -FROM t -``` - -``` text -┌─I─────────┬─F───────────────────────────────────┬─D─────────────────────┐ -│ [0,0,1,3] │ [0.275,0.82500005,1.9250001,3.8675] │ [0.27,0.82,1.92,3.86] │ -└───────────┴─────────────────────────────────────┴───────────────────────┘ -``` - -``` sql -SELECT - groupArrayMovingAvg(2)(int) AS I, - groupArrayMovingAvg(2)(float) AS F, - groupArrayMovingAvg(2)(dec) AS D -FROM t -``` - -``` text -┌─I─────────┬─F────────────────────────────────┬─D─────────────────────┐ -│ [0,1,3,5] │ [0.55,1.6500001,3.3000002,6.085] │ [0.55,1.65,3.30,6.08] │ -└───────────┴──────────────────────────────────┴───────────────────────┘ -``` - -## groupUniqArray(x) groupUniqArray(max_size)(x) {#groupuniqarrayx-groupuniqarraymax-sizex} - -مجموعه ای از مقادیر مختلف استدلال ایجاد می کند. مصرف حافظه همان است که برای `uniqExact` تابع. - -نسخه دوم (با `max_size` پارامتر) اندازه مجموعه حاصل را محدود می کند `max_size` عناصر. -به عنوان مثال, `groupUniqArray(1)(x)` معادل است `[any(x)]`. - -## quantile {#quantile} - -محاسبه تقریبی [quantile](https://en.wikipedia.org/wiki/Quantile) از یک توالی داده های عددی. - -این تابع اعمال می شود [نمونه برداری مخزن](https://en.wikipedia.org/wiki/Reservoir_sampling) با اندازه مخزن تا 8192 و یک مولد عدد تصادفی برای نمونه برداری. نتیجه غیر قطعی است. برای دریافت یک کمی دقیق, استفاده از [کوانتوم](#quantileexact) تابع. - -هنگام استفاده از چندین `quantile*` توابع با سطوح مختلف در پرس و جو, کشورهای داخلی در ترکیب نیست (به این معنا که, پرس و جو کار می کند موثر کمتر از می تواند). در این مورد از [quantiles](#quantiles) تابع. - -**نحو** - -``` sql -quantile(level)(expr) -``` - -نام مستعار: `median`. - -**پارامترها** - -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` مقدار در محدوده `[0.01, 0.99]`. مقدار پیش فرض: 0.5. در `level=0.5` تابع محاسبه می کند [میانه](https://en.wikipedia.org/wiki/Median). -- `expr` — Expression over the column values resulting in numeric [انواع داده ها](../../sql-reference/data-types/index.md#data_types), [تاریخ](../../sql-reference/data-types/date.md) یا [DateTime](../../sql-reference/data-types/datetime.md). - -**مقدار بازگشتی** - -- کمی تقریبی از سطح مشخص شده است. - -نوع: - -- [جسم شناور64](../../sql-reference/data-types/float.md) برای ورودی نوع داده عددی. -- [تاریخ](../../sql-reference/data-types/date.md) اگر مقادیر ورودی `Date` نوع. -- [DateTime](../../sql-reference/data-types/datetime.md) اگر مقادیر ورودی `DateTime` نوع. - -**مثال** - -جدول ورودی: - -``` text -┌─val─┐ -│ 1 │ -│ 1 │ -│ 2 │ -│ 3 │ -└─────┘ -``` - -پرسوجو: - -``` sql -SELECT quantile(val) FROM t -``` - -نتیجه: - -``` text -┌─quantile(val)─┐ -│ 1.5 │ -└───────────────┘ -``` - -**همچنین نگاه کنید به** - -- [میانه](#median) -- [quantiles](#quantiles) - -## نامعینیهای کوانتی {#quantiledeterministic} - -محاسبه تقریبی [quantile](https://en.wikipedia.org/wiki/Quantile) از یک توالی داده های عددی. - -این تابع اعمال می شود [نمونه برداری مخزن](https://en.wikipedia.org/wiki/Reservoir_sampling) با اندازه مخزن تا 8192 و الگوریتم قطعی نمونه گیری. نتیجه قطعی است. برای دریافت یک کمی دقیق, استفاده از [کوانتوم](#quantileexact) تابع. - -هنگام استفاده از چندین `quantile*` توابع با سطوح مختلف در پرس و جو, کشورهای داخلی در ترکیب نیست (به این معنا که, پرس و جو کار می کند موثر کمتر از می تواند). در این مورد از [quantiles](#quantiles) تابع. - -**نحو** - -``` sql -quantileDeterministic(level)(expr, determinator) -``` - -نام مستعار: `medianDeterministic`. - -**پارامترها** - -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` مقدار در محدوده `[0.01, 0.99]`. مقدار پیش فرض: 0.5. در `level=0.5` تابع محاسبه می کند [میانه](https://en.wikipedia.org/wiki/Median). -- `expr` — Expression over the column values resulting in numeric [انواع داده ها](../../sql-reference/data-types/index.md#data_types), [تاریخ](../../sql-reference/data-types/date.md) یا [DateTime](../../sql-reference/data-types/datetime.md). -- `determinator` — Number whose hash is used instead of a random number generator in the reservoir sampling algorithm to make the result of sampling deterministic. As a determinator you can use any deterministic positive number, for example, a user id or an event id. If the same determinator value occures too often, the function works incorrectly. - -**مقدار بازگشتی** - -- کمی تقریبی از سطح مشخص شده است. - -نوع: - -- [جسم شناور64](../../sql-reference/data-types/float.md) برای ورودی نوع داده عددی. -- [تاریخ](../../sql-reference/data-types/date.md) اگر مقادیر ورودی `Date` نوع. -- [DateTime](../../sql-reference/data-types/datetime.md) اگر مقادیر ورودی `DateTime` نوع. - -**مثال** - -جدول ورودی: - -``` text -┌─val─┐ -│ 1 │ -│ 1 │ -│ 2 │ -│ 3 │ -└─────┘ -``` - -پرسوجو: - -``` sql -SELECT quantileDeterministic(val, 1) FROM t -``` - -نتیجه: - -``` text -┌─quantileDeterministic(val, 1)─┐ -│ 1.5 │ -└───────────────────────────────┘ -``` - -**همچنین نگاه کنید به** - -- [میانه](#median) -- [quantiles](#quantiles) - -## کوانتوم {#quantileexact} - -دقیقا محاسبه می کند [quantile](https://en.wikipedia.org/wiki/Quantile) از یک توالی داده های عددی. - -To get exact value, all the passed values ​​are combined into an array, which is then partially sorted. Therefore, the function consumes `O(n)` حافظه, جایی که `n` تعدادی از ارزش هایی که تصویب شد. اما, برای تعداد کمی از ارزش, تابع بسیار موثر است. - -هنگام استفاده از چندین `quantile*` توابع با سطوح مختلف در پرس و جو, کشورهای داخلی در ترکیب نیست (به این معنا که, پرس و جو کار می کند موثر کمتر از می تواند). در این مورد از [quantiles](#quantiles) تابع. - -**نحو** - -``` sql -quantileExact(level)(expr) -``` - -نام مستعار: `medianExact`. - -**پارامترها** - -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` مقدار در محدوده `[0.01, 0.99]`. مقدار پیش فرض: 0.5. در `level=0.5` تابع محاسبه می کند [میانه](https://en.wikipedia.org/wiki/Median). -- `expr` — Expression over the column values resulting in numeric [انواع داده ها](../../sql-reference/data-types/index.md#data_types), [تاریخ](../../sql-reference/data-types/date.md) یا [DateTime](../../sql-reference/data-types/datetime.md). - -**مقدار بازگشتی** - -- Quantile از سطح مشخص شده. - -نوع: - -- [جسم شناور64](../../sql-reference/data-types/float.md) برای ورودی نوع داده عددی. -- [تاریخ](../../sql-reference/data-types/date.md) اگر مقادیر ورودی `Date` نوع. -- [DateTime](../../sql-reference/data-types/datetime.md) اگر مقادیر ورودی `DateTime` نوع. - -**مثال** - -پرسوجو: - -``` sql -SELECT quantileExact(number) FROM numbers(10) -``` - -نتیجه: - -``` text -┌─quantileExact(number)─┐ -│ 5 │ -└───────────────────────┘ -``` - -**همچنین نگاه کنید به** - -- [میانه](#median) -- [quantiles](#quantiles) - -## نمایش سایت {#quantileexactweighted} - -دقیقا محاسبه می کند [quantile](https://en.wikipedia.org/wiki/Quantile) از یک توالی داده های عددی, با در نظر گرفتن وزن هر عنصر. - -To get exact value, all the passed values ​​are combined into an array, which is then partially sorted. Each value is counted with its weight, as if it is present `weight` times. A hash table is used in the algorithm. Because of this, if the passed values ​​are frequently repeated, the function consumes less RAM than [کوانتوم](#quantileexact). شما می توانید این تابع به جای استفاده از `quantileExact` و وزن 1 را مشخص کنید. - -هنگام استفاده از چندین `quantile*` توابع با سطوح مختلف در پرس و جو, کشورهای داخلی در ترکیب نیست (به این معنا که, پرس و جو کار می کند موثر کمتر از می تواند). در این مورد از [quantiles](#quantiles) تابع. - -**نحو** - -``` sql -quantileExactWeighted(level)(expr, weight) -``` - -نام مستعار: `medianExactWeighted`. - -**پارامترها** - -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` مقدار در محدوده `[0.01, 0.99]`. مقدار پیش فرض: 0.5. در `level=0.5` تابع محاسبه می کند [میانه](https://en.wikipedia.org/wiki/Median). -- `expr` — Expression over the column values resulting in numeric [انواع داده ها](../../sql-reference/data-types/index.md#data_types), [تاریخ](../../sql-reference/data-types/date.md) یا [DateTime](../../sql-reference/data-types/datetime.md). -- `weight` — Column with weights of sequence members. Weight is a number of value occurrences. - -**مقدار بازگشتی** - -- Quantile از سطح مشخص شده. - -نوع: - -- [جسم شناور64](../../sql-reference/data-types/float.md) برای ورودی نوع داده عددی. -- [تاریخ](../../sql-reference/data-types/date.md) اگر مقادیر ورودی `Date` نوع. -- [DateTime](../../sql-reference/data-types/datetime.md) اگر مقادیر ورودی `DateTime` نوع. - -**مثال** - -جدول ورودی: - -``` text -┌─n─┬─val─┐ -│ 0 │ 3 │ -│ 1 │ 2 │ -│ 2 │ 1 │ -│ 5 │ 4 │ -└───┴─────┘ -``` - -پرسوجو: - -``` sql -SELECT quantileExactWeighted(n, val) FROM t -``` - -نتیجه: - -``` text -┌─quantileExactWeighted(n, val)─┐ -│ 1 │ -└───────────────────────────────┘ -``` - -**همچنین نگاه کنید به** - -- [میانه](#median) -- [quantiles](#quantiles) - -## زمان کمی {#quantiletiming} - -با دقت تعیین شده محاسبه می شود [quantile](https://en.wikipedia.org/wiki/Quantile) از یک توالی داده های عددی. - -نتیجه قطعی است(به سفارش پردازش پرس و جو بستگی ندارد). این تابع برای کار با توالی هایی که توزیع هایی مانند بارگذاری صفحات وب بار یا زمان پاسخ باطن را توصیف می کنند بهینه شده است. - -هنگام استفاده از چندین `quantile*` توابع با سطوح مختلف در پرس و جو, کشورهای داخلی در ترکیب نیست (به این معنا که, پرس و جو کار می کند موثر کمتر از می تواند). در این مورد از [quantiles](#quantiles) تابع. - -**نحو** - -``` sql -quantileTiming(level)(expr) -``` - -نام مستعار: `medianTiming`. - -**پارامترها** - -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` مقدار در محدوده `[0.01, 0.99]`. مقدار پیش فرض: 0.5. در `level=0.5` تابع محاسبه می کند [میانه](https://en.wikipedia.org/wiki/Median). - -- `expr` — [عبارت](../syntax.md#syntax-expressions) بیش از یک مقادیر ستون بازگشت [شناور\*](../../sql-reference/data-types/float.md)- نوع شماره . - - - If negative values are passed to the function, the behavior is undefined. - - If the value is greater than 30,000 (a page loading time of more than 30 seconds), it is assumed to be 30,000. - -**دقت** - -محاسبه دقیق است اگر: - -- تعداد کل مقادیر 5670 تجاوز نمی کند. -- تعداد کل مقادیر بیش از 5670, اما زمان بارگذاری صفحه کمتر از است 1024خانم. - -در غیر این صورت, نتیجه محاسبه به نزدیکترین چند از گرد 16 خانم. - -!!! note "یادداشت" - برای محاسبه زمان بارگذاری صفحه quantiles این تابع این است که موثر تر و دقیق تر از [quantile](#quantile). - -**مقدار بازگشتی** - -- Quantile از سطح مشخص شده. - -نوع: `Float32`. - -!!! note "یادداشت" - اگر هیچ ارزش به تابع منتقل می شود (هنگام استفاده از `quantileTimingIf`), [نان](../../sql-reference/data-types/float.md#data_type-float-nan-inf) بازگشته است. هدف از این است که افتراق این موارد از مواردی که منجر به صفر. ببینید [ORDER BY](../statements/select/order-by.md#select-order-by) برای یادداشت ها در مرتب سازی `NaN` ارزشهای خبری عبارتند از: - -**مثال** - -جدول ورودی: - -``` text -┌─response_time─┐ -│ 72 │ -│ 112 │ -│ 126 │ -│ 145 │ -│ 104 │ -│ 242 │ -│ 313 │ -│ 168 │ -│ 108 │ -└───────────────┘ -``` - -پرسوجو: - -``` sql -SELECT quantileTiming(response_time) FROM t -``` - -نتیجه: - -``` text -┌─quantileTiming(response_time)─┐ -│ 126 │ -└───────────────────────────────┘ -``` - -**همچنین نگاه کنید به** - -- [میانه](#median) -- [quantiles](#quantiles) - -## زمان کمی {#quantiletimingweighted} - -با دقت تعیین شده محاسبه می شود [quantile](https://en.wikipedia.org/wiki/Quantile) از یک توالی داده های عددی با توجه به وزن هر یک از اعضای دنباله. - -نتیجه قطعی است(به سفارش پردازش پرس و جو بستگی ندارد). این تابع برای کار با توالی هایی که توزیع هایی مانند بارگذاری صفحات وب بار یا زمان پاسخ باطن را توصیف می کنند بهینه شده است. - -هنگام استفاده از چندین `quantile*` توابع با سطوح مختلف در پرس و جو, کشورهای داخلی در ترکیب نیست (به این معنا که, پرس و جو کار می کند موثر کمتر از می تواند). در این مورد از [quantiles](#quantiles) تابع. - -**نحو** - -``` sql -quantileTimingWeighted(level)(expr, weight) -``` - -نام مستعار: `medianTimingWeighted`. - -**پارامترها** - -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` مقدار در محدوده `[0.01, 0.99]`. مقدار پیش فرض: 0.5. در `level=0.5` تابع محاسبه می کند [میانه](https://en.wikipedia.org/wiki/Median). - -- `expr` — [عبارت](../syntax.md#syntax-expressions) بیش از یک مقادیر ستون بازگشت [شناور\*](../../sql-reference/data-types/float.md)- نوع شماره . - - - If negative values are passed to the function, the behavior is undefined. - - If the value is greater than 30,000 (a page loading time of more than 30 seconds), it is assumed to be 30,000. - -- `weight` — Column with weights of sequence elements. Weight is a number of value occurrences. - -**دقت** - -محاسبه دقیق است اگر: - -- تعداد کل مقادیر 5670 تجاوز نمی کند. -- تعداد کل مقادیر بیش از 5670, اما زمان بارگذاری صفحه کمتر از است 1024خانم. - -در غیر این صورت, نتیجه محاسبه به نزدیکترین چند از گرد 16 خانم. - -!!! note "یادداشت" - برای محاسبه زمان بارگذاری صفحه quantiles این تابع این است که موثر تر و دقیق تر از [quantile](#quantile). - -**مقدار بازگشتی** - -- Quantile از سطح مشخص شده. - -نوع: `Float32`. - -!!! note "یادداشت" - اگر هیچ ارزش به تابع منتقل می شود (هنگام استفاده از `quantileTimingIf`), [نان](../../sql-reference/data-types/float.md#data_type-float-nan-inf) بازگشته است. هدف از این است که افتراق این موارد از مواردی که منجر به صفر. ببینید [ORDER BY](../statements/select/order-by.md#select-order-by) برای یادداشت ها در مرتب سازی `NaN` ارزشهای خبری عبارتند از: - -**مثال** - -جدول ورودی: - -``` text -┌─response_time─┬─weight─┐ -│ 68 │ 1 │ -│ 104 │ 2 │ -│ 112 │ 3 │ -│ 126 │ 2 │ -│ 138 │ 1 │ -│ 162 │ 1 │ -└───────────────┴────────┘ -``` - -پرسوجو: - -``` sql -SELECT quantileTimingWeighted(response_time, weight) FROM t -``` - -نتیجه: - -``` text -┌─quantileTimingWeighted(response_time, weight)─┐ -│ 112 │ -└───────────────────────────────────────────────┘ -``` - -**همچنین نگاه کنید به** - -- [میانه](#median) -- [quantiles](#quantiles) - -## مقدار کمی {#quantiletdigest} - -محاسبه تقریبی [quantile](https://en.wikipedia.org/wiki/Quantile) از یک توالی داده های عددی با استفاده از [خلاصه](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) الگوریتم. - -حداکثر خطا است 1%. مصرف حافظه است `log(n)` کجا `n` تعدادی از ارزش است. نتیجه بستگی دارد منظور از در حال اجرا پرس و جو و nondeterministic. - -عملکرد تابع کمتر از عملکرد است [quantile](#quantile) یا [زمان کمی](#quantiletiming). از لحاظ نسبت اندازه دولت به دقت, این تابع بسیار بهتر از `quantile`. - -هنگام استفاده از چندین `quantile*` توابع با سطوح مختلف در پرس و جو, کشورهای داخلی در ترکیب نیست (به این معنا که, پرس و جو کار می کند موثر کمتر از می تواند). در این مورد از [quantiles](#quantiles) تابع. - -**نحو** - -``` sql -quantileTDigest(level)(expr) -``` - -نام مستعار: `medianTDigest`. - -**پارامترها** - -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` مقدار در محدوده `[0.01, 0.99]`. مقدار پیش فرض: 0.5. در `level=0.5` تابع محاسبه می کند [میانه](https://en.wikipedia.org/wiki/Median). -- `expr` — Expression over the column values resulting in numeric [انواع داده ها](../../sql-reference/data-types/index.md#data_types), [تاریخ](../../sql-reference/data-types/date.md) یا [DateTime](../../sql-reference/data-types/datetime.md). - -**مقدار بازگشتی** - -- کمی تقریبی از سطح مشخص شده است. - -نوع: - -- [جسم شناور64](../../sql-reference/data-types/float.md) برای ورودی نوع داده عددی. -- [تاریخ](../../sql-reference/data-types/date.md) اگر مقادیر ورودی `Date` نوع. -- [DateTime](../../sql-reference/data-types/datetime.md) اگر مقادیر ورودی `DateTime` نوع. - -**مثال** - -پرسوجو: - -``` sql -SELECT quantileTDigest(number) FROM numbers(10) -``` - -نتیجه: - -``` text -┌─quantileTDigest(number)─┐ -│ 4.5 │ -└─────────────────────────┘ -``` - -**همچنین نگاه کنید به** - -- [میانه](#median) -- [quantiles](#quantiles) - -## نمایش سایت {#quantiletdigestweighted} - -محاسبه تقریبی [quantile](https://en.wikipedia.org/wiki/Quantile) از یک توالی داده های عددی با استفاده از [خلاصه](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) الگوریتم. تابع طول می کشد را به حساب وزن هر یک از اعضای دنباله. حداکثر خطا است 1%. مصرف حافظه است `log(n)` کجا `n` تعدادی از ارزش است. - -عملکرد تابع کمتر از عملکرد است [quantile](#quantile) یا [زمان کمی](#quantiletiming). از لحاظ نسبت اندازه دولت به دقت, این تابع بسیار بهتر از `quantile`. - -نتیجه بستگی دارد منظور از در حال اجرا پرس و جو و nondeterministic. - -هنگام استفاده از چندین `quantile*` توابع با سطوح مختلف در پرس و جو, کشورهای داخلی در ترکیب نیست (به این معنا که, پرس و جو کار می کند موثر کمتر از می تواند). در این مورد از [quantiles](#quantiles) تابع. - -**نحو** - -``` sql -quantileTDigest(level)(expr) -``` - -نام مستعار: `medianTDigest`. - -**پارامترها** - -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` مقدار در محدوده `[0.01, 0.99]`. مقدار پیش فرض: 0.5. در `level=0.5` تابع محاسبه می کند [میانه](https://en.wikipedia.org/wiki/Median). -- `expr` — Expression over the column values resulting in numeric [انواع داده ها](../../sql-reference/data-types/index.md#data_types), [تاریخ](../../sql-reference/data-types/date.md) یا [DateTime](../../sql-reference/data-types/datetime.md). -- `weight` — Column with weights of sequence elements. Weight is a number of value occurrences. - -**مقدار بازگشتی** - -- کمی تقریبی از سطح مشخص شده است. - -نوع: - -- [جسم شناور64](../../sql-reference/data-types/float.md) برای ورودی نوع داده عددی. -- [تاریخ](../../sql-reference/data-types/date.md) اگر مقادیر ورودی `Date` نوع. -- [DateTime](../../sql-reference/data-types/datetime.md) اگر مقادیر ورودی `DateTime` نوع. - -**مثال** - -پرسوجو: - -``` sql -SELECT quantileTDigestWeighted(number, 1) FROM numbers(10) -``` - -نتیجه: - -``` text -┌─quantileTDigestWeighted(number, 1)─┐ -│ 4.5 │ -└────────────────────────────────────┘ -``` - -**همچنین نگاه کنید به** - -- [میانه](#median) -- [quantiles](#quantiles) - -## میانه {#median} - -این `median*` توابع نام مستعار برای مربوطه `quantile*` توابع. متوسط یک نمونه داده عددی را محاسبه می کنند. - -توابع: - -- `median` — Alias for [quantile](#quantile). -- `medianDeterministic` — Alias for [نامعینیهای کوانتی](#quantiledeterministic). -- `medianExact` — Alias for [کوانتوم](#quantileexact). -- `medianExactWeighted` — Alias for [نمایش سایت](#quantileexactweighted). -- `medianTiming` — Alias for [زمان کمی](#quantiletiming). -- `medianTimingWeighted` — Alias for [زمان کمی](#quantiletimingweighted). -- `medianTDigest` — Alias for [مقدار کمی](#quantiletdigest). -- `medianTDigestWeighted` — Alias for [نمایش سایت](#quantiletdigestweighted). - -**مثال** - -جدول ورودی: - -``` text -┌─val─┐ -│ 1 │ -│ 1 │ -│ 2 │ -│ 3 │ -└─────┘ -``` - -پرسوجو: - -``` sql -SELECT medianDeterministic(val, 1) FROM t -``` - -نتیجه: - -``` text -┌─medianDeterministic(val, 1)─┐ -│ 1.5 │ -└─────────────────────────────┘ -``` - -## quantiles(level1, level2, …)(x) {#quantiles} - -تمام quantile توابع نیز مربوط quantiles توابع: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. این توابع محاسبه تمام کوانتوم از سطوح ذکر شده در یک پاس, و بازگشت مجموعه ای از مقادیر حاصل. - -## اطلاعات دقیق) {#varsampx} - -محاسبه مقدار `Σ((x - x̅)^2) / (n - 1)` کجا `n` اندازه نمونه است و `x̅`مقدار متوسط است `x`. - -این نشان دهنده یک تخمین بی طرفانه از واریانس یک متغیر تصادفی اگر ارزش گذشت نمونه خود را تشکیل می دهند. - -بازگشت `Float64`. چه زمانی `n <= 1`, بازگشت `+∞`. - -!!! note "یادداشت" - این تابع با استفاده از الگوریتم عددی ناپایدار. اگر شما نیاز دارید [پایداری عددی](https://en.wikipedia.org/wiki/Numerical_stability) در محاسبات, استفاده از `varSampStable` تابع. این کار کندتر, فراهم می کند اما خطای محاسباتی کمتر. - -## هشدار داده می شود) {#varpopx} - -محاسبه مقدار `Σ((x - x̅)^2) / n` کجا `n` اندازه نمونه است و `x̅`مقدار متوسط است `x`. - -به عبارت دیگر, پراکندگی برای مجموعه ای از ارزش. بازگشت `Float64`. - -!!! note "یادداشت" - این تابع با استفاده از الگوریتم عددی ناپایدار. اگر شما نیاز دارید [پایداری عددی](https://en.wikipedia.org/wiki/Numerical_stability) در محاسبات, استفاده از `varPopStable` تابع. این کار کندتر, فراهم می کند اما خطای محاسباتی کمتر. - -## اطلاعات دقیق) {#stddevsampx} - -نتیجه برابر با ریشه مربع است `varSamp(x)`. - -!!! note "یادداشت" - این تابع با استفاده از الگوریتم عددی ناپایدار. اگر شما نیاز دارید [پایداری عددی](https://en.wikipedia.org/wiki/Numerical_stability) در محاسبات, استفاده از `stddevSampStable` تابع. این کار کندتر, فراهم می کند اما خطای محاسباتی کمتر. - -## اطلاعات دقیق) {#stddevpopx} - -نتیجه برابر با ریشه مربع است `varPop(x)`. - -!!! note "یادداشت" - این تابع با استفاده از الگوریتم عددی ناپایدار. اگر شما نیاز دارید [پایداری عددی](https://en.wikipedia.org/wiki/Numerical_stability) در محاسبات, استفاده از `stddevPopStable` تابع. این کار کندتر, فراهم می کند اما خطای محاسباتی کمتر. - -## topK(N)(x) {#topknx} - -بازگرداندن مجموعه ای از مقادیر تقریبا شایع ترین در ستون مشخص. مجموعه حاصل به ترتیب نزولی فرکانس تقریبی ارزش ها (نه با ارزش های خود) طبقه بندی شده اند. - -پیاده سازی [فیلتر صرفه جویی در فضا](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf) الگوریتم برای تجزیه و تحلیل توپک, بر اساس الگوریتم کاهش و ترکیب از [صرفه جویی در فضای موازی](https://arxiv.org/pdf/1401.0702.pdf). - -``` sql -topK(N)(column) -``` - -این تابع یک نتیجه تضمین شده را فراهم نمی کند. در شرایط خاص, اشتباهات ممکن است رخ دهد و ممکن است مقادیر مکرر که مقادیر شایع ترین نیست بازگشت. - -ما توصیه می کنیم با استفاده از `N < 10` عملکرد با بزرگ کاهش می یابد `N` ارزشهای خبری عبارتند از: حداکثر مقدار `N = 65536`. - -**پارامترها** - -- ‘N’ است تعدادی از عناصر به بازگشت. - -اگر پارامتر حذف شده است, مقدار پیش فرض 10 استفاده شده است. - -**نشانوندها** - -- ' x ' – The value to calculate frequency. - -**مثال** - -نگاهی به [به موقع](../../getting-started/example-datasets/ontime.md) مجموعه داده ها و انتخاب سه ارزش اغلب اتفاق می افتد در `AirlineID` ستون. - -``` sql -SELECT topK(3)(AirlineID) AS res -FROM ontime -``` - -``` text -┌─res─────────────────┐ -│ [19393,19790,19805] │ -└─────────────────────┘ -``` - -## کشتی کج {#topkweighted} - -مشابه به `topK` اما طول می کشد یک استدلال اضافی از نوع صحیح - `weight`. هر مقدار به حساب `weight` بار برای محاسبه فرکانس. - -**نحو** - -``` sql -topKWeighted(N)(x, weight) -``` - -**پارامترها** - -- `N` — The number of elements to return. - -**نشانوندها** - -- `x` – The value. -- `weight` — The weight. [UInt8](../../sql-reference/data-types/int-uint.md). - -**مقدار بازگشتی** - -بازگرداندن مجموعه ای از مقادیر با حداکثر مجموع تقریبی وزن. - -**مثال** - -پرسوجو: - -``` sql -SELECT topKWeighted(10)(number, number) FROM numbers(1000) -``` - -نتیجه: - -``` text -┌─topKWeighted(10)(number, number)──────────┐ -│ [999,998,997,996,995,994,993,992,991,990] │ -└───────────────────────────────────────────┘ -``` - -## هشدار داده می شود) {#covarsampx-y} - -محاسبه ارزش `Σ((x - x̅)(y - y̅)) / (n - 1)`. - -را برمی گرداند شناور64. چه زمانی `n <= 1`, returns +∞. - -!!! note "یادداشت" - این تابع با استفاده از الگوریتم عددی ناپایدار. اگر شما نیاز دارید [پایداری عددی](https://en.wikipedia.org/wiki/Numerical_stability) در محاسبات, استفاده از `covarSampStable` تابع. این کار کندتر, فراهم می کند اما خطای محاسباتی کمتر. - -## نمایش سایت) {#covarpopx-y} - -محاسبه ارزش `Σ((x - x̅)(y - y̅)) / n`. - -!!! note "یادداشت" - این تابع با استفاده از الگوریتم عددی ناپایدار. اگر شما نیاز دارید [پایداری عددی](https://en.wikipedia.org/wiki/Numerical_stability) در محاسبات, استفاده از `covarPopStable` تابع. این کار کندتر فراهم می کند اما یک خطای محاسباتی کمتر. - -## هشدار داده می شود) {#corrx-y} - -محاسبه ضریب همبستگی پیرسون: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. - -!!! note "یادداشت" - این تابع با استفاده از الگوریتم عددی ناپایدار. اگر شما نیاز دارید [پایداری عددی](https://en.wikipedia.org/wiki/Numerical_stability) در محاسبات, استفاده از `corrStable` تابع. این کار کندتر, فراهم می کند اما خطای محاسباتی کمتر. - -## طبقه بندی فرمول بندی {#categoricalinformationvalue} - -محاسبه ارزش `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` برای هر دسته. - -``` sql -categoricalInformationValue(category1, category2, ..., tag) -``` - -نتیجه نشان می دهد که چگونه یک ویژگی گسسته (قطعی) `[category1, category2, ...]` کمک به یک مدل یادگیری که پیش بینی ارزش `tag`. - -## ساده سازی مقررات {#simplelinearregression} - -انجام ساده (unidimensional) رگرسیون خطی. - -``` sql -simpleLinearRegression(x, y) -``` - -پارامترها: - -- `x` — Column with dependent variable values. -- `y` — Column with explanatory variable values. - -مقادیر بازگشتی: - -ثابتها `(a, b)` از خط نتیجه `y = a*x + b`. - -**مثالها** - -``` sql -SELECT arrayReduce('simpleLinearRegression', [0, 1, 2, 3], [0, 1, 2, 3]) -``` - -``` text -┌─arrayReduce('simpleLinearRegression', [0, 1, 2, 3], [0, 1, 2, 3])─┐ -│ (1,0) │ -└───────────────────────────────────────────────────────────────────┘ -``` - -``` sql -SELECT arrayReduce('simpleLinearRegression', [0, 1, 2, 3], [3, 4, 5, 6]) -``` - -``` text -┌─arrayReduce('simpleLinearRegression', [0, 1, 2, 3], [3, 4, 5, 6])─┐ -│ (1,3) │ -└───────────────────────────────────────────────────────────────────┘ -``` - -## تنظیم مقررات {#agg_functions-stochasticlinearregression} - -این تابع پیاده سازی رگرسیون خطی تصادفی. این پشتیبانی از پارامترهای سفارشی برای نرخ یادگیری, ل2 ضریب منظم, اندازه مینی دسته ای و دارای چند روش برای به روز رسانی وزن ([ادام](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Adam) (به طور پیش فرض استفاده می شود), [اطلاعات دقیق](https://en.wikipedia.org/wiki/Stochastic_gradient_descent), [شتاب](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Momentum), [نستروف](https://mipt.ru/upload/medialibrary/d7e/41-91.pdf)). - -### پارامترها {#agg_functions-stochasticlinearregression-parameters} - -4 پارامتر قابل تنظیم وجود دارد. به ترتیب تابع منتقل می شود اما بدون نیاز به تصویب تمام مقادیر چهار پیش فرض استفاده می شود با این حال مدل خوب مورد نیاز برخی از تنظیم پارامتر وجود دارد. - -``` text -stochasticLinearRegression(1.0, 1.0, 10, 'SGD') -``` - -1. `learning rate` ضریب در طول گام است, زمانی که گام گرادیان تبار انجام شده است. نرخ یادگیری بیش از حد بزرگ ممکن است وزن بی نهایت از مدل شود. پیشفرض `0.00001`. -2. `l2 regularization coefficient` که ممکن است کمک به جلوگیری از سوراخ سوراخ شدن بیش از حد. پیشفرض `0.1`. -3. `mini-batch size` مجموعه تعدادی از عناصر که شیب محاسبه خواهد شد و خلاصه به انجام یک مرحله از گرادیان تبار. تبار تصادفی خالص با استفاده از یک عنصر, با این حال داشتن دسته های کوچک(در باره 10 عناصر) را گام شیب پایدار تر. پیشفرض `15`. -4. `method for updating weights` اونا: `Adam` (به طور پیش فرض), `SGD`, `Momentum`, `Nesterov`. `Momentum` و `Nesterov` نیاز به کمی بیشتر محاسبات و حافظه, اما آنها به اتفاق مفید از نظر سرعت convergance و ثبات stochastic gradient روش. - -### استفاده {#agg_functions-stochasticlinearregression-usage} - -`stochasticLinearRegression` در دو مرحله استفاده می شود: اتصالات مدل و پیش بینی بر روی داده های جدید. به منظور متناسب با مدل و صرفه جویی در دولت خود را برای استفاده های بعدی استفاده می کنیم `-State` ترکیب کننده, که اساسا موجب صرفه جویی در دولت (وزن مدل, و غیره). -برای پیش بینی ما با استفاده از تابع [ارزیابی](../functions/machine-learning-functions.md#machine_learning_methods-evalmlmethod), که طول می کشد یک دولت به عنوان یک استدلال و همچنین ویژگی های به پیش بینی در. - - - -**1.** اتصالات - -چنین پرس و جو ممکن است مورد استفاده قرار گیرد. - -``` sql -CREATE TABLE IF NOT EXISTS train_data -( - param1 Float64, - param2 Float64, - target Float64 -) ENGINE = Memory; - -CREATE TABLE your_model ENGINE = Memory AS SELECT -stochasticLinearRegressionState(0.1, 0.0, 5, 'SGD')(target, param1, param2) -AS state FROM train_data; -``` - -در اینجا ما همچنین نیاز به وارد کردن داده ها به `train_data` جدول تعداد پارامترهای ثابت نیست, این تنها در تعدادی از استدلال بستگی دارد, گذشت به `linearRegressionState`. همه باید مقادیر عددی باشد. -توجه داشته باشید که ستون با ارزش هدف(که ما می خواهم برای یادگیری به پیش بینی) به عنوان اولین استدلال قرار داده شده است. - -**2.** پیش بینی - -پس از ذخیره یک دولت به جدول, ما ممکن است چندین بار برای پیش بینی استفاده, و یا حتی با کشورهای دیگر ادغام و ایجاد مدل های جدید و حتی بهتر. - -``` sql -WITH (SELECT state FROM your_model) AS model SELECT -evalMLMethod(model, param1, param2) FROM test_data -``` - -پرس و جو یک ستون از مقادیر پیش بینی شده بازگشت. توجه داشته باشید که استدلال اول `evalMLMethod` هست `AggregateFunctionState` هدف, بعدی ستون از ویژگی های هستند. - -`test_data` یک جدول مانند `train_data` اما ممکن است حاوی ارزش هدف نیست. - -### یادداشتها {#agg_functions-stochasticlinearregression-notes} - -1. برای ادغام دو مدل کاربر ممکن است چنین پرس و جو ایجاد کنید: - `sql SELECT state1 + state2 FROM your_models` - کجا `your_models` جدول شامل هر دو مدل. این پرس و جو جدید باز خواهد گشت `AggregateFunctionState` اعتراض. - -2. کاربر ممکن است وزن مدل ایجاد شده برای اهداف خود را بدون صرفه جویی در مدل اگر هیچ واکشی `-State` ترکیب استفاده شده است. - `sql SELECT stochasticLinearRegression(0.01)(target, param1, param2) FROM train_data` - چنین پرس و جو خواهد مدل مناسب و بازگشت وزن خود را - برای اولین بار وزن هستند, که به پارامترهای مدل مطابقت, یکی از گذشته تعصب است. بنابراین در مثال بالا پرس و جو یک ستون با 3 مقدار بازگشت. - -**همچنین نگاه کنید به** - -- [سرکوب مقررات عمومی](#agg_functions-stochasticlogisticregression) -- [تفاوت رگرسیون خطی و لجستیک](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression) - -## سرکوب مقررات عمومی {#agg_functions-stochasticlogisticregression} - -این تابع پیاده سازی رگرسیون لجستیک تصادفی. این را می توان برای مشکل طبقه بندی دودویی استفاده, پشتیبانی از پارامترهای سفارشی به عنوان مقررات زدایی و کار به همان شیوه. - -### پارامترها {#agg_functions-stochasticlogisticregression-parameters} - -پارامترها دقیقا مشابه در تنظیم مقررات است: -`learning rate`, `l2 regularization coefficient`, `mini-batch size`, `method for updating weights`. -برای اطلاعات بیشتر نگاه کنید به [پارامترها](#agg_functions-stochasticlinearregression-parameters). - -``` text -stochasticLogisticRegression(1.0, 1.0, 10, 'SGD') -``` - -1. اتصالات - - - - See the `Fitting` section in the [stochasticLinearRegression](#stochasticlinearregression-usage-fitting) description. - - Predicted labels have to be in \[-1, 1\]. - -1. پیش بینی - - - - Using saved state we can predict probability of object having label `1`. - - ``` sql - WITH (SELECT state FROM your_model) AS model SELECT - evalMLMethod(model, param1, param2) FROM test_data - ``` - - The query will return a column of probabilities. Note that first argument of `evalMLMethod` is `AggregateFunctionState` object, next are columns of features. - - We can also set a bound of probability, which assigns elements to different labels. - - ``` sql - SELECT ans < 1.1 AND ans > 0.5 FROM - (WITH (SELECT state FROM your_model) AS model SELECT - evalMLMethod(model, param1, param2) AS ans FROM test_data) - ``` - - Then the result will be labels. - - `test_data` is a table like `train_data` but may not contain target value. - -**همچنین نگاه کنید به** - -- [تنظیم مقررات](#agg_functions-stochasticlinearregression) -- [تفاوت بین رگرسیون خطی و لجستیک.](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression) - -## گروهبیتمافند {#groupbitmapand} - -محاسبات و یک بیت مپ ستون بازگشت cardinality از نوع UInt64 اگر اضافه کردن پسوند -دولت بازگشت [شی نگاشت بیت](../../sql-reference/functions/bitmap-functions.md). - -``` sql -groupBitmapAnd(expr) -``` - -**پارامترها** - -`expr` – An expression that results in `AggregateFunction(groupBitmap, UInt*)` نوع. - -**مقدار بازگشتی** - -ارزش `UInt64` نوع. - -**مثال** - -``` sql -DROP TABLE IF EXISTS bitmap_column_expr_test2; -CREATE TABLE bitmap_column_expr_test2 -( - tag_id String, - z AggregateFunction(groupBitmap, UInt32) -) -ENGINE = MergeTree -ORDER BY tag_id; - -INSERT INTO bitmap_column_expr_test2 VALUES ('tag1', bitmapBuild(cast([1,2,3,4,5,6,7,8,9,10] as Array(UInt32)))); -INSERT INTO bitmap_column_expr_test2 VALUES ('tag2', bitmapBuild(cast([6,7,8,9,10,11,12,13,14,15] as Array(UInt32)))); -INSERT INTO bitmap_column_expr_test2 VALUES ('tag3', bitmapBuild(cast([2,4,6,8,10,12] as Array(UInt32)))); - -SELECT groupBitmapAnd(z) FROM bitmap_column_expr_test2 WHERE like(tag_id, 'tag%'); -┌─groupBitmapAnd(z)─┐ -│ 3 │ -└───────────────────┘ - -SELECT arraySort(bitmapToArray(groupBitmapAndState(z))) FROM bitmap_column_expr_test2 WHERE like(tag_id, 'tag%'); -┌─arraySort(bitmapToArray(groupBitmapAndState(z)))─┐ -│ [6,8,10] │ -└──────────────────────────────────────────────────┘ -``` - -## گروهبیتمافور {#groupbitmapor} - -محاسبات و یا یک بیت مپ ستون بازگشت cardinality از نوع UInt64 اگر اضافه کردن پسوند -دولت بازگشت [شی نگاشت بیت](../../sql-reference/functions/bitmap-functions.md). این معادل است `groupBitmapMerge`. - -``` sql -groupBitmapOr(expr) -``` - -**پارامترها** - -`expr` – An expression that results in `AggregateFunction(groupBitmap, UInt*)` نوع. - -**مقدار بازگشتی** - -ارزش `UInt64` نوع. - -**مثال** - -``` sql -DROP TABLE IF EXISTS bitmap_column_expr_test2; -CREATE TABLE bitmap_column_expr_test2 -( - tag_id String, - z AggregateFunction(groupBitmap, UInt32) -) -ENGINE = MergeTree -ORDER BY tag_id; - -INSERT INTO bitmap_column_expr_test2 VALUES ('tag1', bitmapBuild(cast([1,2,3,4,5,6,7,8,9,10] as Array(UInt32)))); -INSERT INTO bitmap_column_expr_test2 VALUES ('tag2', bitmapBuild(cast([6,7,8,9,10,11,12,13,14,15] as Array(UInt32)))); -INSERT INTO bitmap_column_expr_test2 VALUES ('tag3', bitmapBuild(cast([2,4,6,8,10,12] as Array(UInt32)))); - -SELECT groupBitmapOr(z) FROM bitmap_column_expr_test2 WHERE like(tag_id, 'tag%'); -┌─groupBitmapOr(z)─┐ -│ 15 │ -└──────────────────┘ - -SELECT arraySort(bitmapToArray(groupBitmapOrState(z))) FROM bitmap_column_expr_test2 WHERE like(tag_id, 'tag%'); -┌─arraySort(bitmapToArray(groupBitmapOrState(z)))─┐ -│ [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] │ -└─────────────────────────────────────────────────┘ -``` - -## گروهبیتمافکر {#groupbitmapxor} - -محاسبات XOR یک بیت مپ ستون بازگشت cardinality از نوع UInt64 اگر اضافه کردن پسوند -دولت بازگشت [شی نگاشت بیت](../../sql-reference/functions/bitmap-functions.md). - -``` sql -groupBitmapOr(expr) -``` - -**پارامترها** - -`expr` – An expression that results in `AggregateFunction(groupBitmap, UInt*)` نوع. - -**مقدار بازگشتی** - -ارزش `UInt64` نوع. - -**مثال** - -``` sql -DROP TABLE IF EXISTS bitmap_column_expr_test2; -CREATE TABLE bitmap_column_expr_test2 -( - tag_id String, - z AggregateFunction(groupBitmap, UInt32) -) -ENGINE = MergeTree -ORDER BY tag_id; - -INSERT INTO bitmap_column_expr_test2 VALUES ('tag1', bitmapBuild(cast([1,2,3,4,5,6,7,8,9,10] as Array(UInt32)))); -INSERT INTO bitmap_column_expr_test2 VALUES ('tag2', bitmapBuild(cast([6,7,8,9,10,11,12,13,14,15] as Array(UInt32)))); -INSERT INTO bitmap_column_expr_test2 VALUES ('tag3', bitmapBuild(cast([2,4,6,8,10,12] as Array(UInt32)))); - -SELECT groupBitmapXor(z) FROM bitmap_column_expr_test2 WHERE like(tag_id, 'tag%'); -┌─groupBitmapXor(z)─┐ -│ 10 │ -└───────────────────┘ - -SELECT arraySort(bitmapToArray(groupBitmapXorState(z))) FROM bitmap_column_expr_test2 WHERE like(tag_id, 'tag%'); -┌─arraySort(bitmapToArray(groupBitmapXorState(z)))─┐ -│ [1,3,5,6,8,10,11,13,14,15] │ -└──────────────────────────────────────────────────┘ -``` - -[مقاله اصلی](https://clickhouse.tech/docs/en/query_language/agg_functions/reference/) diff --git a/docs/fa/sql-reference/ansi.md b/docs/fa/sql-reference/ansi.md deleted file mode 100644 index 5be2c353157..00000000000 --- a/docs/fa/sql-reference/ansi.md +++ /dev/null @@ -1,180 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: ad252bbb4f7e2899c448eb42ecc39ff195c8faa1 -toc_priority: 40 -toc_title: "\u0633\u0627\u0632\u06AF\u0627\u0631\u06CC \u0627\u0646\u0633\u06CC" ---- - -# سازگاری انسی گذاشتن گویش فاحشه خانه {#ansi-sql-compatibility-of-clickhouse-sql-dialect} - -!!! note "یادداشت" - این مقاله متکی بر جدول 38, “Feature taxonomy and definition for mandatory features”, Annex F of ISO/IEC CD 9075-2:2013. - -## تفاوت در رفتار {#differences-in-behaviour} - -جدول زیر لیست موارد زمانی که از ویژگی های پرس و جو کار می کند در خانه رعیتی, اما رفتار نه به عنوان مشخص شده در انسی گذاشتن. - -| Feature ID | نام ویژگی | تفاوت | -|------------|---------------------------------------|--------------------------------------------------------------------------------------------| -| E011 | انواع داده های عددی | تحت اللفظی عددی با دوره به عنوان تقریبی تفسیر شده است (`Float64`) به جای دقیق (`Decimal`) | -| E051-05 | انتخاب موارد را می توان تغییر نام داد | تغییر نام مورد یک دامنه دید گسترده تر از فقط نتیجه را انتخاب کنید | -| E141-01 | محدودیت NOT NULL | `NOT NULL` برای ستون های جدول به طور پیش فرض ضمنی | -| E011-04 | اپراتورهای ریاضی | سرریز کلیک به جای حساب بررسی می شود و نوع داده نتیجه را بر اساس قوانین سفارشی تغییر می دهد | - -## وضعیت ویژگی {#feature-status} - -| Feature ID | نام ویژگی | وضعیت | توضیح | -|------------|---------------------------------------------------------------------------------------------------|-------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **E011** | **انواع داده های عددی** | **نسبی**{.text-warning} | | -| E011-01 | عدد صحیح و SMALLINT انواع داده ها | بله {.text-success} | | -| E011-02 | انواع داده های دقیق و دوگانه واقعی و شناور | نسبی {.text-warning} | `FLOAT()`, `REAL` و `DOUBLE PRECISION` پشتیبانی نمیشود | -| E011-03 | دهدهی و انواع داده های عددی | نسبی {.text-warning} | فقط `DECIMAL(p,s)` پشتیبانی می شود, نه `NUMERIC` | -| E011-04 | اپراتورهای ریاضی | بله {.text-success} | | -| E011-05 | مقایسه عددی | بله {.text-success} | | -| E011-06 | ریخته گری ضمنی در میان انواع داده های عددی | نه {.text-danger} | انسی گذاشتن اجازه می دهد تا بازیگران ضمنی دلخواه بین انواع عددی, در حالی که تاتر متکی بر توابع داشتن اضافه بار متعدد به جای بازیگران ضمنی | -| **E021** | **انواع رشته شخصیت** | **نسبی**{.text-warning} | | -| E021-01 | نوع دادههای نویسه | نه {.text-danger} | | -| E021-02 | شخصیت های مختلف نوع داده ها | نه {.text-danger} | `String` رفتار مشابه, اما بدون محدودیت طول در پرانتز | -| E021-03 | شخصیت literals | نسبی {.text-warning} | بدون الحاق خودکار از لیتر متوالی و شخصیت پشتیبانی مجموعه | -| E021-04 | تابع _شخصی | نسبی {.text-warning} | نه `USING` بند | -| E021-05 | تابع اکتبر | نه {.text-danger} | `LENGTH` رفتار مشابه | -| E021-06 | SUBSTRING | نسبی {.text-warning} | هیچ پشتیبانی برای `SIMILAR` و `ESCAPE` بند نه `SUBSTRING_REGEX` گزینه | -| E021-07 | الحاق شخصیت | نسبی {.text-warning} | نه `COLLATE` بند | -| E021-08 | توابع بالا و پایین | بله {.text-success} | | -| E021-09 | تابع اصلاح | بله {.text-success} | | -| E021-10 | ریخته گری ضمنی در میان ثابت طول و متغیر طول انواع رشته شخصیت | نه {.text-danger} | انسی گذاشتن اجازه می دهد تا بازیگران ضمنی دلخواه بین انواع رشته, در حالی که تاتر متکی بر توابع داشتن اضافه بار متعدد به جای بازیگران ضمنی | -| E021-11 | تابع موقعیت | نسبی {.text-warning} | هیچ پشتیبانی برای `IN` و `USING` بند نه `POSITION_REGEX` گزینه | -| E021-12 | مقایسه شخصیت | بله {.text-success} | | -| **E031** | **شناسهها** | **نسبی**{.text-warning} | | -| E031-01 | شناسه های محدود | نسبی {.text-warning} | پشتیبانی تحت اللفظی یونیکد محدود است | -| E031-02 | شناسه های مورد پایین | بله {.text-success} | | -| E031-03 | انتهایی تاکید | بله {.text-success} | | -| **E051** | **مشخصات پرس و جو عمومی** | **نسبی**{.text-warning} | | -| E051-01 | SELECT DISTINCT | بله {.text-success} | | -| E051-02 | گروه بر اساس بند | بله {.text-success} | | -| E051-04 | گروه توسط می تواند ستون ها را شامل نمی شود `` | Evet {.text-success} | | -| E051-05 | Seçme öğeler yeniden adlandırılabilir | Evet {.text-success} | | -| E051-06 | Fık HAVİNGRA olması | Evet {.text-success} | | -| E051-07 | Nitelikli \* seçme listesinde | Evet {.text-success} | | -| E051-08 | From madd theesindeki korelasyon adı | Evet {.text-success} | | -| E051-09 | FROM yan tümcesinde sütunları Yeniden Adlandır | Hayır {.text-danger} | | -| **E061** | **Temel yüklemler ve arama koşulları** | **Kısmi**{.text-warning} | | -| E061-01 | Karşılaştırma yüklemi | Evet {.text-success} | | -| E061-02 | Yüklem arasında | Kısmi {.text-warning} | Hayır `SYMMETRIC` ve `ASYMMETRIC` yan | -| E061-03 | Değerler listesi ile yüklemde | Evet {.text-success} | | -| E061-04 | Yüklem gibi | Evet {.text-success} | | -| E061-05 | Yüklem gibi: kaçış maddesi | Hayır {.text-danger} | | -| E061-06 | Boş yüklem | Evet {.text-success} | | -| E061-07 | Sayısal karşılaştırma yüklemi | Hayır {.text-danger} | | -| E061-08 | Var yüklemi | Hayır {.text-danger} | | -| E061-09 | Karşılaştırma yükleminde alt sorgular | Evet {.text-success} | | -| E061-11 | Yüklemde alt sorgular | Evet {.text-success} | | -| E061-12 | Sayısal karşılaştırma yükleminde alt sorgular | Hayır {.text-danger} | | -| E061-13 | İlişkili alt sorgular | Hayır {.text-danger} | | -| E061-14 | Arama koşulu | Evet {.text-success} | | -| **E071** | **Temel sorgu ifadeleri** | **Kısmi**{.text-warning} | | -| E071-01 | Sendika farklı tablo operatörü | Hayır {.text-danger} | | -| E071-02 | UNİON ALL table operat operatoror | Evet {.text-success} | | -| E071-03 | Dist DİSTİNCTİNC tablet table operatörü hariç | Hayır {.text-danger} | | -| E071-05 | Tablo operatörleri ile birleştirilen sütunların tam olarak aynı veri türüne sahip olması gerekmez | Evet {.text-success} | | -| E071-06 | Alt sorgularda tablo işleçleri | Evet {.text-success} | | -| **E081** | **Temel ayrıcalıklar** | **Kısmi**{.text-warning} | Çalışmalar sürüyor | -| **E091** | **Set fonksiyonları** | **Evet**{.text-success} | | -| E091-01 | AVG | Evet {.text-success} | | -| E091-02 | COUNT | Evet {.text-success} | | -| E091-03 | MAX | Evet {.text-success} | | -| E091-04 | MIN | Evet {.text-success} | | -| E091-05 | SUM | Evet {.text-success} | | -| E091-06 | Tüm niceleyici | Hayır {.text-danger} | | -| E091-07 | Farklı niceleyici | Kısmi {.text-warning} | Tüm toplama işlevleri desteklenmiyor | -| **E101** | **Temel veri manipülasyonu** | **Kısmi**{.text-warning} | | -| E101-01 | INSERT deyimi | Evet {.text-success} | Not: Clickhouse'daki birincil anahtar, `UNIQUE` kısıtlama | -| E101-03 | Güncelleme deyimi Aran UPDATEDI | Hayır {.text-danger} | Bir `ALTER UPDATE` toplu veri değiştirme bildirimi | -| E101-04 | Aranan DELETE deyimi | Hayır {.text-danger} | Bir `ALTER DELETE` toplu veri kaldırma bildirimi | -| **E111** | **Tek sıra SELECT deyimi** | **Hayır**{.text-danger} | | -| **E121** | **Temel imleç desteği** | **Hayır**{.text-danger} | | -| E121-01 | DECLARE CURSOR | Hayır {.text-danger} | | -| E121-02 | Sütunlara göre siparişin seçim listesinde olması gerekmez | Hayır {.text-danger} | | -| E121-03 | CLA clauseuse by ORDER in Value ifadeleri | Hayır {.text-danger} | | -| E121-04 | Açık ifade | Hayır {.text-danger} | | -| E121-06 | Konumlandırılmış güncelleme bildirimi | Hayır {.text-danger} | | -| E121-07 | Konumlandırılmış silme deyimi | Hayır {.text-danger} | | -| E121-08 | Kapat deyimi | Hayır {.text-danger} | | -| E121-10 | FETCH deyimi: örtük sonraki | Hayır {.text-danger} | | -| E121-17 | Tut imleçler ile | Hayır {.text-danger} | | -| **E131** | **Boş değer desteği (değerler yerine boş değerler)** | **Kısmi**{.text-warning} | Bazı kısıtlamalar geçerlidir | -| **E141** | **Temel bütünlük kısıtlamaları** | **Kısmi**{.text-warning} | | -| E141-01 | NOT NULL kısıtlamaları | Evet {.text-success} | Not: `NOT NULL` tablo sütunları için varsayılan olarak ima edilir | -| E141-02 | NULL olmayan sütunların benzersiz kısıtlaması | Hayır {.text-danger} | | -| E141-03 | Birincil anahtar kısıtlamaları | Hayır {.text-danger} | | -| E141-04 | Hem referans silme eylemi hem de referans güncelleme eylemi için eylem yok varsayılanıyla temel yabancı anahtar kısıtlaması | Hayır {.text-danger} | | -| E141-06 | Kontrol kısıt CHECKLAMASI | Evet {.text-success} | | -| E141-07 | Sütun varsayılanları | Evet {.text-success} | | -| E141-08 | NOT NULL birincil anahtar üzerinde çıkarıldı | Evet {.text-success} | | -| E141-10 | Yabancı bir anahtardaki isimler herhangi bir sırada belirtilebilir | Hayır {.text-danger} | | -| **E151** | **İşlem desteği** | **Hayır**{.text-danger} | | -| E151-01 | Taahhüt deyimi | Hayır {.text-danger} | | -| E151-02 | ROLBACKL statementback deyimi | Hayır {.text-danger} | | -| **E152** | **Temel SET işlem deyimi** | **Hayır**{.text-danger} | | -| E152-01 | Set TRANSACTİON deyimi: izolasyon düzeyi SERİALİZABLE yan tümcesi | Hayır {.text-danger} | | -| E152-02 | Set TRANSACTİON deyimi: salt okunur ve okuma yazma yan tümceleri | Hayır {.text-danger} | | -| **E153** | **Alt sorgularla güncellenebilir sorgular** | **Hayır**{.text-danger} | | -| **E161** | **Lider çift eksi kullanarak SQL yorumlar ** | **Evet**{.text-success} | | -| **E171** | **SQLSTATE desteği** | **Hayır**{.text-danger} | | -| **E182** | **Ana bilgisayar dili bağlama** | **Hayır**{.text-danger} | | -| **F031** | **Temel şema manipülasyonu** | **Kısmi**{.text-warning} | | -| F031-01 | Kalıcı temel tablolar oluşturmak için tablo deyimi oluşturma | Kısmi {.text-warning} | Hayır `SYSTEM VERSIONING`, `ON COMMIT`, `GLOBAL`, `LOCAL`, `PRESERVE`, `DELETE`, `REF IS`, `WITH OPTIONS`, `UNDER`, `LIKE`, `PERIOD FOR` yan tümceleri ve kullanıcı çözümlenmiş veri türleri için destek yok | -| F031-02 | Görünüm deyimi oluştur | Kısmi {.text-warning} | Hayır `RECURSIVE`, `CHECK`, `UNDER`, `WITH OPTIONS` yan tümceleri ve kullanıcı çözümlenmiş veri türleri için destek yok | -| F031-03 | Hibe beyanı | Evet {.text-success} | | -| F031-04 | ALTER TABLE deyimi: sütun yan tümcesi Ekle | Kısmi {.text-warning} | İçin destek yok `GENERATED` fık andra ve sistem süresi | -| F031-13 | Dro :p TABLE deyimi: kısıt :lamak | Hayır {.text-danger} | | -| F031-16 | Dro :p VİEW deyimi: kısıt :lamak | Hayır {.text-danger} | | -| F031-19 | Rev REVOKEOKE deyimi: kısıt clauselamak | Hayır {.text-danger} | | -| **F041** | **Temel birleştirilmiş tablo** | **Kısmi**{.text-warning} | | -| F041-01 | Inner join (ancak mutlaka iç anahtar kelime değil) | Evet {.text-success} | | -| F041-02 | İç anahtar kelime | Evet {.text-success} | | -| F041-03 | LEFT OUTER JOIN | Evet {.text-success} | | -| F041-04 | RIGHT OUTER JOIN | Evet {.text-success} | | -| F041-05 | Dış birleşimler iç içe geçmiş olabilir | Evet {.text-success} | | -| F041-07 | Sol veya sağ dış birleşimdeki iç tablo, bir iç birleşimde de kullanılabilir | Evet {.text-success} | | -| F041-08 | Tüm karşılaştırma operatörleri desteklenir (sadece =yerine) | Hayır {.text-danger} | | -| **F051** | **Temel tarih ve saat** | **Kısmi**{.text-warning} | | -| F051-01 | Tarih veri türü (tarih literal desteği dahil) | Kısmi {.text-warning} | Hiçbir edebi | -| F051-02 | En az 0 kesirli saniye hassasiyetle zaman veri türü (zaman literal desteği dahil) | Hayır {.text-danger} | | -| F051-03 | Zaman damgası veri türü (zaman damgası literal desteği dahil) en az 0 ve 6 kesirli saniye hassasiyetle | Hayır {.text-danger} | `DateTime64` zaman benzer işlevsellik sağlar | -| F051-04 | Tarih, Saat ve zaman damgası veri türlerinde karşılaştırma yüklemi | Kısmi {.text-warning} | Yalnızca bir veri türü kullanılabilir | -| F051-05 | Datetime türleri ve karakter dizesi türleri arasında açık döküm | Evet {.text-success} | | -| F051-06 | CURRENT_DATE | Hayır {.text-danger} | `today()` benzer mi | -| F051-07 | LOCALTIME | Hayır {.text-danger} | `now()` benzer mi | -| F051-08 | LOCALTIMESTAMP | Hayır {.text-danger} | | -| **F081** | **Sendika ve görüş EXCEPTLERDE** | **Kısmi**{.text-warning} | | -| **F131** | **Grup operationslandırılmış işlemler** | **Kısmi**{.text-warning} | | -| F131-01 | WHERE, GROUP BY ve gruplandırılmış görünümlere sahip sorgularda desteklenen yan tümceleri olması | Evet {.text-success} | | -| F131-02 | Gruplandırılmış görünümlere sahip sorgularda desteklenen birden çok tablo | Evet {.text-success} | | -| F131-03 | Gruplandırılmış görünümlere sahip sorgularda desteklenen işlevleri ayarlayın | Evet {.text-success} | | -| F131-04 | GROUP BY ile alt sorgular ve yan tümceleri ve gruplandırılmış görünümler | Evet {.text-success} | | -| F131-05 | GROUP BY ile tek satır seçme ve yan tümceleri ve gruplandırılmış görünümleri sahip | Hayır {.text-danger} | | -| **F181** | **Çoklu modül desteği** | **Hayır**{.text-danger} | | -| **F201** | **Döküm fonksiyonu** | **Evet**{.text-success} | | -| **F221** | **Açık varsayılan** | **Hayır**{.text-danger} | | -| **F261** | **Durum ifadesi** | **Evet**{.text-success} | | -| F261-01 | Basit durum | Evet {.text-success} | | -| F261-02 | Aranan dava | Evet {.text-success} | | -| F261-03 | NULLIF | Evet {.text-success} | | -| F261-04 | COALESCE | Evet {.text-success} | | -| **F311** | **Şema tanımı deyimi** | **Kısmi**{.text-warning} | | -| F311-01 | CREATE SCHEMA | Hayır {.text-danger} | | -| F311-02 | Kalıcı temel tablolar için tablo oluşturma | Evet {.text-success} | | -| F311-03 | CREATE VIEW | Evet {.text-success} | | -| F311-04 | CREATE VIEW: WITH CHECK OPTION | Hayır {.text-danger} | | -| F311-05 | Hibe beyanı | Evet {.text-success} | | -| **F471** | **Skaler alt sorgu değerleri** | **Evet**{.text-success} | | -| **F481** | **Genişletilmiş boş yüklem** | **Evet**{.text-success} | | -| **F812** | **Temel işaretleme** | **Hayır**{.text-danger} | | -| **T321** | **Temel SQL-çağrılan rutinleri** | **Hayır**{.text-danger} | | -| T321-01 | Hiçbir aşırı yükleme ile kullanıcı tanımlı fonksiyonlar | Hayır {.text-danger} | | -| T321-02 | Hiçbir aşırı yükleme ile kullanıcı tanımlı saklı yordamlar | Hayır {.text-danger} | | -| T321-03 | Fonksiyon çağırma | Hayır {.text-danger} | | -| T321-04 | Çağrı bildirimi | Hayır {.text-danger} | | -| T321-05 | Ret statementurn deyimi | Hayır {.text-danger} | | -| **T631** | **Bir liste öğesi ile yüklemde** | **Evet**{.text-success} | | diff --git a/docs/tr/sql-reference/data-types/aggregatefunction.md b/docs/tr/sql-reference/data-types/aggregatefunction.md deleted file mode 100644 index 45847352ad5..00000000000 --- a/docs/tr/sql-reference/data-types/aggregatefunction.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 52 -toc_title: AggregateFunction (ad, types_of_arguments...) ---- - -# AggregateFunction(name, types_of_arguments…) {#data-type-aggregatefunction} - -Aggregate functions can have an implementation-defined intermediate state that can be serialized to an AggregateFunction(…) data type and stored in a table, usually, by means of [materyalize bir görünüm](../../sql-reference/statements/create.md#create-view). Bir toplama işlevi durumu üretmek için ortak yolu ile toplama işlevi çağırarak olduğunu `-State` sonek. Gelecekte toplanmanın nihai sonucunu elde etmek için, aynı toplama işlevini `-Merge`sonek. - -`AggregateFunction` — parametric data type. - -**Parametre** - -- Toplama işlevinin adı. - - If the function is parametric, specify its parameters too. - -- Toplama işlevi bağımsız değişkenleri türleri. - -**Örnek** - -``` sql -CREATE TABLE t -( - column1 AggregateFunction(uniq, UInt64), - column2 AggregateFunction(anyIf, String, UInt8), - column3 AggregateFunction(quantiles(0.5, 0.9), UInt64) -) ENGINE = ... -``` - -[uniq](../../sql-reference/aggregate-functions/reference.md#agg_function-uniq), anyİf ([herhangi](../../sql-reference/aggregate-functions/reference.md#agg_function-any)+[Eğer](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-if)) ve [quantiles](../../sql-reference/aggregate-functions/reference.md) ClickHouse desteklenen toplam işlevleri vardır. - -## Kullanma {#usage} - -### Veri Ekleme {#data-insertion} - -Veri eklemek için şunları kullanın `INSERT SELECT` agr aggregateega ile `-State`- işlevler. - -**Fonksiyon örnekleri** - -``` sql -uniqState(UserID) -quantilesState(0.5, 0.9)(SendTiming) -``` - -Karşılık gelen fonksiyonların aksine `uniq` ve `quantiles`, `-State`- fonksiyonlar son değer yerine durumu döndürür. Başka bir deyişle, bir değer döndürür `AggregateFunction` tür. - -Sonuç inlarında `SELECT` sorgu, değerleri `AggregateFunction` türü, Tüm ClickHouse çıktı biçimleri için uygulamaya özgü ikili gösterime sahiptir. Örneğin, veri dökümü, `TabSeparated` ile format `SELECT` sorgu, daha sonra bu dökümü kullanarak geri yüklenebilir `INSERT` sorgu. - -### Veri Seçimi {#data-selection} - -Veri seçerken `AggregatingMergeTree` tablo kullanın `GROUP BY` yan tümce ve veri eklerken aynı toplama işlevleri, ancak kullanarak `-Merge`sonek. - -Bir toplama fonksiyonu ile `-Merge` sonek, bir dizi durum alır, bunları birleştirir ve tam veri toplama sonucunu döndürür. - -Örneğin, aşağıdaki iki sorgu aynı sonucu döndürür: - -``` sql -SELECT uniq(UserID) FROM table - -SELECT uniqMerge(state) FROM (SELECT uniqState(UserID) AS state FROM table GROUP BY RegionID) -``` - -## Kullanım Örneği {#usage-example} - -Görmek [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) motor açıklaması. - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/nested_data_structures/aggregatefunction/) diff --git a/docs/tr/sql-reference/data-types/array.md b/docs/tr/sql-reference/data-types/array.md deleted file mode 100644 index 718a41b0bb7..00000000000 --- a/docs/tr/sql-reference/data-types/array.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 51 -toc_title: Dizi(T) ---- - -# Dizi(t) {#data-type-array} - -Bir dizi `T`- tip öğeleri. `T` herhangi bir veri türü, bir dizi dahil edilebilir. - -## Bir dizi oluşturma {#creating-an-array} - -Bir dizi oluşturmak için bir işlev kullanabilirsiniz: - -``` sql -array(T) -``` - -Köşeli parantez de kullanabilirsiniz. - -``` sql -[] -``` - -Bir dizi oluşturma örneği: - -``` sql -SELECT array(1, 2) AS x, toTypeName(x) -``` - -``` text -┌─x─────┬─toTypeName(array(1, 2))─┐ -│ [1,2] │ Array(UInt8) │ -└───────┴─────────────────────────┘ -``` - -``` sql -SELECT [1, 2] AS x, toTypeName(x) -``` - -``` text -┌─x─────┬─toTypeName([1, 2])─┐ -│ [1,2] │ Array(UInt8) │ -└───────┴────────────────────┘ -``` - -## Veri türleri ile çalışma {#working-with-data-types} - -Anında bir dizi oluştururken, ClickHouse bağımsız değişken türünü otomatik olarak listelenen tüm bağımsız değişkenleri depolayabilen en dar veri türü olarak tanımlar. Eğer herhangi bir [Nullable](nullable.md#data_type-nullable) veya edebi [NULL](../../sql-reference/syntax.md#null-literal) değerler, bir dizi öğesinin türü de olur [Nullable](nullable.md). - -ClickHouse veri türünü belirleyemedi, bir özel durum oluşturur. Örneğin, aynı anda dizeler ve sayılarla bir dizi oluşturmaya çalışırken bu olur (`SELECT array(1, 'a')`). - -Otomatik veri türü algılama örnekleri: - -``` sql -SELECT array(1, 2, NULL) AS x, toTypeName(x) -``` - -``` text -┌─x──────────┬─toTypeName(array(1, 2, NULL))─┐ -│ [1,2,NULL] │ Array(Nullable(UInt8)) │ -└────────────┴───────────────────────────────┘ -``` - -Uyumsuz veri türleri dizisi oluşturmaya çalışırsanız, ClickHouse bir özel durum atar: - -``` sql -SELECT array(1, 'a') -``` - -``` text -Received exception from server (version 1.1.54388): -Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: There is no supertype for types UInt8, String because some of them are String/FixedString and some of them are not. -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/array/) diff --git a/docs/tr/sql-reference/data-types/boolean.md b/docs/tr/sql-reference/data-types/boolean.md deleted file mode 100644 index 8bf711b9060..00000000000 --- a/docs/tr/sql-reference/data-types/boolean.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 43 -toc_title: Boeanoleanean ---- - -# Boole Değerleri {#boolean-values} - -Boole değerleri için ayrı bir tür yoktur. 0 veya 1 değerleriyle sınırlı Uİnt8 türünü kullanın. - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/boolean/) diff --git a/docs/tr/sql-reference/data-types/date.md b/docs/tr/sql-reference/data-types/date.md deleted file mode 100644 index 05827f5b353..00000000000 --- a/docs/tr/sql-reference/data-types/date.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 47 -toc_title: Tarihli ---- - -# Tarihli {#date} - -Tarihli. 1970-01-01 (imzasız) gün sayısı olarak iki bayt olarak saklanır. Unix döneminin başlangıcından hemen sonra, derleme aşamasında bir sabit tarafından tanımlanan üst eşiğe kadar değerlerin depolanmasına izin verir (şu anda, bu 2106 yılına kadar, ancak tam olarak desteklenen son yıl 2105'tir). - -Tarih değeri saat dilimi olmadan depolanır. - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/date/) diff --git a/docs/tr/sql-reference/data-types/datetime.md b/docs/tr/sql-reference/data-types/datetime.md deleted file mode 100644 index 52656d2afb7..00000000000 --- a/docs/tr/sql-reference/data-types/datetime.md +++ /dev/null @@ -1,129 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 48 -toc_title: DateTime ---- - -# Datetime {#data_type-datetime} - -Bir takvim tarih ve bir günün bir saat olarak ifade edilebilir, zaman içinde bir anlık saklamak için izin verir. - -Sözdizimi: - -``` sql -DateTime([timezone]) -``` - -Desteklenen değerler aralığı: \[1970-01-01 00:00:00, 2105-12-31 23:59:59\]. - -Çözünürlük: 1 saniye. - -## Kullanım Açıklamaları {#usage-remarks} - -Zaman içindeki nokta bir [Unix zaman damgası](https://en.wikipedia.org/wiki/Unix_time), ne olursa olsun saat dilimi veya gün ışığından yararlanma saati. Ayrıca, `DateTime` tür, tüm sütun için aynı olan saat dilimini depolayabilir, bu da `DateTime` tür değerleri metin biçiminde görüntülenir ve dizeler olarak belirtilen değerlerin nasıl ayrıştırılır (‘2020-01-01 05:00:01’). Saat dilimi tablo (veya resultset) satırlarında depolanır, ancak sütun meta verileri depolanır. -Desteklenen saat dilimlerinin bir listesi şu adreste bulunabilir: [IANA Saat Dilimi veritabanı](https://www.iana.org/time-zones). -Bu `tzdata` paket, içeren [IANA Saat Dilimi veritabanı](https://www.iana.org/time-zones), sisteme Kurul .malıdır. Kullan... `timedatectl list-timezones` yerel bir sistem tarafından bilinen zaman dilimlerini listelemek için komut. - -İçin bir saat dilimi açıkça ayarlayabilirsiniz `DateTime`- bir tablo oluştururken sütunları yazın. Saat dilimi ayarlanmamışsa, ClickHouse değerini kullanır [saat dilimi](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) sunucu ayarlarında veya ClickHouse sunucusunun başlatıldığı anda işletim sistemi ayarlarında parametre. - -Bu [clickhouse-müşteri](../../interfaces/cli.md) veri türünü başlatırken bir saat dilimi açıkça ayarlanmamışsa, sunucu saat dilimini varsayılan olarak uygular. İstemci saat dilimini kullanmak için `clickhouse-client` ile... `--use_client_time_zone` parametre. - -ClickHouse çıkış değerleri `YYYY-MM-DD hh:mm:ss` varsayılan olarak metin biçimi. Çıkış ile değiştirebilirsiniz [formatDateTime](../../sql-reference/functions/date-time-functions.md#formatdatetime) İşlev. - -Clickhouse'a veri eklerken, Tarih ve saat dizelerinin farklı biçimlerini kullanabilirsiniz. [date_time_input_format](../../operations/settings/settings.md#settings-date_time_input_format) ayar. - -## Örnekler {#examples} - -**1.** Bir tablo ile bir tablo oluşturma `DateTime`- sütun yazın ve içine veri ekleme: - -``` sql -CREATE TABLE dt -( - `timestamp` DateTime('Europe/Moscow'), - `event_id` UInt8 -) -ENGINE = TinyLog; -``` - -``` sql -INSERT INTO dt Values (1546300800, 1), ('2019-01-01 00:00:00', 2); -``` - -``` sql -SELECT * FROM dt; -``` - -``` text -┌───────────timestamp─┬─event_id─┐ -│ 2019-01-01 03:00:00 │ 1 │ -│ 2019-01-01 00:00:00 │ 2 │ -└─────────────────────┴──────────┘ -``` - -- Bir tamsayı olarak datetime eklerken, Unıx Zaman Damgası (UTC) olarak kabul edilir. `1546300800` temsil etmek `'2019-01-01 00:00:00'` UTC. Ancak, `timestamp` sütun vardır `Europe/Moscow` (UTC+3) belirtilen zaman dilimi, dize olarak çıkış yaparken değer olarak gösterilecektir `'2019-01-01 03:00:00'` -- Dize değerini datetime olarak eklerken, sütun saat diliminde olduğu kabul edilir. `'2019-01-01 00:00:00'` will gibi muamele `Europe/Moscow` saat dilimi ve farklı kaydedildi `1546290000`. - -**2.** Üzerinde filtreleme `DateTime` değerler - -``` sql -SELECT * FROM dt WHERE timestamp = toDateTime('2019-01-01 00:00:00', 'Europe/Moscow') -``` - -``` text -┌───────────timestamp─┬─event_id─┐ -│ 2019-01-01 00:00:00 │ 2 │ -└─────────────────────┴──────────┘ -``` - -`DateTime` sütun değerleri, bir dize değeri kullanılarak filtrelenebilir `WHERE` yüklem. Dönüştürül willecektir `DateTime` otomatik olarak: - -``` sql -SELECT * FROM dt WHERE timestamp = '2019-01-01 00:00:00' -``` - -``` text -┌───────────timestamp─┬─event_id─┐ -│ 2019-01-01 03:00:00 │ 1 │ -└─────────────────────┴──────────┘ -``` - -**3.** Bir saat dilimi almak `DateTime`- type Col columnum columnn: - -``` sql -SELECT toDateTime(now(), 'Europe/Moscow') AS column, toTypeName(column) AS x -``` - -``` text -┌──────────────column─┬─x─────────────────────────┐ -│ 2019-10-16 04:12:04 │ DateTime('Europe/Moscow') │ -└─────────────────────┴───────────────────────────┘ -``` - -**4.** Zaman dilimi dönüştürme - -``` sql -SELECT -toDateTime(timestamp, 'Europe/London') as lon_time, -toDateTime(timestamp, 'Europe/Moscow') as mos_time -FROM dt -``` - -``` text -┌───────────lon_time──┬────────────mos_time─┐ -│ 2019-01-01 00:00:00 │ 2019-01-01 03:00:00 │ -│ 2018-12-31 21:00:00 │ 2019-01-01 00:00:00 │ -└─────────────────────┴─────────────────────┘ -``` - -## Ayrıca Bakınız {#see-also} - -- [Tip dönüştürme fonksiyonları](../../sql-reference/functions/type-conversion-functions.md) -- [Tarih ve saatlerle çalışmak için işlevler](../../sql-reference/functions/date-time-functions.md) -- [Dizilerle çalışmak için işlevler](../../sql-reference/functions/array-functions.md) -- [Bu `date_time_input_format` ayar](../../operations/settings/settings.md#settings-date_time_input_format) -- [Bu `timezone` sunucu yapılandırma parametresi](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) -- [Tarih ve saatlerle çalışmak için operatörler](../../sql-reference/operators/index.md#operators-datetime) -- [Bu `Date` veri türü](date.md) - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/datetime/) diff --git a/docs/tr/sql-reference/data-types/datetime64.md b/docs/tr/sql-reference/data-types/datetime64.md deleted file mode 100644 index 28c95e0df22..00000000000 --- a/docs/tr/sql-reference/data-types/datetime64.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 49 -toc_title: DateTime64 ---- - -# Datetime64 {#data_type-datetime64} - -Tanımlanmış alt saniye hassasiyetle, bir takvim tarihi ve bir günün saati olarak ifade edilebilir, zaman içinde bir anlık saklamak için izin verir - -Kene boyutu (hassas): 10-hassaslık ikincilikler - -Sözdizimi: - -``` sql -DateTime64(precision, [timezone]) -``` - -DAHİLİ olarak, verileri bir dizi olarak saklar ‘ticks’ epoch başlangıçtan beri (1970-01-01 00:00:00 UTC) Int64 olarak. Kene çözünürlüğü hassasiyet parametresi tarafından belirlenir. Ayrıca, `DateTime64` tür, tüm sütun için aynı olan saat dilimini depolayabilir, bu da `DateTime64` tür değerleri metin biçiminde görüntülenir ve dizeler olarak belirtilen değerlerin nasıl ayrıştırılır (‘2020-01-01 05:00:01.000’). Saat dilimi tablo (veya resultset) satırlarında depolanır, ancak sütun meta verileri depolanır. Ayrıntıları görün [DateTime](datetime.md). - -## Örnekler {#examples} - -**1.** İle bir tablo oluşturma `DateTime64`- sütun yazın ve içine veri ekleme: - -``` sql -CREATE TABLE dt -( - `timestamp` DateTime64(3, 'Europe/Moscow'), - `event_id` UInt8 -) -ENGINE = TinyLog -``` - -``` sql -INSERT INTO dt Values (1546300800000, 1), ('2019-01-01 00:00:00', 2) -``` - -``` sql -SELECT * FROM dt -``` - -``` text -┌───────────────timestamp─┬─event_id─┐ -│ 2019-01-01 03:00:00.000 │ 1 │ -│ 2019-01-01 00:00:00.000 │ 2 │ -└─────────────────────────┴──────────┘ -``` - -- Bir tamsayı olarak datetime eklerken, uygun şekilde ölçeklendirilmiş bir Unıx Zaman Damgası (UTC) olarak kabul edilir. `1546300800000` (hassas 3 ile) temsil eder `'2019-01-01 00:00:00'` UTC. Ancak, `timestamp` sütun vardır `Europe/Moscow` (UTC+3) belirtilen zaman dilimi, bir dize olarak çıkış yaparken değer olarak gösterilir `'2019-01-01 03:00:00'` -- Dize değerini datetime olarak eklerken, sütun saat diliminde olduğu kabul edilir. `'2019-01-01 00:00:00'` will gibi muamele `Europe/Moscow` saat dilimi ve olarak saklanır `1546290000000`. - -**2.** Üzerinde filtreleme `DateTime64` değerler - -``` sql -SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Europe/Moscow') -``` - -``` text -┌───────────────timestamp─┬─event_id─┐ -│ 2019-01-01 00:00:00.000 │ 2 │ -└─────────────────────────┴──────────┘ -``` - -Aksine `DateTime`, `DateTime64` değerler dönüştürülmez `String` otomatik olarak - -**3.** Bir saat dilimi almak `DateTime64`- tip değeri: - -``` sql -SELECT toDateTime64(now(), 3, 'Europe/Moscow') AS column, toTypeName(column) AS x -``` - -``` text -┌──────────────────column─┬─x──────────────────────────────┐ -│ 2019-10-16 04:12:04.000 │ DateTime64(3, 'Europe/Moscow') │ -└─────────────────────────┴────────────────────────────────┘ -``` - -**4.** Zaman dilimi dönüştürme - -``` sql -SELECT -toDateTime64(timestamp, 3, 'Europe/London') as lon_time, -toDateTime64(timestamp, 3, 'Europe/Moscow') as mos_time -FROM dt -``` - -``` text -┌───────────────lon_time──┬────────────────mos_time─┐ -│ 2019-01-01 00:00:00.000 │ 2019-01-01 03:00:00.000 │ -│ 2018-12-31 21:00:00.000 │ 2019-01-01 00:00:00.000 │ -└─────────────────────────┴─────────────────────────┘ -``` - -## Ayrıca Bakınız {#see-also} - -- [Tip dönüştürme fonksiyonları](../../sql-reference/functions/type-conversion-functions.md) -- [Tarih ve saatlerle çalışmak için işlevler](../../sql-reference/functions/date-time-functions.md) -- [Dizilerle çalışmak için işlevler](../../sql-reference/functions/array-functions.md) -- [Bu `date_time_input_format` ayar](../../operations/settings/settings.md#settings-date_time_input_format) -- [Bu `timezone` sunucu yapılandırma parametresi](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) -- [Tarih ve saatlerle çalışmak için operatörler](../../sql-reference/operators/index.md#operators-datetime) -- [`Date` veri türü](date.md) -- [`DateTime` veri türü](datetime.md) diff --git a/docs/tr/sql-reference/data-types/decimal.md b/docs/tr/sql-reference/data-types/decimal.md deleted file mode 100644 index 805d0ec30b5..00000000000 --- a/docs/tr/sql-reference/data-types/decimal.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 42 -toc_title: "Ondal\u0131k" ---- - -# Ondalık(P, S), Decimal32 (S), Decimal64( S), Decimal128 (S) {#decimalp-s-decimal32s-decimal64s-decimal128s} - -Ekleme, çıkarma ve çarpma işlemleri sırasında hassasiyeti koruyan imzalı sabit noktalı sayılar. Bölünme için en az önemli basamak atılır (yuvarlatılmamış). - -## Parametre {#parameters} - -- P-hassas. Geçerli Aralık: \[1: 38 \]. Kaç ondalık basamak sayısı (kesir dahil) olabilir belirler. -- S-scale. Geçerli Aralık: \[0: P\]. Kaç ondalık basamak kesir olabilir belirler. - -P parametre değerine bağlı olarak ondalık (P, S) bir eşanlamlıdır: -- P \[ 1 : 9\] - Decimal32(S) için) -- P \[ 10 : 18\] - Decimal64(ler) için) -- P \[ 19 : 38\] - Decimal128(ler) için) - -## Ondalık Değer Aralıkları {#decimal-value-ranges} - -- Decimal32(S) - ( -1 \* 10^(9 - S), 1 \* 10^(9-S) ) -- Decimal64(S) - ( -1 \* 10^(18 - S), 1 \* 10^(18-S) ) -- Decimal128(S) - ( -1 \* 10^(38 - S), 1 \* 10^(38-S) ) - -Örneğin, Decimal32 (4) -99999.9999 99999.9999 0.0001 adım ile sayılar içerebilir. - -## İç Temsil {#internal-representation} - -Dahili veri, ilgili bit genişliğine sahip normal imzalı tamsayılar olarak temsil edilir. Bellekte saklanabilen gerçek değer aralıkları, yukarıda belirtilenden biraz daha büyüktür ve yalnızca bir dizeden dönüştürmede kontrol edilir. - -Modern CPU 128-bit tamsayıları doğal olarak desteklemediğinden, Decimal128 üzerindeki işlemler öykünülür. Bu Decimal128 nedeniyle Decimal32/Decimal64'ten önemli ölçüde daha yavaş çalışır. - -## İşlemler ve sonuç türü {#operations-and-result-type} - -Ondalık sonuçtaki ikili işlemler daha geniş sonuç türünde (herhangi bir bağımsız değişken sırası ile) sonuçlanır. - -- `Decimal64(S1) Decimal32(S2) -> Decimal64(S)` -- `Decimal128(S1) Decimal32(S2) -> Decimal128(S)` -- `Decimal128(S1) Decimal64(S2) -> Decimal128(S)` - -Ölçek kuralları: - -- ekleme, çıkarma: s = max (S1, S2). -- multuply: S = S1 + S2. -- böl: S = S1. - -Ondalık ve tamsayılar arasındaki benzer işlemler için sonuç, bir bağımsız değişkenle aynı boyutta ondalık olur. - -Ondalık ve Float32 / Float64 arasındaki işlemler tanımlanmamıştır. Bunlara ihtiyacınız varsa, todecimal32, toDecimal64, toDecimal128 veya toFloat32, toFloat64 builtins kullanarak bağımsız değişkenlerden birini açıkça yayınlayabilirsiniz. Sonucun hassasiyeti kaybedeceğini ve tür dönüşümünün hesaplamalı olarak pahalı bir işlem olduğunu unutmayın. - -Float64 (örneğin, var veya stddev) ondalık dönüş sonucu bazı işlevler. Ara hesaplamalar hala Float64 ve aynı değerlere sahip ondalık girişler arasında farklı sonuçlara yol açabilecek ondalık olarak gerçekleştirilebilir. - -## Taşma Kontrolleri {#overflow-checks} - -Ondalık hesaplamalar sırasında tamsayı taşmaları gerçekleşebilir. Bir kesirdeki aşırı rakamlar atılır (yuvarlatılmamış). Tamsayı bölümünde aşırı basamak bir istisna yol açacaktır. - -``` sql -SELECT toDecimal32(2, 4) AS x, x / 3 -``` - -``` text -┌──────x─┬─divide(toDecimal32(2, 4), 3)─┐ -│ 2.0000 │ 0.6666 │ -└────────┴──────────────────────────────┘ -``` - -``` sql -SELECT toDecimal32(4.2, 8) AS x, x * x -``` - -``` text -DB::Exception: Scale is out of bounds. -``` - -``` sql -SELECT toDecimal32(4.2, 8) AS x, 6 * x -``` - -``` text -DB::Exception: Decimal math overflow. -``` - -Taşma kontrolleri operasyonların yavaşlamasına neden olur. Taşmaların mümkün olmadığı biliniyorsa, kontrolleri kullanarak devre dışı bırakmak mantıklıdır `decimal_check_overflow` ayar. Kontroller devre dışı bırakıldığında ve taşma gerçekleştiğinde, sonuç yanlış olacaktır: - -``` sql -SET decimal_check_overflow = 0; -SELECT toDecimal32(4.2, 8) AS x, 6 * x -``` - -``` text -┌──────────x─┬─multiply(6, toDecimal32(4.2, 8))─┐ -│ 4.20000000 │ -17.74967296 │ -└────────────┴──────────────────────────────────┘ -``` - -Taşma kontrolleri sadece aritmetik işlemlerde değil, değer karşılaştırmasında da gerçekleşir: - -``` sql -SELECT toDecimal32(1, 8) < 100 -``` - -``` text -DB::Exception: Can't compare. -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/decimal/) diff --git a/docs/tr/sql-reference/data-types/domains/index.md b/docs/tr/sql-reference/data-types/domains/index.md deleted file mode 100644 index 7ef688b3578..00000000000 --- a/docs/tr/sql-reference/data-types/domains/index.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 58 -toc_title: "Genel bak\u0131\u015F" -toc_folder_title: Etkiler -toc_priority: 56 ---- - -# Etkiler {#domains} - -Etki alanları, varolan temel türün üstüne bazı ek özellikler ekleyen, ancak temel veri türünün kablolu ve disk üstü biçimini sağlam bırakan özel amaçlı türlerdir. Şu anda, ClickHouse kullanıcı tanımlı etki alanlarını desteklemiyor. - -Örneğin, ilgili taban türünün kullanılabileceği her yerde etki alanlarını kullanabilirsiniz: - -- Etki alanı türünde bir sütun oluşturma -- Alan sütunundan/alanına değerleri okuma / yazma -- Bir temel türü bir dizin olarak kullanılabilir, bir dizin olarak kullanın -- Etki alanı sütun değerleri ile çağrı fonksiyonları - -### Alanların ekstra özellikleri {#extra-features-of-domains} - -- Açık sütun türü adı `SHOW CREATE TABLE` veya `DESCRIBE TABLE` -- İle insan dostu format inputtan giriş `INSERT INTO domain_table(domain_column) VALUES(...)` -- İçin insan dostu forma outputta çıktı `SELECT domain_column FROM domain_table` -- Harici bir kaynaktan insan dostu biçimde veri yükleme: `INSERT INTO domain_table FORMAT CSV ...` - -### Sınırlamalar {#limitations} - -- Temel türün dizin sütununu etki alanı türüne dönüştürülemiyor `ALTER TABLE`. -- Başka bir sütun veya tablodan veri eklerken dize değerlerini dolaylı olarak etki alanı değerlerine dönüştüremez. -- Etki alanı, depolanan değerler üzerinde hiçbir kısıtlama ekler. - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/domains/overview) diff --git a/docs/tr/sql-reference/data-types/domains/ipv4.md b/docs/tr/sql-reference/data-types/domains/ipv4.md deleted file mode 100644 index 4caf031c0c3..00000000000 --- a/docs/tr/sql-reference/data-types/domains/ipv4.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 59 -toc_title: Ipv44 ---- - -## Ipv44 {#ipv4} - -`IPv4` dayalı bir doma aindir `UInt32` tip ve IPv4 değerlerini depolamak için yazılan bir yedek olarak hizmet eder. İnsan dostu giriş-çıkış biçimi ve muayene ile ilgili sütun tipi bilgileri ile kompakt depolama sağlar. - -### Temel Kullanım {#basic-usage} - -``` sql -CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY url; - -DESCRIBE TABLE hits; -``` - -``` text -┌─name─┬─type───┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┐ -│ url │ String │ │ │ │ │ -│ from │ IPv4 │ │ │ │ │ -└──────┴────────┴──────────────┴────────────────────┴─────────┴──────────────────┘ -``` - -Veya IPv4 etki alanını anahtar olarak kullanabilirsiniz: - -``` sql -CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY from; -``` - -`IPv4` etki alanı IPv4 dizeleri olarak özel giriş biçimini destekler: - -``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.tech/docs/en/', '116.106.34.242'); - -SELECT * FROM hits; -``` - -``` text -┌─url────────────────────────────────┬───────────from─┐ -│ https://clickhouse.tech/docs/en/ │ 116.106.34.242 │ -│ https://wikipedia.org │ 116.253.40.133 │ -│ https://clickhouse.tech │ 183.247.232.58 │ -└────────────────────────────────────┴────────────────┘ -``` - -Değerler kompakt ikili formda saklanır: - -``` sql -SELECT toTypeName(from), hex(from) FROM hits LIMIT 1; -``` - -``` text -┌─toTypeName(from)─┬─hex(from)─┐ -│ IPv4 │ B7F7E83A │ -└──────────────────┴───────────┘ -``` - -Etki alanı değerleri örtülü olarak dışındaki türlere dönüştürülemez `UInt32`. -Dönüştürmek istiyorsanız `IPv4` bir dizeye değer, bunu açıkça yapmak zorundasınız `IPv4NumToString()` işlev: - -``` sql -SELECT toTypeName(s), IPv4NumToString(from) as s FROM hits LIMIT 1; -``` - - ┌─toTypeName(IPv4NumToString(from))─┬─s──────────────┐ - │ String │ 183.247.232.58 │ - └───────────────────────────────────┴────────────────┘ - -Ya da bir döküm `UInt32` değer: - -``` sql -SELECT toTypeName(i), CAST(from as UInt32) as i FROM hits LIMIT 1; -``` - -``` text -┌─toTypeName(CAST(from, 'UInt32'))─┬──────────i─┐ -│ UInt32 │ 3086477370 │ -└──────────────────────────────────┴────────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/domains/ipv4) diff --git a/docs/tr/sql-reference/data-types/domains/ipv6.md b/docs/tr/sql-reference/data-types/domains/ipv6.md deleted file mode 100644 index 7f721cc07f6..00000000000 --- a/docs/tr/sql-reference/data-types/domains/ipv6.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 60 -toc_title: IPv6 ---- - -## IPv6 {#ipv6} - -`IPv6` dayalı bir doma aindir `FixedString(16)` tip ve IPv6 değerlerini depolamak için yazılan bir yedek olarak hizmet eder. İnsan dostu giriş-çıkış biçimi ve muayene ile ilgili sütun tipi bilgileri ile kompakt depolama sağlar. - -### Temel Kullanım {#basic-usage} - -``` sql -CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY url; - -DESCRIBE TABLE hits; -``` - -``` text -┌─name─┬─type───┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┐ -│ url │ String │ │ │ │ │ -│ from │ IPv6 │ │ │ │ │ -└──────┴────────┴──────────────┴────────────────────┴─────────┴──────────────────┘ -``` - -Veya kullanabilirsiniz `IPv6` anahtar olarak etki alanı: - -``` sql -CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from; -``` - -`IPv6` etki alanı IPv6 dizeleri olarak özel girişi destekler: - -``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.tech/docs/en/', '2a02:e980:1e::1'); - -SELECT * FROM hits; -``` - -``` text -┌─url────────────────────────────────┬─from──────────────────────────┐ -│ https://clickhouse.tech │ 2001:44c8:129:2632:33:0:252:2 │ -│ https://clickhouse.tech/docs/en/ │ 2a02:e980:1e::1 │ -│ https://wikipedia.org │ 2a02:aa08:e000:3100::2 │ -└────────────────────────────────────┴───────────────────────────────┘ -``` - -Değerler kompakt ikili formda saklanır: - -``` sql -SELECT toTypeName(from), hex(from) FROM hits LIMIT 1; -``` - -``` text -┌─toTypeName(from)─┬─hex(from)────────────────────────┐ -│ IPv6 │ 200144C8012926320033000002520002 │ -└──────────────────┴──────────────────────────────────┘ -``` - -Etki alanı değerleri örtülü olarak dışındaki türlere dönüştürülemez `FixedString(16)`. -Dönüştürmek istiyorsanız `IPv6` bir dizeye değer, bunu açıkça yapmak zorundasınız `IPv6NumToString()` işlev: - -``` sql -SELECT toTypeName(s), IPv6NumToString(from) as s FROM hits LIMIT 1; -``` - -``` text -┌─toTypeName(IPv6NumToString(from))─┬─s─────────────────────────────┐ -│ String │ 2001:44c8:129:2632:33:0:252:2 │ -└───────────────────────────────────┴───────────────────────────────┘ -``` - -Ya da bir döküm `FixedString(16)` değer: - -``` sql -SELECT toTypeName(i), CAST(from as FixedString(16)) as i FROM hits LIMIT 1; -``` - -``` text -┌─toTypeName(CAST(from, 'FixedString(16)'))─┬─i───────┐ -│ FixedString(16) │ ��� │ -└───────────────────────────────────────────┴─────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/domains/ipv6) diff --git a/docs/tr/sql-reference/data-types/enum.md b/docs/tr/sql-reference/data-types/enum.md deleted file mode 100644 index a2e9d8cca88..00000000000 --- a/docs/tr/sql-reference/data-types/enum.md +++ /dev/null @@ -1,132 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 50 -toc_title: Enum ---- - -# Enum {#enum} - -Adlandırılmış değerlerden oluşan numaralandırılmış tür. - -Adlandırılmış değerler olarak bildirilmelidir `'string' = integer` çiftliler. ClickHouse yalnızca sayıları saklar, ancak adları aracılığıyla değerlerle işlemleri destekler. - -ClickHouse destekler: - -- 8-bit `Enum`. En fazla 256 değerleri numaralandırılmış içerebilir `[-128, 127]` Aralık. -- 16-bit `Enum`. En fazla 65536 değerleri numaralandırılmış içerebilir `[-32768, 32767]` Aralık. - -ClickHouse otomatik olarak türünü seçer `Enum` veri eklendiğinde. Ayrıca kullanabilirsiniz `Enum8` veya `Enum16` türleri depolama boyutunda emin olmak için. - -## Kullanım Örnekleri {#usage-examples} - -Burada bir tablo oluşturuyoruz `Enum8('hello' = 1, 'world' = 2)` type Col columnum columnn: - -``` sql -CREATE TABLE t_enum -( - x Enum('hello' = 1, 'world' = 2) -) -ENGINE = TinyLog -``` - -Sütun `x` yalnızca tür tanımında listelenen değerleri depolayabilir: `'hello'` veya `'world'`. Başka bir değer kaydetmeye çalışırsanız, ClickHouse bir özel durum yükseltir. Bunun için 8-bit boyutu `Enum` otomatik olarak seçilir. - -``` sql -INSERT INTO t_enum VALUES ('hello'), ('world'), ('hello') -``` - -``` text -Ok. -``` - -``` sql -INSERT INTO t_enum values('a') -``` - -``` text -Exception on client: -Code: 49. DB::Exception: Unknown element 'a' for type Enum('hello' = 1, 'world' = 2) -``` - -Tablodan veri sorguladığınızda, ClickHouse dize değerleri `Enum`. - -``` sql -SELECT * FROM t_enum -``` - -``` text -┌─x─────┐ -│ hello │ -│ world │ -│ hello │ -└───────┘ -``` - -Satırların sayısal eşdeğerlerini görmeniz gerekiyorsa, `Enum` tamsayı türüne değer. - -``` sql -SELECT CAST(x, 'Int8') FROM t_enum -``` - -``` text -┌─CAST(x, 'Int8')─┐ -│ 1 │ -│ 2 │ -│ 1 │ -└─────────────────┘ -``` - -Bir sorguda bir Enum değeri oluşturmak için, ayrıca kullanmanız gerekir `CAST`. - -``` sql -SELECT toTypeName(CAST('a', 'Enum(\'a\' = 1, \'b\' = 2)')) -``` - -``` text -┌─toTypeName(CAST('a', 'Enum(\'a\' = 1, \'b\' = 2)'))─┐ -│ Enum8('a' = 1, 'b' = 2) │ -└─────────────────────────────────────────────────────┘ -``` - -## Genel Kurallar ve kullanım {#general-rules-and-usage} - -Değerlerin her birine aralıkta bir sayı atanır `-128 ... 127` için `Enum8` veya aralık inta `-32768 ... 32767` için `Enum16`. Tüm dizeler ve sayılar farklı olmalıdır. Boş bir dize izin verilir. Bu tür belirtilmişse (bir tablo tanımında), sayılar rasgele bir sırada olabilir. Ancak, sipariş önemli değil. - -Ne dize ne de sayısal değer bir `Enum` olabilir [NULL](../../sql-reference/syntax.md). - -Bir `Enum` içerdiği olabilir [Nullable](nullable.md) tür. Yani sorguyu kullanarak bir tablo oluşturursanız - -``` sql -CREATE TABLE t_enum_nullable -( - x Nullable( Enum8('hello' = 1, 'world' = 2) ) -) -ENGINE = TinyLog -``` - -bu mağaza değil sadece `'hello'` ve `'world'`, ama `NULL`, yanında. - -``` sql -INSERT INTO t_enum_nullable Values('hello'),('world'),(NULL) -``` - -RAM, bir `Enum` sütun aynı şekilde saklanır `Int8` veya `Int16` karşılık gelen sayısal değerlerin. - -Metin formunda okurken, ClickHouse değeri bir dize olarak ayrıştırır ve karşılık gelen dizeyi Enum değerleri kümesinden arar. Bulunmazsa, bir istisna atılır. Metin biçiminde okurken, dize okunur ve karşılık gelen sayısal değer aranır. Bulunmazsa bir istisna atılır. -Metin formunda yazarken, değeri karşılık gelen dize olarak yazar. Sütun verileri çöp içeriyorsa (geçerli kümeden olmayan sayılar), bir özel durum atılır. İkili formda okurken ve yazarken, Int8 ve Int16 veri türleri ile aynı şekilde çalışır. -Örtülü varsayılan değer, en düşük sayıya sahip değerdir. - -Sırasında `ORDER BY`, `GROUP BY`, `IN`, `DISTINCT` ve böylece, Enumlar karşılık gelen sayılarla aynı şekilde davranır. Örneğin, sipariş onları sayısal olarak sıralar. Eşitlik ve karşılaştırma işleçleri, alttaki sayısal değerler üzerinde yaptıkları gibi Enumlarda aynı şekilde çalışır. - -Enum değerleri sayılarla karşılaştırılamaz. Enums sabit bir dize ile karşılaştırılabilir. Karşılaştırılan dize Enum için geçerli bir değer değilse, bir özel durum atılır. IN operatörü, sol taraftaki Enum ve sağ taraftaki bir dizi dizeyle desteklenir. Dizeler, karşılık gelen Enumun değerleridir. - -Most numeric and string operations are not defined for Enum values, e.g. adding a number to an Enum or concatenating a string to an Enum. -Ancak, Enum doğal bir `toString` dize değerini döndüren işlev. - -Enum değerleri de kullanarak sayısal türlere dönüştürülebilir `toT` fonksiyon, burada t sayısal bir türdür. T enum'un temel sayısal türüne karşılık geldiğinde, bu dönüşüm sıfır maliyetlidir. -Enum türü, yalnızca değer kümesi değiştirilirse, alter kullanılarak maliyet olmadan değiştirilebilir. Her iki ekleme ve Alter kullanarak Enum üyeleri kaldırmak mümkündür (kaldırma yalnızca kaldırılan değer tabloda hiç kullanılmadıysa güvenlidir). Bir koruma olarak, önceden tanımlanmış bir Enum üyesinin sayısal değerini değiştirmek bir istisna atar. - -ALTER kullanarak, bir Enum8 için bir Enum16 veya tam tersi, Int8 için Int16 değiştirme gibi değiştirmek mümkündür. - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/enum/) diff --git a/docs/tr/sql-reference/data-types/fixedstring.md b/docs/tr/sql-reference/data-types/fixedstring.md deleted file mode 100644 index e3075ebbb40..00000000000 --- a/docs/tr/sql-reference/data-types/fixedstring.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 45 -toc_title: FixedString(N) ---- - -# Fixedstring {#fixedstring} - -Sabit uzunlukta bir dize `N` bayt (ne karakter ne de kod noktaları). - -Bir sütun bildirmek için `FixedString` yazın, aşağıdaki sözdizimini kullanın: - -``` sql - FixedString(N) -``` - -Nerede `N` doğal bir sayıdır. - -Bu `FixedString` veri tam olarak uzunluğa sahip olduğunda tür etkilidir `N` baytlar. Diğer tüm durumlarda, verimliliği düşürmesi muhtemeldir. - -Verimli bir şekilde depolan theabilen değerlere örnekler `FixedString`- yazılan sütunlar: - -- IP adreslerinin ikili gösterimi (`FixedString(16)` IPv6 için). -- Language codes (ru_RU, en_US … ). -- Currency codes (USD, RUB … ). -- Karma ikili gösterimi (`FixedString(16)` MD5 için, `FixedString(32)` SHA256 için). - -UUID değerlerini depolamak için [UUID](uuid.md) veri türü. - -Verileri eklerken, ClickHouse: - -- Dize daha az içeriyorsa, boş bayt ile bir dize tamamlar `N` baytlar. -- Atar `Too large value for FixedString(N)` dize birden fazla içeriyorsa, özel durum `N` baytlar. - -Verileri seçerken, ClickHouse dize sonunda boş bayt kaldırmaz. Eğer kullanıyorsanız `WHERE` yan tümcesi, null bayt el ile eşleştirmek için eklemelisiniz `FixedString` değer. Kullanımı için aşağıdaki örnek, nasıl gösterir `WHERE` fık withra ile `FixedString`. - -Aşağıdaki tabloyu tek ile düşünelim `FixedString(2)` sütun: - -``` text -┌─name──┐ -│ b │ -└───────┘ -``` - -Sorgu `SELECT * FROM FixedStringTable WHERE a = 'b'` sonuç olarak herhangi bir veri döndürmez. Filtre desenini boş baytlarla tamamlamalıyız. - -``` sql -SELECT * FROM FixedStringTable -WHERE a = 'b\0' -``` - -``` text -┌─a─┐ -│ b │ -└───┘ -``` - -Bu davranış için MySQL farklıdır `CHAR` tür (burada dizeler boşluklarla doldurulur ve boşluklar çıktı için kaldırılır). - -Not uzunluğu `FixedString(N)` değer sabittir. Bu [uzunluk](../../sql-reference/functions/array-functions.md#array_functions-length) fonksiyon döndürür `N` hatta eğer `FixedString(N)` değer yalnızca boş baytlarla doldurulur, ancak [boş](../../sql-reference/functions/string-functions.md#empty) fonksiyon döndürür `1` bu durumda. - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/fixedstring/) diff --git a/docs/tr/sql-reference/data-types/float.md b/docs/tr/sql-reference/data-types/float.md deleted file mode 100644 index 1bfbddf7a34..00000000000 --- a/docs/tr/sql-reference/data-types/float.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 41 -toc_title: Float32, Float64 ---- - -# Float32, Float64 {#float32-float64} - -[Kayan nokta numaraları](https://en.wikipedia.org/wiki/IEEE_754). - -Türleri C türlerine eşdeğerdir: - -- `Float32` - `float` -- `Float64` - `double` - -Verileri mümkün olduğunda tamsayı biçiminde saklamanızı öneririz. Örneğin, sabit hassas sayıları parasal tutarlar veya sayfa yükleme süreleri gibi milisaniye cinsinden tamsayı değerlerine dönüştürün. - -## Kayan noktalı sayıları kullanma {#using-floating-point-numbers} - -- Kayan noktalı sayılarla yapılan hesaplamalar yuvarlama hatası oluşturabilir. - - - -``` sql -SELECT 1 - 0.9 -``` - -``` text -┌───────minus(1, 0.9)─┐ -│ 0.09999999999999998 │ -└─────────────────────┘ -``` - -- Hesaplamanın sonucu hesaplama yöntemine (bilgisayar sisteminin işlemci tipi ve mimarisi) bağlıdır. -- Kayan nokta hesaplamaları, sonsuzluk gibi sayılarla sonuçlanabilir (`Inf`) ve “not-a-number” (`NaN`). Hesaplamaların sonuçlarını işlerken bu dikkate alınmalıdır. -- Kayan noktalı sayıları metinden ayrıştırırken, sonuç en yakın makine tarafından temsil edilebilir sayı olmayabilir. - -## N andan ve In andf {#data_type-float-nan-inf} - -Standart SQL aksine, ClickHouse kayan noktalı sayılar aşağıdaki kategorileri destekler: - -- `Inf` – Infinity. - - - -``` sql -SELECT 0.5 / 0 -``` - -``` text -┌─divide(0.5, 0)─┐ -│ inf │ -└────────────────┘ -``` - -- `-Inf` – Negative infinity. - - - -``` sql -SELECT -0.5 / 0 -``` - -``` text -┌─divide(-0.5, 0)─┐ -│ -inf │ -└─────────────────┘ -``` - -- `NaN` – Not a number. - - - -``` sql -SELECT 0 / 0 -``` - -``` text -┌─divide(0, 0)─┐ -│ nan │ -└──────────────┘ -``` - - See the rules for `NaN` sorting in the section [ORDER BY clause](../sql_reference/statements/select/order-by.md). - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/float/) diff --git a/docs/tr/sql-reference/data-types/index.md b/docs/tr/sql-reference/data-types/index.md deleted file mode 100644 index a7a5f685152..00000000000 --- a/docs/tr/sql-reference/data-types/index.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "Veri T\xFCrleri" -toc_priority: 37 -toc_title: "Giri\u015F" ---- - -# Veri Türleri {#data_types} - -ClickHouse tablo hücrelerinde veri çeşitli saklayabilirsiniz. - -Bu bölümde desteklenen veri türleri ve varsa bunları kullanmak ve/veya uygulamak için özel hususlar açıklanmaktadır. - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/) diff --git a/docs/tr/sql-reference/data-types/int-uint.md b/docs/tr/sql-reference/data-types/int-uint.md deleted file mode 100644 index 5ec9dcd0785..00000000000 --- a/docs/tr/sql-reference/data-types/int-uint.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 40 -toc_title: "U\u0130nt8, U\u0130nt16, U\u0130nt32, Uint64, Int8, Int16, Int32, Int64" ---- - -# Uİnt8, Uİnt16, Uİnt32, Uint64, Int8, Int16, Int32, Int64 {#uint8-uint16-uint32-uint64-int8-int16-int32-int64} - -Sabit uzunlukta tamsayılar, veya bir işareti olmadan. - -## İnt Aralıkları {#int-ranges} - -- Int8- \[-128: 127\] -- Int16 - \[-32768: 32767\] -- Int32 - \[-2147483648: 2147483647\] -- Int64 - \[-9223372036854775808: 9223372036854775807\] - -## Uint Aralıkları {#uint-ranges} - -- Uİnt8- \[0: 255\] -- Uİnt16- \[0: 65535\] -- Uİnt32- \[0: 4294967295\] -- Uİnt64 - \[0: 18446744073709551615\] - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/int_uint/) diff --git a/docs/tr/sql-reference/data-types/lowcardinality.md b/docs/tr/sql-reference/data-types/lowcardinality.md deleted file mode 100644 index dd3a9aa1c0d..00000000000 --- a/docs/tr/sql-reference/data-types/lowcardinality.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/data-types/lowcardinality.md \ No newline at end of file diff --git a/docs/tr/sql-reference/data-types/nested-data-structures/index.md b/docs/tr/sql-reference/data-types/nested-data-structures/index.md deleted file mode 100644 index 75c2b85c558..00000000000 --- a/docs/tr/sql-reference/data-types/nested-data-structures/index.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0130\xE7 \u0130\xE7e Veri Yap\u0131lar\u0131" -toc_hidden: true -toc_priority: 54 -toc_title: "gizlenmi\u015F" ---- - -# İç İçe Veri Yapıları {#nested-data-structures} - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/nested_data_structures/) diff --git a/docs/tr/sql-reference/data-types/nested-data-structures/nested.md b/docs/tr/sql-reference/data-types/nested-data-structures/nested.md deleted file mode 100644 index c0881025e8c..00000000000 --- a/docs/tr/sql-reference/data-types/nested-data-structures/nested.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 57 -toc_title: "\u0130\xE7 \u0130\xE7e (Name1 Type1, Name2 Type2,...)" ---- - -# Nested(name1 Type1, Name2 Type2, …) {#nestedname1-type1-name2-type2} - -A nested data structure is like a table inside a cell. The parameters of a nested data structure – the column names and types – are specified the same way as in a [CREATE TABLE](../../../sql-reference/statements/create.md) sorgu. Her tablo satırı, iç içe geçmiş veri yapısındaki herhangi bir sayıda satıra karşılık gelebilir. - -Örnek: - -``` sql -CREATE TABLE test.visits -( - CounterID UInt32, - StartDate Date, - Sign Int8, - IsNew UInt8, - VisitID UInt64, - UserID UInt64, - ... - Goals Nested - ( - ID UInt32, - Serial UInt32, - EventTime DateTime, - Price Int64, - OrderID String, - CurrencyID UInt32 - ), - ... -) ENGINE = CollapsingMergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192, Sign) -``` - -Bu örnek bildirir `Goals` dönüşümlerle ilgili verileri içeren iç içe veri yapısı (ulaşılan hedefler). Her satır içinde ‘visits’ tablo sıfır veya dönüşüm herhangi bir sayıda karşılık gelebilir. - -Sadece tek bir yuvalama seviyesi desteklenir. Diziler içeren iç içe geçmiş yapıların sütunları çok boyutlu dizilere eşdeğerdir, bu nedenle sınırlı desteğe sahiptirler (bu sütunları MergeTree altyapısı ile tablolarda depolamak için destek yoktur). - -Çoğu durumda, iç içe geçmiş bir veri yapısıyla çalışırken, sütunları bir nokta ile ayrılmış sütun adlarıyla belirtilir. Bu sütunlar eşleşen türleri bir dizi oluşturur. Tek bir iç içe geçmiş veri yapısının tüm sütun dizileri aynı uzunluğa sahiptir. - -Örnek: - -``` sql -SELECT - Goals.ID, - Goals.EventTime -FROM test.visits -WHERE CounterID = 101500 AND length(Goals.ID) < 5 -LIMIT 10 -``` - -``` text -┌─Goals.ID───────────────────────┬─Goals.EventTime───────────────────────────────────────────────────────────────────────────┐ -│ [1073752,591325,591325] │ ['2014-03-17 16:38:10','2014-03-17 16:38:48','2014-03-17 16:42:27'] │ -│ [1073752] │ ['2014-03-17 00:28:25'] │ -│ [1073752] │ ['2014-03-17 10:46:20'] │ -│ [1073752,591325,591325,591325] │ ['2014-03-17 13:59:20','2014-03-17 22:17:55','2014-03-17 22:18:07','2014-03-17 22:18:51'] │ -│ [] │ [] │ -│ [1073752,591325,591325] │ ['2014-03-17 11:37:06','2014-03-17 14:07:47','2014-03-17 14:36:21'] │ -│ [] │ [] │ -│ [] │ [] │ -│ [591325,1073752] │ ['2014-03-17 00:46:05','2014-03-17 00:46:05'] │ -│ [1073752,591325,591325,591325] │ ['2014-03-17 13:28:33','2014-03-17 13:30:26','2014-03-17 18:51:21','2014-03-17 18:51:45'] │ -└────────────────────────────────┴───────────────────────────────────────────────────────────────────────────────────────────┘ -``` - -İç içe geçmiş bir veri yapısını aynı uzunlukta birden çok sütun dizisi kümesi olarak düşünmek en kolay yoldur. - -Bir SELECT sorgusunun tek tek sütunlar yerine tüm iç içe geçmiş veri yapısının adını belirtebileceği tek yer array JOIN yan tümcesi. Daha fazla bilgi için, bkz. “ARRAY JOIN clause”. Örnek: - -``` sql -SELECT - Goal.ID, - Goal.EventTime -FROM test.visits -ARRAY JOIN Goals AS Goal -WHERE CounterID = 101500 AND length(Goals.ID) < 5 -LIMIT 10 -``` - -``` text -┌─Goal.ID─┬──────Goal.EventTime─┐ -│ 1073752 │ 2014-03-17 16:38:10 │ -│ 591325 │ 2014-03-17 16:38:48 │ -│ 591325 │ 2014-03-17 16:42:27 │ -│ 1073752 │ 2014-03-17 00:28:25 │ -│ 1073752 │ 2014-03-17 10:46:20 │ -│ 1073752 │ 2014-03-17 13:59:20 │ -│ 591325 │ 2014-03-17 22:17:55 │ -│ 591325 │ 2014-03-17 22:18:07 │ -│ 591325 │ 2014-03-17 22:18:51 │ -│ 1073752 │ 2014-03-17 11:37:06 │ -└─────────┴─────────────────────┘ -``` - -İç içe geçmiş veri yapısının tamamı için SELECT gerçekleştiremezsiniz. Yalnızca bir parçası olan tek tek sütunları açıkça listeleyebilirsiniz. - -Bir INSERT sorgusu için, iç içe geçmiş bir veri yapısının tüm bileşen sütun dizilerini ayrı ayrı (tek tek sütun dizileri gibi) iletmelisiniz. Ekleme sırasında, sistem aynı uzunluğa sahip olduklarını kontrol eder. - -Bir tanımlama sorgusu için, iç içe geçmiş bir veri yapısındaki sütunlar aynı şekilde ayrı olarak listelenir. - -İç içe geçmiş bir veri yapısındaki öğeler için ALTER sorgusu sınırlamaları vardır. - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/nested_data_structures/nested/) diff --git a/docs/tr/sql-reference/data-types/nullable.md b/docs/tr/sql-reference/data-types/nullable.md deleted file mode 100644 index fe61ce96a2d..00000000000 --- a/docs/tr/sql-reference/data-types/nullable.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 54 -toc_title: Nullable ---- - -# Nullable (typename) {#data_type-nullable} - -Özel işaretleyici saklamak için izin verir ([NULL](../../sql-reference/syntax.md)) bu ifade eder “missing value” tarafından izin verilen normal değerlerin yanında `TypeName`. Örneğin, bir `Nullable(Int8)` tipi sütun saklayabilirsiniz `Int8` değerleri yazın ve değeri olmayan satırlar depolayacaktır `NULL`. - -İçin... `TypeName`, bileşik veri türlerini kullanamazsınız [Dizi](array.md) ve [Demet](tuple.md). Bileşik veri türleri şunları içerebilir `Nullable` gibi tür değerleri `Array(Nullable(Int8))`. - -A `Nullable` tür alanı tablo dizinlerine dahil edilemez. - -`NULL` herhangi biri için varsayılan değer mi `Nullable` ClickHouse sunucu yapılandırmasında aksi belirtilmediği sürece yazın. - -## Depolama Özellikleri {#storage-features} - -İçermek `Nullable` bir tablo sütunundaki değerleri yazın, ClickHouse ile ayrı bir dosya kullanır `NULL` değerleri ile normal dosyaya ek olarak Maskeler. Maskeli girişleri ClickHouse ayırt izin dosyası `NULL` ve her tablo satırı için karşılık gelen veri türünün varsayılan değeri. Ek bir dosya nedeniyle, `Nullable` sütun, benzer bir normal olana kıyasla ek depolama alanı tüketir. - -!!! info "Not" - Kullanım `Nullable` neredeyse her zaman performansı olumsuz etkiler, veritabanlarınızı tasarlarken bunu aklınızda bulundurun. - -## Kullanım Örneği {#usage-example} - -``` sql -CREATE TABLE t_null(x Int8, y Nullable(Int8)) ENGINE TinyLog -``` - -``` sql -INSERT INTO t_null VALUES (1, NULL), (2, 3) -``` - -``` sql -SELECT x + y FROM t_null -``` - -``` text -┌─plus(x, y)─┐ -│ ᴺᵁᴸᴸ │ -│ 5 │ -└────────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/nullable/) diff --git a/docs/tr/sql-reference/data-types/simpleaggregatefunction.md b/docs/tr/sql-reference/data-types/simpleaggregatefunction.md deleted file mode 100644 index 8ecaf174688..00000000000 --- a/docs/tr/sql-reference/data-types/simpleaggregatefunction.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 71d72c1f237f4a553fe91ba6c6c633e81a49e35b ---- - -# SimpleAggregateFunction {#data-type-simpleaggregatefunction} - -`SimpleAggregateFunction(name, types_of_arguments…)` veri türü, toplama işlevinin geçerli değerini depolar ve tam durumunu şu şekilde depolamaz [`AggregateFunction`](../../sql-reference/data-types/aggregatefunction.md) yapar. Bu optimizasyon, aşağıdaki özelliğin bulunduğu işlevlere uygulanabilir: bir işlev uygulama sonucu `f` bir satır kümesi için `S1 UNION ALL S2` uygulayarak elde edilebilir `f` satır parçalarına ayrı ayrı ayarlayın ve sonra tekrar uygulayın `f` sonuçlara: `f(S1 UNION ALL S2) = f(f(S1) UNION ALL f(S2))`. Bu özellik, kısmi toplama sonuçlarının Birleşik olanı hesaplamak için yeterli olduğunu garanti eder, bu nedenle herhangi bir ek veri depolamak ve işlemek zorunda kalmayız. - -Aşağıdaki toplama işlevleri desteklenir: - -- [`any`](../../sql-reference/aggregate-functions/reference.md#agg_function-any) -- [`anyLast`](../../sql-reference/aggregate-functions/reference.md#anylastx) -- [`min`](../../sql-reference/aggregate-functions/reference.md#agg_function-min) -- [`max`](../../sql-reference/aggregate-functions/reference.md#agg_function-max) -- [`sum`](../../sql-reference/aggregate-functions/reference.md#agg_function-sum) -- [`groupBitAnd`](../../sql-reference/aggregate-functions/reference.md#groupbitand) -- [`groupBitOr`](../../sql-reference/aggregate-functions/reference.md#groupbitor) -- [`groupBitXor`](../../sql-reference/aggregate-functions/reference.md#groupbitxor) -- [`groupArrayArray`](../../sql-reference/aggregate-functions/reference.md#agg_function-grouparray) -- [`groupUniqArrayArray`](../../sql-reference/aggregate-functions/reference.md#groupuniqarrayx-groupuniqarraymax-sizex) - -Değerleri `SimpleAggregateFunction(func, Type)` bak ve aynı şekilde saklanır `Type`, bu yüzden fonksiyonları ile uygulamak gerekmez `-Merge`/`-State` sonekler. `SimpleAggregateFunction` daha iyi performans vardır `AggregateFunction` aynı toplama fonksiyonu ile. - -**Parametre** - -- Toplama işlevinin adı. -- Toplama işlevi bağımsız değişkenleri türleri. - -**Örnek** - -``` sql -CREATE TABLE t -( - column1 SimpleAggregateFunction(sum, UInt64), - column2 SimpleAggregateFunction(any, String) -) ENGINE = ... -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/simpleaggregatefunction/) diff --git a/docs/tr/sql-reference/data-types/special-data-types/expression.md b/docs/tr/sql-reference/data-types/special-data-types/expression.md deleted file mode 100644 index 4cd9a2f8211..00000000000 --- a/docs/tr/sql-reference/data-types/special-data-types/expression.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 58 -toc_title: "\u0130fade" ---- - -# İfade {#expression} - -İfadeler, lambda'ları yüksek mertebeden işlevlerde temsil etmek için kullanılır. - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/special_data_types/expression/) diff --git a/docs/tr/sql-reference/data-types/special-data-types/index.md b/docs/tr/sql-reference/data-types/special-data-types/index.md deleted file mode 100644 index ee0b9ac0630..00000000000 --- a/docs/tr/sql-reference/data-types/special-data-types/index.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\xD6zel Veri T\xFCrleri" -toc_hidden: true -toc_priority: 55 -toc_title: "gizlenmi\u015F" ---- - -# Özel Veri Türleri {#special-data-types} - -Özel veri türü değerleri Tablo veya çıktı sorgu sonuçlarında kaydetmek için seri hale getirilemez, ancak sorgu yürütme sırasında bir ara sonuç olarak kullanılabilir. - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/special_data_types/) diff --git a/docs/tr/sql-reference/data-types/special-data-types/interval.md b/docs/tr/sql-reference/data-types/special-data-types/interval.md deleted file mode 100644 index d817f2c5214..00000000000 --- a/docs/tr/sql-reference/data-types/special-data-types/interval.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 61 -toc_title: "Aral\u0131kl\u0131" ---- - -# Aralıklı {#data-type-interval} - -Zaman ve Tarih aralıklarını temsil eden veri türleri ailesi. Ortaya çıkan türleri [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) operatör. - -!!! warning "Uyarıcı" - `Interval` veri türü değerleri tablolarda saklanamaz. - -Yapılı: - -- İmzasız bir tamsayı değeri olarak zaman aralığı. -- Bir aralık türü. - -Desteklenen Aralık türleri: - -- `SECOND` -- `MINUTE` -- `HOUR` -- `DAY` -- `WEEK` -- `MONTH` -- `QUARTER` -- `YEAR` - -Her Aralık türü için ayrı bir veri türü vardır. Örneğin, `DAY` Aralık karşılık gelir `IntervalDay` veri türü: - -``` sql -SELECT toTypeName(INTERVAL 4 DAY) -``` - -``` text -┌─toTypeName(toIntervalDay(4))─┐ -│ IntervalDay │ -└──────────────────────────────┘ -``` - -## Kullanım Açıklamaları {#data-type-interval-usage-remarks} - -Kullanabilirsiniz `Interval`- aritmetik işlemlerde değerler yazın [Tarihli](../../../sql-reference/data-types/date.md) ve [DateTime](../../../sql-reference/data-types/datetime.md)- tip değerleri. Örneğin, geçerli saate 4 gün ekleyebilirsiniz: - -``` sql -SELECT now() as current_date_time, current_date_time + INTERVAL 4 DAY -``` - -``` text -┌───current_date_time─┬─plus(now(), toIntervalDay(4))─┐ -│ 2019-10-23 10:58:45 │ 2019-10-27 10:58:45 │ -└─────────────────────┴───────────────────────────────┘ -``` - -Farklı tiplere sahip aralıklar birleştirilemez. Gibi aralıklarla kullanamazsınız `4 DAY 1 HOUR`. Aralıkların, örneğin aralığın en küçük birimine eşit veya daha küçük olan birimlerdeki aralıkları belirtin `1 day and an hour` aralık olarak ifade edilebilir `25 HOUR` veya `90000 SECOND`. - -İle aritmetik işlemler yapamazsınız `Interval`- değerleri yazın, ancak farklı türde aralıklar ekleyebilirsiniz. `Date` veya `DateTime` veri türleri. Mesela: - -``` sql -SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL 3 HOUR -``` - -``` text -┌───current_date_time─┬─plus(plus(now(), toIntervalDay(4)), toIntervalHour(3))─┐ -│ 2019-10-23 11:16:28 │ 2019-10-27 14:16:28 │ -└─────────────────────┴────────────────────────────────────────────────────────┘ -``` - -Aşağıdaki sorgu bir özel duruma neden olur: - -``` sql -select now() AS current_date_time, current_date_time + (INTERVAL 4 DAY + INTERVAL 3 HOUR) -``` - -``` text -Received exception from server (version 19.14.1): -Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Wrong argument types for function plus: if one argument is Interval, then another must be Date or DateTime.. -``` - -## Ayrıca Bakınız {#see-also} - -- [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) operatör -- [toİnterval](../../../sql-reference/functions/type-conversion-functions.md#function-tointerval) tip dönüştürme işlevleri diff --git a/docs/tr/sql-reference/data-types/special-data-types/nothing.md b/docs/tr/sql-reference/data-types/special-data-types/nothing.md deleted file mode 100644 index 32017b962dc..00000000000 --- a/docs/tr/sql-reference/data-types/special-data-types/nothing.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 60 -toc_title: "Hi\xE7bir \u015Fey" ---- - -# Hiçbir şey {#nothing} - -Bu veri türünün tek amacı, bir değerin beklenmediği durumları temsil etmektir. Yani bir oluşturamazsınız `Nothing` type value. - -Örneğin, literal [NULL](../../../sql-reference/syntax.md#null-literal) türü vardır `Nullable(Nothing)`. Daha fazla görmek [Nullable](../../../sql-reference/data-types/nullable.md). - -Bu `Nothing` tür boş dizileri belirtmek için de kullanılabilir: - -``` sql -SELECT toTypeName(array()) -``` - -``` text -┌─toTypeName(array())─┐ -│ Array(Nothing) │ -└─────────────────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/special_data_types/nothing/) diff --git a/docs/tr/sql-reference/data-types/special-data-types/set.md b/docs/tr/sql-reference/data-types/special-data-types/set.md deleted file mode 100644 index 10b4219f733..00000000000 --- a/docs/tr/sql-reference/data-types/special-data-types/set.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 59 -toc_title: Koymak ---- - -# Koymak {#set} - -Sağ yarısı için kullanılan bir [IN](../../operators/in.md#select-in-operators) İfade. - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/special_data_types/set/) diff --git a/docs/tr/sql-reference/data-types/string.md b/docs/tr/sql-reference/data-types/string.md deleted file mode 100644 index 52d61f1ff9b..00000000000 --- a/docs/tr/sql-reference/data-types/string.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 44 -toc_title: Dize ---- - -# Dize {#string} - -Keyfi uzunlukta dizeler. Uzunluk sınırlı değildir. Değer, boş bayt da dahil olmak üzere rasgele bir bayt kümesi içerebilir. -Dize türü türleri değiştirir VARCHAR, BLOB, CLOB, ve diğerleri diğer DBMSs. - -## Kodlamalar {#encodings} - -ClickHouse kodlama kavramına sahip değildir. Dizeler, depolanan ve olduğu gibi çıkan rasgele bir bayt kümesi içerebilir. -Metinleri saklamanız gerekiyorsa, UTF-8 kodlamasını kullanmanızı öneririz. En azından, terminaliniz UTF-8 kullanıyorsa (önerildiği gibi), dönüşüm yapmadan değerlerinizi okuyabilir ve yazabilirsiniz. -Benzer şekilde, dizelerle çalışmak için belirli işlevler, dizenin UTF-8 kodlu bir metni temsil eden bir bayt kümesi içerdiği varsayımı altında çalışan ayrı varyasyonlara sahiptir. -Örneğin, ‘length’ işlev, bayt cinsinden dize uzunluğunu hesaplar; ‘lengthUTF8’ işlev, değerin UTF-8 kodlanmış olduğunu varsayarak Unicode kod noktalarındaki dize uzunluğunu hesaplar. - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/string/) diff --git a/docs/tr/sql-reference/data-types/tuple.md b/docs/tr/sql-reference/data-types/tuple.md deleted file mode 100644 index 5eb0435d3cd..00000000000 --- a/docs/tr/sql-reference/data-types/tuple.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 53 -toc_title: Tuple (T1, T2, ...) ---- - -# Tuple(t1, T2, …) {#tuplet1-t2} - -Elemanlarının bir demet, her bir birey olması [tür](index.md#data_types). - -Tuples geçici sütun gruplama için kullanılır. Sütunlar, bir sorguda bir In ifadesi kullanıldığında ve lambda işlevlerinin belirli resmi parametrelerini belirtmek için gruplandırılabilir. Daha fazla bilgi için bölümlere bakın [Operatör İNLERDE](../../sql-reference/operators/in.md) ve [Yüksek mertebeden fonksiyonlar](../../sql-reference/functions/higher-order-functions.md). - -Tuples bir sorgunun sonucu olabilir. Bu durumda, json dışındaki metin formatları için değerler köşeli parantez içinde virgülle ayrılır. JSON formatlarında, tuples diziler olarak çıktılanır (köşeli parantez içinde). - -## Bir Tuple oluşturma {#creating-a-tuple} - -Bir tuple oluşturmak için bir işlev kullanabilirsiniz: - -``` sql -tuple(T1, T2, ...) -``` - -Bir tuple oluşturma örneği: - -``` sql -SELECT tuple(1,'a') AS x, toTypeName(x) -``` - -``` text -┌─x───────┬─toTypeName(tuple(1, 'a'))─┐ -│ (1,'a') │ Tuple(UInt8, String) │ -└─────────┴───────────────────────────┘ -``` - -## Veri türleri ile çalışma {#working-with-data-types} - -Anında bir tuple oluştururken, ClickHouse her bağımsız değişkenin türünü bağımsız değişken değerini depolayabilen türlerin en azı olarak otomatik olarak algılar. Argüman ise [NULL](../../sql-reference/syntax.md#null-literal), tuple elemanının türü [Nullable](nullable.md). - -Otomatik veri türü algılama örneği: - -``` sql -SELECT tuple(1, NULL) AS x, toTypeName(x) -``` - -``` text -┌─x────────┬─toTypeName(tuple(1, NULL))──────┐ -│ (1,NULL) │ Tuple(UInt8, Nullable(Nothing)) │ -└──────────┴─────────────────────────────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/tuple/) diff --git a/docs/tr/sql-reference/data-types/uuid.md b/docs/tr/sql-reference/data-types/uuid.md deleted file mode 100644 index c98ef123f3d..00000000000 --- a/docs/tr/sql-reference/data-types/uuid.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 46 -toc_title: UUID ---- - -# UUID {#uuid-data-type} - -Evrensel olarak benzersiz bir tanımlayıcı (UUID), kayıtları tanımlamak için kullanılan 16 baytlık bir sayıdır. UUID hakkında ayrıntılı bilgi için bkz [Vikipedi](https://en.wikipedia.org/wiki/Universally_unique_identifier). - -UUID türü değeri örneği aşağıda temsil edilmektedir: - -``` text -61f0c404-5cb3-11e7-907b-a6006ad3dba0 -``` - -Yeni bir kayıt eklerken UUID sütun değerini belirtmezseniz, UUID değeri sıfır ile doldurulur: - -``` text -00000000-0000-0000-0000-000000000000 -``` - -## Nasıl oluşturulur {#how-to-generate} - -UUID değerini oluşturmak için ClickHouse, [generateuuıdv4](../../sql-reference/functions/uuid-functions.md) İşlev. - -## Kullanım Örneği {#usage-example} - -**Örnek 1** - -Bu örnek, UUID türü sütunuyla bir tablo oluşturma ve tabloya bir değer ekleme gösterir. - -``` sql -CREATE TABLE t_uuid (x UUID, y String) ENGINE=TinyLog -``` - -``` sql -INSERT INTO t_uuid SELECT generateUUIDv4(), 'Example 1' -``` - -``` sql -SELECT * FROM t_uuid -``` - -``` text -┌────────────────────────────────────x─┬─y─────────┐ -│ 417ddc5d-e556-4d27-95dd-a34d84e46a50 │ Example 1 │ -└──────────────────────────────────────┴───────────┘ -``` - -**Örnek 2** - -Bu örnekte, yeni bir kayıt eklerken UUID sütun değeri belirtilmedi. - -``` sql -INSERT INTO t_uuid (y) VALUES ('Example 2') -``` - -``` sql -SELECT * FROM t_uuid -``` - -``` text -┌────────────────────────────────────x─┬─y─────────┐ -│ 417ddc5d-e556-4d27-95dd-a34d84e46a50 │ Example 1 │ -│ 00000000-0000-0000-0000-000000000000 │ Example 2 │ -└──────────────────────────────────────┴───────────┘ -``` - -## Kısıtlama {#restrictions} - -UUID veri türü sadece hangi fonksiyonları destekler [Dize](string.md) veri türü de destekler (örneğin, [dakika](../../sql-reference/aggregate-functions/reference.md#agg_function-min), [maksimum](../../sql-reference/aggregate-functions/reference.md#agg_function-max), ve [sayma](../../sql-reference/aggregate-functions/reference.md#agg_function-count)). - -UUID veri türü aritmetik işlemler tarafından desteklenmez (örneğin, [abs](../../sql-reference/functions/arithmetic-functions.md#arithm_func-abs)) veya toplama fonksiyonları gibi [toplam](../../sql-reference/aggregate-functions/reference.md#agg_function-sum) ve [avg](../../sql-reference/aggregate-functions/reference.md#agg_function-avg). - -[Orijinal makale](https://clickhouse.tech/docs/en/data_types/uuid/) diff --git a/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md b/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md deleted file mode 100644 index f569f844ace..00000000000 --- a/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 45 -toc_title: "Hiyerar\u015Fik s\xF6zl\xFCkler" ---- - -# Hiyerarşik Sözlükler {#hierarchical-dictionaries} - -ClickHouse bir hiyerarşik sözlükler destekler [sayısal tuş](external-dicts-dict-structure.md#ext_dict-numeric-key). - -Aşağıdaki hiyerarşik yapıya bakın: - -``` text -0 (Common parent) -│ -├── 1 (Russia) -│ │ -│ └── 2 (Moscow) -│ │ -│ └── 3 (Center) -│ -└── 4 (Great Britain) - │ - └── 5 (London) -``` - -Bu hiyerarşi aşağıdaki sözlük tablosu olarak ifade edilebilir. - -| region_id | parent_region | region_name | -|------------|----------------|--------------| -| 1 | 0 | Rusya | -| 2 | 1 | Moskova | -| 3 | 2 | Merkezli | -| 4 | 0 | İngiltere | -| 5 | 4 | Londra | - -Bu tablo bir sütun içerir `parent_region` bu öğe için en yakın ebeveynin anahtarını içerir. - -ClickHouse destekler [hiyerarşik](external-dicts-dict-structure.md#hierarchical-dict-attr) için mülkiyet [dış sözlük](index.md) öznitelik. Bu özellik, yukarıda açıklanana benzer hiyerarşik sözlüğü yapılandırmanıza izin verir. - -Bu [dictGetHierarchy](../../../sql-reference/functions/ext-dict-functions.md#dictgethierarchy) fonksiyonu bir elemanın üst zincir almak için izin verir. - -Örneğimiz için, sözlüğün yapısı aşağıdaki gibi olabilir: - -``` xml - - - - region_id - - - - parent_region - UInt64 - 0 - true - - - - region_name - String - - - - - -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_hierarchical/) diff --git a/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md deleted file mode 100644 index cc45231fe72..00000000000 --- a/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ /dev/null @@ -1,396 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 41 -toc_title: "S\xF6zl\xFCkleri bellekte saklama" ---- - -# Sözlükleri bellekte saklama {#dicts-external-dicts-dict-layout} - -Sözlükleri bellekte saklamanın çeşitli yolları vardır. - -Biz tavsiye [düzlük](#flat), [karıştırıyordu](#dicts-external_dicts_dict_layout-hashed) ve [complex_key_hashed](#complex-key-hashed). hangi optimum işleme hızı sağlamak. - -Önbelleğe alma, potansiyel olarak düşük performans ve en uygun parametreleri seçmede zorluklar nedeniyle önerilmez. Bölümünde devamını oku “[önbellek](#cache)”. - -Sözlük performansını artırmanın birkaç yolu vardır: - -- Sonra sözlük ile çalışmak için işlevi çağırın `GROUP BY`. -- Mark enjektif olarak ayıklamak için nitelikler. Farklı öznitelik değerleri farklı anahtarlara karşılık geliyorsa, bir öznitelik ınjective olarak adlandırılır. Yani ne zaman `GROUP BY` anahtar tarafından bir öznitelik değeri getiren bir işlev kullanır, bu işlev otomatik olarak dışarı alınır `GROUP BY`. - -ClickHouse, sözlüklerdeki hatalar için bir istisna oluşturur. Hata örnekleri: - -- Erişilen sözlük yüklenemedi. -- Bir sorgulama hatası `cached` sözlük. - -Sen dış sözlükler ve durumları listesini görüntüleyebilirsiniz `system.dictionaries` Tablo. - -Yapılandırma şöyle görünüyor: - -``` xml - - - ... - - - - - - ... - - -``` - -İlgili [DDL-sorgu](../../statements/create.md#create-dictionary-query): - -``` sql -CREATE DICTIONARY (...) -... -LAYOUT(LAYOUT_TYPE(param value)) -- layout settings -... -``` - -## Sözlükleri bellekte saklamanın yolları {#ways-to-store-dictionaries-in-memory} - -- [düzlük](#flat) -- [karıştırıyordu](#dicts-external_dicts_dict_layout-hashed) -- [sparse_hashed](#dicts-external_dicts_dict_layout-sparse_hashed) -- [önbellek](#cache) -- [direkt](#direct) -- [range_hashed](#range-hashed) -- [complex_key_hashed](#complex-key-hashed) -- [complex_key_cache](#complex-key-cache) -- [ıp_trie](#ip-trie) - -### düzlük {#flat} - -Sözlük tamamen düz diziler şeklinde bellekte saklanır. Sözlük ne kadar bellek kullanıyor? Miktar, en büyük anahtarın boyutuyla orantılıdır (kullanılan alanda). - -Sözlük anahtarı vardır `UInt64` yazın ve değeri 500.000 ile sınırlıdır. Sözlük oluştururken daha büyük bir anahtar bulunursa, ClickHouse bir özel durum atar ve sözlüğü oluşturmaz. - -Her türlü kaynak desteklenmektedir. Güncellerken, veriler (bir dosyadan veya bir tablodan) bütünüyle okunur. - -Bu yöntem, sözlüğü saklamak için mevcut tüm yöntemler arasında en iyi performansı sağlar. - -Yapılandırma örneği: - -``` xml - - - -``` - -veya - -``` sql -LAYOUT(FLAT()) -``` - -### karıştırıyordu {#dicts-external_dicts_dict_layout-hashed} - -Sözlük tamamen bir karma tablo şeklinde bellekte saklanır. Sözlük, uygulamada herhangi bir tanımlayıcıya sahip herhangi bir sayıda öğe içerebilir, anahtar sayısı on milyonlarca öğeye ulaşabilir. - -Her türlü kaynak desteklenmektedir. Güncellerken, veriler (bir dosyadan veya bir tablodan) bütünüyle okunur. - -Yapılandırma örneği: - -``` xml - - - -``` - -veya - -``` sql -LAYOUT(HASHED()) -``` - -### sparse_hashed {#dicts-external_dicts_dict_layout-sparse_hashed} - -Benzer `hashed`, ancak daha fazla CPU kullanımı lehine daha az bellek kullanır. - -Yapılandırma örneği: - -``` xml - - - -``` - -``` sql -LAYOUT(SPARSE_HASHED()) -``` - -### complex_key_hashed {#complex-key-hashed} - -Bu tür depolama kompozit ile kullanım içindir [anahtarlar](external-dicts-dict-structure.md). Benzer `hashed`. - -Yapılandırma örneği: - -``` xml - - - -``` - -``` sql -LAYOUT(COMPLEX_KEY_HASHED()) -``` - -### range_hashed {#range-hashed} - -Sözlük, sıralı bir aralık dizisi ve bunlara karşılık gelen değerleri olan bir karma tablo şeklinde bellekte saklanır. - -Bu depolama yöntemi, hashed ile aynı şekilde çalışır ve anahtara ek olarak tarih/saat (rasgele sayısal tür) aralıklarının kullanılmasına izin verir. - -Örnek: tablo, her reklamveren için biçimdeki indirimleri içerir: - -``` text -+---------|-------------|-------------|------+ -| advertiser id | discount start date | discount end date | amount | -+===============+=====================+===================+========+ -| 123 | 2015-01-01 | 2015-01-15 | 0.15 | -+---------|-------------|-------------|------+ -| 123 | 2015-01-16 | 2015-01-31 | 0.25 | -+---------|-------------|-------------|------+ -| 456 | 2015-01-01 | 2015-01-15 | 0.05 | -+---------|-------------|-------------|------+ -``` - -Tarih aralıkları için bir örnek kullanmak için, `range_min` ve `range_max` element inler [yapılı](external-dicts-dict-structure.md). Bu elemanlar elemanları içermelidir `name` ve`type` (eğer `type` belirtilmemişse, varsayılan tür kullanılır-Tarih). `type` herhangi bir sayısal tür olabilir (Date / DateTime / Uint64 / Int32 / others). - -Örnek: - -``` xml - - - Id - - - first - Date - - - last - Date - - ... -``` - -veya - -``` sql -CREATE DICTIONARY somedict ( - id UInt64, - first Date, - last Date -) -PRIMARY KEY id -LAYOUT(RANGE_HASHED()) -RANGE(MIN first MAX last) -``` - -Bu sözlüklerle çalışmak için, `dictGetT` bir aralığın seçildiği işlev: - -``` sql -dictGetT('dict_name', 'attr_name', id, date) -``` - -Bu işlev belirtilen değerin değerini döndürür `id`s ve geçirilen tarihi içeren tarih aralığı. - -Algoritmanın detayları: - -- Eğer... `id` not fo orund veya a range is not fo aund for the `id`, sözlük için varsayılan değeri döndürür. -- Çakışan aralıklar varsa, herhangi birini kullanabilirsiniz. -- Aralık sınırlayıcı ise `NULL` veya geçersiz bir tarih (örneğin 1900-01-01 veya 2039-01-01), Aralık açık bırakılır. Aralık her iki tarafta da açık olabilir. - -Yapılandırma örneği: - -``` xml - - - - ... - - - - - - - - Abcdef - - - StartTimeStamp - UInt64 - - - EndTimeStamp - UInt64 - - - XXXType - String - - - - - - -``` - -veya - -``` sql -CREATE DICTIONARY somedict( - Abcdef UInt64, - StartTimeStamp UInt64, - EndTimeStamp UInt64, - XXXType String DEFAULT '' -) -PRIMARY KEY Abcdef -RANGE(MIN StartTimeStamp MAX EndTimeStamp) -``` - -### önbellek {#cache} - -Sözlük, sabit sayıda hücre içeren bir önbellekte saklanır. Bu hücreler sık kullanılan elementleri içerir. - -Bir sözlük ararken, önce önbellek aranır. Her veri bloğu için, önbellekte bulunmayan veya güncel olmayan tüm anahtarlar, kaynak kullanılarak istenir `SELECT attrs... FROM db.table WHERE id IN (k1, k2, ...)`. Alınan veriler daha sonra önbelleğe yazılır. - -Önbellek sözlükleri için, sona erme [ömür](external-dicts-dict-lifetime.md) önbellekteki verilerin ayarlanabilir. Eğer daha fazla zaman `lifetime` bir hücrede veri yüklenmesinden bu yana geçti, hücrenin değeri kullanılmaz ve bir dahaki sefere kullanılması gerektiğinde yeniden istenir. -Bu, sözlükleri saklamanın tüm yollarından en az etkilidir. Önbelleğin hızı, doğru ayarlara ve kullanım senaryosuna bağlıdır. Bir önbellek türü sözlüğü, yalnızca isabet oranları yeterince yüksek olduğunda (önerilen %99 ve daha yüksek) iyi performans gösterir. Sen ortalama isabet oranı görebilirsiniz `system.dictionaries` Tablo. - -Önbellek performansını artırmak için bir alt sorgu ile kullanın `LIMIT`, ve harici sözlük ile işlevini çağırın. - -Destek [kaynaklar](external-dicts-dict-sources.md): MySQL, ClickHouse, yürütülebilir, HTTP. - -Ayarlar örneği: - -``` xml - - - - 1000000000 - - -``` - -veya - -``` sql -LAYOUT(CACHE(SIZE_IN_CELLS 1000000000)) -``` - -Yeterince büyük bir önbellek boyutu ayarlayın. Sen hücre sayısını seçmek için deneme gerekir: - -1. Bazı değer ayarlayın. -2. Önbellek tamamen doluncaya kadar sorguları çalıştırın. -3. Kullanarak bellek tüketimini değerlendirmek `system.dictionaries` Tablo. -4. Gerekli bellek tüketimine ulaşılana kadar hücre sayısını artırın veya azaltın. - -!!! warning "Uyarıcı" - Rasgele okuma ile sorguları işlemek için yavaş olduğundan, ClickHouse kaynak olarak kullanmayın. - -### complex_key_cache {#complex-key-cache} - -Bu tür depolama kompozit ile kullanım içindir [anahtarlar](external-dicts-dict-structure.md). Benzer `cache`. - -### direkt {#direct} - -Sözlük bellekte saklanmaz ve bir isteğin işlenmesi sırasında doğrudan kaynağa gider. - -Sözlük anahtarı vardır `UInt64` tür. - -Her türlü [kaynaklar](external-dicts-dict-sources.md), yerel dosyalar dışında desteklenir. - -Yapılandırma örneği: - -``` xml - - - -``` - -veya - -``` sql -LAYOUT(DIRECT()) -``` - -### ıp_trie {#ip-trie} - -Bu tür depolama, ağ öneklerini (IP adresleri) asn gibi meta verilere eşlemek içindir. - -Örnek: tablo, ağ önekleri ve bunlara karşılık gelen sayı ve ülke kodu içerir: - -``` text - +-----------|-----|------+ - | prefix | asn | cca2 | - +=================+=======+========+ - | 202.79.32.0/20 | 17501 | NP | - +-----------|-----|------+ - | 2620:0:870::/48 | 3856 | US | - +-----------|-----|------+ - | 2a02:6b8:1::/48 | 13238 | RU | - +-----------|-----|------+ - | 2001:db8::/32 | 65536 | ZZ | - +-----------|-----|------+ -``` - -Bu tür bir düzen kullanırken, yapının bileşik bir anahtarı olmalıdır. - -Örnek: - -``` xml - - - - prefix - String - - - - asn - UInt32 - - - - cca2 - String - ?? - - ... -``` - -veya - -``` sql -CREATE DICTIONARY somedict ( - prefix String, - asn UInt32, - cca2 String DEFAULT '??' -) -PRIMARY KEY prefix -``` - -Anahtarın izin verilen bir IP öneki içeren yalnızca bir dize türü özniteliği olması gerekir. Diğer türler henüz desteklenmiyor. - -Sorgular için aynı işlevleri kullanmanız gerekir (`dictGetT` bir tuple ile) kompozit tuşları ile sözlükler gelince: - -``` sql -dictGetT('dict_name', 'attr_name', tuple(ip)) -``` - -İşlev ya alır `UInt32` IPv4 için veya `FixedString(16)` IPv6 için: - -``` sql -dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1'))) -``` - -Diğer türler henüz desteklenmiyor. İşlev, bu IP adresine karşılık gelen önek için özniteliği döndürür. Örtüşen önekler varsa, en spesifik olanı döndürülür. - -Veri bir saklanan `trie`. Tamamen RAM'e uyması gerekir. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_layout/) diff --git a/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md b/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md deleted file mode 100644 index 648a8bbd5a9..00000000000 --- a/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md +++ /dev/null @@ -1,91 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 42 -toc_title: "S\xF6zl\xFCk G\xFCncellemeleri" ---- - -# Sözlük Güncellemeleri {#dictionary-updates} - -ClickHouse sözlükleri periyodik olarak günceller. Tam olarak karşıdan yüklenen sözlükler için Güncelleştirme aralığı ve önbelleğe alınmış sözlükler için geçersiz kılma aralığı `` saniyeler içinde etiketleyin. - -Sözlük güncelleştirmeleri (ilk kullanım için yükleme dışında) sorguları engellemez. Güncellemeler sırasında, bir sözlüğün eski sürümü kullanılır. Güncelleştirme sırasında bir hata oluşursa, hata sunucu günlüğüne yazılır ve sorgular sözlüklerin eski sürümünü kullanmaya devam eder. - -Ayarlar örneği: - -``` xml - - ... - 300 - ... - -``` - -``` sql -CREATE DICTIONARY (...) -... -LIFETIME(300) -... -``` - -Ayar `0` (`LIFETIME(0)`) söz dictionarieslük .lerin güncel updatinglenmesini engeller. - -Yükseltmeler için bir zaman aralığı ayarlayabilirsiniz ve ClickHouse bu aralıkta eşit rastgele bir zaman seçecektir. Bu, çok sayıda sunucuda yükseltme yaparken yükü sözlük kaynağına dağıtmak için gereklidir. - -Ayarlar örneği: - -``` xml - - ... - - 300 - 360 - - ... - -``` - -veya - -``` sql -LIFETIME(MIN 300 MAX 360) -``` - -Eğer `0` ve `0`, ClickHouse sözlüğü zaman aşımı ile yeniden yüklemez. -Bu durumda, Sözlük yapılandırma dosyası değiştirilmişse veya ClickHouse sözlüğü daha önce yeniden yükleyebilir. `SYSTEM RELOAD DICTIONARY` komut yürütüldü. - -Sözlükleri yükseltirken, ClickHouse sunucusu türüne bağlı olarak farklı mantık uygular [kaynaklı](external-dicts-dict-sources.md): - -Sözlükleri yükseltirken, ClickHouse sunucusu türüne bağlı olarak farklı mantık uygular [kaynaklı](external-dicts-dict-sources.md): - -- Bir metin dosyası için değişiklik zamanını kontrol eder. Zaman önceden kaydedilmiş zaman farklıysa, sözlük güncelleştirilir. -- Myısam tabloları için, değişiklik zamanı bir `SHOW TABLE STATUS` sorgu. -- Diğer kaynaklardan gelen sözlükler varsayılan olarak her zaman güncellenir. - -MySQL (InnoDB), ODBC ve ClickHouse kaynakları için, sözlükleri her seferinde değil, gerçekten değiştiyse güncelleyecek bir sorgu ayarlayabilirsiniz. Bunu yapmak için şu adımları izleyin: - -- Sözlük tablosu, kaynak verileri güncelleştirildiğinde her zaman değişen bir alana sahip olmalıdır. -- Kaynak ayarları, değişen alanı alan bir sorgu belirtmeniz gerekir. ClickHouse sunucu sorgu sonucu bir satır olarak yorumlar ve bu satır önceki durumuna göre değişmişse, sözlük güncelleştirilir. Sorguda belirtme `` için ayar fieldlardaki alan [kaynaklı](external-dicts-dict-sources.md). - -Ayarlar örneği: - -``` xml - - ... - - ... - SELECT update_time FROM dictionary_source where id = 1 - - ... - -``` - -veya - -``` sql -... -SOURCE(ODBC(... invalidate_query 'SELECT update_time FROM dictionary_source where id = 1')) -... -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_lifetime/) diff --git a/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md deleted file mode 100644 index 901248231e2..00000000000 --- a/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ /dev/null @@ -1,630 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 43 -toc_title: "D\u0131\u015F S\xF6zl\xFCklerin kaynaklar\u0131" ---- - -# Dış Sözlüklerin kaynakları {#dicts-external-dicts-dict-sources} - -Harici bir sözlük birçok farklı kaynaktan bağlanabilir. - -Sözlük xml dosyası kullanılarak yapılandırılmışsa, yapılandırma şöyle görünür: - -``` xml - - - ... - - - - - - ... - - ... - -``` - -Durumunda [DDL-sorgu](../../statements/create.md#create-dictionary-query), eşit yapılandırma gibi görünüyor olacak: - -``` sql -CREATE DICTIONARY dict_name (...) -... -SOURCE(SOURCE_TYPE(param1 val1 ... paramN valN)) -- Source configuration -... -``` - -Kaynak yapılandırılmış `source` bölme. - -Kaynak türleri için [Yerel dosya](#dicts-external_dicts_dict_sources-local_file), [Yürütülebilir dosya](#dicts-external_dicts_dict_sources-executable), [HTTP (s)](#dicts-external_dicts_dict_sources-http), [ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) -isteğe bağlı ayarlar mevcuttur: - -``` xml - - - /opt/dictionaries/os.tsv - TabSeparated - - - 0 - - -``` - -veya - -``` sql -SOURCE(FILE(path '/opt/dictionaries/os.tsv' format 'TabSeparated')) -SETTINGS(format_csv_allow_single_quotes = 0) -``` - -Kaynak türleri (`source_type`): - -- [Yerel dosya](#dicts-external_dicts_dict_sources-local_file) -- [Yürütülebilir dosya](#dicts-external_dicts_dict_sources-executable) -- [HTTP (s)](#dicts-external_dicts_dict_sources-http) -- DBMS - - [ODBC](#dicts-external_dicts_dict_sources-odbc) - - [MySQL](#dicts-external_dicts_dict_sources-mysql) - - [ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) - - [MongoDB](#dicts-external_dicts_dict_sources-mongodb) - - [Redis](#dicts-external_dicts_dict_sources-redis) - -## Yerel Dosya {#dicts-external_dicts_dict_sources-local_file} - -Ayarlar örneği: - -``` xml - - - /opt/dictionaries/os.tsv - TabSeparated - - -``` - -veya - -``` sql -SOURCE(FILE(path '/opt/dictionaries/os.tsv' format 'TabSeparated')) -``` - -Ayar alanları: - -- `path` – The absolute path to the file. -- `format` – The file format. All the formats described in “[Biçimliler](../../../interfaces/formats.md#formats)” desteklenir. - -## Yürütülebilir Dosya {#dicts-external_dicts_dict_sources-executable} - -Yürütülebilir dosyalarla çalışmak Aşağıdakilere bağlıdır [sözlük bellekte nasıl saklanır](external-dicts-dict-layout.md). Sözlük kullanılarak saklan theıyorsa `cache` ve `complex_key_cache` ClickHouse, yürütülebilir dosyanın STDIN'SİNE bir istek göndererek gerekli anahtarları ister. Aksi takdirde, clickhouse yürütülebilir dosyayı başlatır ve çıktısını sözlük verileri olarak değerlendirir. - -Ayarlar örneği: - -``` xml - - - cat /opt/dictionaries/os.tsv - TabSeparated - - -``` - -veya - -``` sql -SOURCE(EXECUTABLE(command 'cat /opt/dictionaries/os.tsv' format 'TabSeparated')) -``` - -Ayar alanları: - -- `command` – The absolute path to the executable file, or the file name (if the program directory is written to `PATH`). -- `format` – The file format. All the formats described in “[Biçimliler](../../../interfaces/formats.md#formats)” desteklenir. - -## Http (s) {#dicts-external_dicts_dict_sources-http} - -Bir HTTP (s) sunucusuyla çalışmak Aşağıdakilere bağlıdır [sözlük bellekte nasıl saklanır](external-dicts-dict-layout.md). Sözlük kullanılarak saklan theıyorsa `cache` ve `complex_key_cache`, ClickHouse aracılığıyla bir istek göndererek gerekli anahtarları ister `POST` yöntem. - -Ayarlar örneği: - -``` xml - - - http://[::1]/os.tsv - TabSeparated - - user - password - - -
- API-KEY - key -
- - - -``` - -veya - -``` sql -SOURCE(HTTP( - url 'http://[::1]/os.tsv' - format 'TabSeparated' - credentials(user 'user' password 'password') - headers(header(name 'API-KEY' value 'key')) -)) -``` - -Clickhouse'un bir HTTPS kaynağına erişebilmesi için şunları yapmanız gerekir [openssl'yi yapılandırma](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-openssl) sunucu yapılandırmasında. - -Ayar alanları: - -- `url` – The source URL. -- `format` – The file format. All the formats described in “[Biçimliler](../../../interfaces/formats.md#formats)” desteklenir. -- `credentials` – Basic HTTP authentication. Optional parameter. - - `user` – Username required for the authentication. - - `password` – Password required for the authentication. -- `headers` – All custom HTTP headers entries used for the HTTP request. Optional parameter. - - `header` – Single HTTP header entry. - - `name` – Identifiant name used for the header send on the request. - - `value` – Value set for a specific identifiant name. - -## ODBC {#dicts-external_dicts_dict_sources-odbc} - -ODBC sürücüsü olan herhangi bir veritabanını bağlamak için bu yöntemi kullanabilirsiniz. - -Ayarlar örneği: - -``` xml - - - DatabaseName - ShemaName.TableName
- DSN=some_parameters - SQL_QUERY - - -``` - -veya - -``` sql -SOURCE(ODBC( - db 'DatabaseName' - table 'SchemaName.TableName' - connection_string 'DSN=some_parameters' - invalidate_query 'SQL_QUERY' -)) -``` - -Ayar alanları: - -- `db` – Name of the database. Omit it if the database name is set in the `` parametre. -- `table` – Name of the table and schema if exists. -- `connection_string` – Connection string. -- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Sözlükleri güncelleme](external-dicts-dict-lifetime.md). - -ClickHouse, ODBC sürücüsünden alıntı sembolleri alır ve sorgulardaki tüm ayarları sürücüye aktarır, bu nedenle tablo adını veritabanındaki tablo adı durumuna göre ayarlamak gerekir. - -Oracle kullanırken kodlamalarla ilgili bir sorununuz varsa, ilgili [FAQ](../../../faq/general.md#oracle-odbc-encodings) Makale. - -### ODBC Sözlük işlevselliği bilinen güvenlik açığı {#known-vulnerability-of-the-odbc-dictionary-functionality} - -!!! attention "Dikkat" - ODBC sürücüsü bağlantı parametresi aracılığıyla veritabanına bağlanırken `Servername` yerine. Bu durumda değerler `USERNAME` ve `PASSWORD` itibaren `odbc.ini` uzak sunucuya gönderilir ve tehlikeye girebilir. - -**Güvensiz kullanım örneği** - -PostgreSQL için unixodbc'yi yapılandıralım. İçeriği `/etc/odbc.ini`: - -``` text -[gregtest] -Driver = /usr/lib/psqlodbca.so -Servername = localhost -PORT = 5432 -DATABASE = test_db -#OPTION = 3 -USERNAME = test -PASSWORD = test -``` - -Daha sonra aşağıdaki gibi bir sorgu yaparsanız - -``` sql -SELECT * FROM odbc('DSN=gregtest;Servername=some-server.com', 'test_db'); -``` - -ODBC sürücüsü değerleri gönderir `USERNAME` ve `PASSWORD` itibaren `odbc.ini` -e doğru `some-server.com`. - -### PostgreSQL bağlanma örneği {#example-of-connecting-postgresql} - -UB .untu OS. - -PostgreSQL için UNİXODBC ve ODBC sürücüsünü yükleme: - -``` bash -$ sudo apt-get install -y unixodbc odbcinst odbc-postgresql -``` - -Yapılandırma `/etc/odbc.ini` (veya `~/.odbc.ini`): - -``` text - [DEFAULT] - Driver = myconnection - - [myconnection] - Description = PostgreSQL connection to my_db - Driver = PostgreSQL Unicode - Database = my_db - Servername = 127.0.0.1 - UserName = username - Password = password - Port = 5432 - Protocol = 9.3 - ReadOnly = No - RowVersioning = No - ShowSystemTables = No - ConnSettings = -``` - -Clickhouse'da sözlük yapılandırması: - -``` xml - - - table_name - - - - - DSN=myconnection - postgresql_table
- - - - 300 - 360 - - - - - - - id - - - some_column - UInt64 - 0 - - - - -``` - -veya - -``` sql -CREATE DICTIONARY table_name ( - id UInt64, - some_column UInt64 DEFAULT 0 -) -PRIMARY KEY id -SOURCE(ODBC(connection_string 'DSN=myconnection' table 'postgresql_table')) -LAYOUT(HASHED()) -LIFETIME(MIN 300 MAX 360) -``` - -Düzenlemeniz gerekebilir `odbc.ini` sürücü ile kitaplığın tam yolunu belirtmek için `DRIVER=/usr/local/lib/psqlodbcw.so`. - -### MS SQL Server bağlanma örneği {#example-of-connecting-ms-sql-server} - -UB .untu OS. - -Sürücüyü yükleme: : - -``` bash -$ sudo apt-get install tdsodbc freetds-bin sqsh -``` - -Sürücüyü yapılandırma: - -``` bash - $ cat /etc/freetds/freetds.conf - ... - - [MSSQL] - host = 192.168.56.101 - port = 1433 - tds version = 7.0 - client charset = UTF-8 - - $ cat /etc/odbcinst.ini - ... - - [FreeTDS] - Description = FreeTDS - Driver = /usr/lib/x86_64-linux-gnu/odbc/libtdsodbc.so - Setup = /usr/lib/x86_64-linux-gnu/odbc/libtdsS.so - FileUsage = 1 - UsageCount = 5 - - $ cat ~/.odbc.ini - ... - - [MSSQL] - Description = FreeTDS - Driver = FreeTDS - Servername = MSSQL - Database = test - UID = test - PWD = test - Port = 1433 -``` - -Clickhouse'da sözlüğü yapılandırma: - -``` xml - - - test - - - dict
- DSN=MSSQL;UID=test;PWD=test - - - - - 300 - 360 - - - - - - - - - k - - - s - String - - - - - -``` - -veya - -``` sql -CREATE DICTIONARY test ( - k UInt64, - s String DEFAULT '' -) -PRIMARY KEY k -SOURCE(ODBC(table 'dict' connection_string 'DSN=MSSQL;UID=test;PWD=test')) -LAYOUT(FLAT()) -LIFETIME(MIN 300 MAX 360) -``` - -## DBMS {#dbms} - -### Mysql {#dicts-external_dicts_dict_sources-mysql} - -Ayarlar örneği: - -``` xml - - - 3306 - clickhouse - qwerty - - example01-1 - 1 - - - example01-2 - 1 - - db_name - table_name
- id=10 - SQL_QUERY - - -``` - -veya - -``` sql -SOURCE(MYSQL( - port 3306 - user 'clickhouse' - password 'qwerty' - replica(host 'example01-1' priority 1) - replica(host 'example01-2' priority 1) - db 'db_name' - table 'table_name' - where 'id=10' - invalidate_query 'SQL_QUERY' -)) -``` - -Ayar alanları: - -- `port` – The port on the MySQL server. You can specify it for all replicas, or for each one individually (inside ``). - -- `user` – Name of the MySQL user. You can specify it for all replicas, or for each one individually (inside ``). - -- `password` – Password of the MySQL user. You can specify it for all replicas, or for each one individually (inside ``). - -- `replica` – Section of replica configurations. There can be multiple sections. - - - `replica/host` – The MySQL host. - - `replica/priority` – The replica priority. When attempting to connect, ClickHouse traverses the replicas in order of priority. The lower the number, the higher the priority. - -- `db` – Name of the database. - -- `table` – Name of the table. - -- `where` – The selection criteria. The syntax for conditions is the same as for `WHERE` MySQL, örneğin, `id > 10 AND id < 20`. İsteğe bağlı parametre. - -- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Sözlükleri güncelleme](external-dicts-dict-lifetime.md). - -MySQL yuva üzerinden yerel bir ana bilgisayara bağlanabilir. Bunu yapmak için, ayarlayın `host` ve `socket`. - -Ayarlar örneği: - -``` xml - - - localhost - /path/to/socket/file.sock - clickhouse - qwerty - db_name - table_name
- id=10 - SQL_QUERY - - -``` - -veya - -``` sql -SOURCE(MYSQL( - host 'localhost' - socket '/path/to/socket/file.sock' - user 'clickhouse' - password 'qwerty' - db 'db_name' - table 'table_name' - where 'id=10' - invalidate_query 'SQL_QUERY' -)) -``` - -### ClickHouse {#dicts-external_dicts_dict_sources-clickhouse} - -Ayarlar örneği: - -``` xml - - - example01-01-1 - 9000 - default - - default - ids
- id=10 - - -``` - -veya - -``` sql -SOURCE(CLICKHOUSE( - host 'example01-01-1' - port 9000 - user 'default' - password '' - db 'default' - table 'ids' - where 'id=10' -)) -``` - -Ayar alanları: - -- `host` – The ClickHouse host. If it is a local host, the query is processed without any network activity. To improve fault tolerance, you can create a [Dağılı](../../../engines/table-engines/special/distributed.md) tablo ve sonraki yapılandırmalarda girin. -- `port` – The port on the ClickHouse server. -- `user` – Name of the ClickHouse user. -- `password` – Password of the ClickHouse user. -- `db` – Name of the database. -- `table` – Name of the table. -- `where` – The selection criteria. May be omitted. -- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Sözlükleri güncelleme](external-dicts-dict-lifetime.md). - -### Mongodb {#dicts-external_dicts_dict_sources-mongodb} - -Ayarlar örneği: - -``` xml - - - localhost - 27017 - - - test - dictionary_source - - -``` - -veya - -``` sql -SOURCE(MONGO( - host 'localhost' - port 27017 - user '' - password '' - db 'test' - collection 'dictionary_source' -)) -``` - -Ayar alanları: - -- `host` – The MongoDB host. -- `port` – The port on the MongoDB server. -- `user` – Name of the MongoDB user. -- `password` – Password of the MongoDB user. -- `db` – Name of the database. -- `collection` – Name of the collection. - -### Redis {#dicts-external_dicts_dict_sources-redis} - -Ayarlar örneği: - -``` xml - - - localhost - 6379 - simple - 0 - - -``` - -veya - -``` sql -SOURCE(REDIS( - host 'localhost' - port 6379 - storage_type 'simple' - db_index 0 -)) -``` - -Ayar alanları: - -- `host` – The Redis host. -- `port` – The port on the Redis server. -- `storage_type` – The structure of internal Redis storage using for work with keys. `simple` basit kaynaklar ve karma tek anahtar kaynaklar içindir, `hash_map` iki anahtarlı karma kaynaklar içindir. Ranged kaynakları ve karmaşık anahtarlı önbellek kaynakları desteklenmez. İhmal edilebilir, varsayılan değer `simple`. -- `db_index` – The specific numeric index of Redis logical database. May be omitted, default value is 0. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_sources/) diff --git a/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md b/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md deleted file mode 100644 index 6fd80f860c2..00000000000 --- a/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md +++ /dev/null @@ -1,175 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 44 -toc_title: "S\xF6zl\xFCk anahtar\u0131 ve alanlar\u0131" ---- - -# Sözlük anahtarı ve alanları {#dictionary-key-and-fields} - -Bu `` yan tümcesi sözlük anahtarı ve sorgular için kullanılabilir alanları açıklar. - -XML açıklaması: - -``` xml - - - - Id - - - - - - - ... - - - -``` - -Nitelikler elemanlarda açıklanmıştır: - -- `` — [Anahtar sütun](external-dicts-dict-structure.md#ext_dict_structure-key). -- `` — [Veri sütunu](external-dicts-dict-structure.md#ext_dict_structure-attributes). Birden fazla sayıda özellik olabilir. - -DDL sorgusu: - -``` sql -CREATE DICTIONARY dict_name ( - Id UInt64, - -- attributes -) -PRIMARY KEY Id -... -``` - -Öznitelikler sorgu gövdesinde açıklanmıştır: - -- `PRIMARY KEY` — [Anahtar sütun](external-dicts-dict-structure.md#ext_dict_structure-key) -- `AttrName AttrType` — [Veri sütunu](external-dicts-dict-structure.md#ext_dict_structure-attributes). Birden fazla sayıda özellik olabilir. - -## Anahtar {#ext_dict_structure-key} - -ClickHouse aşağıdaki anahtar türlerini destekler: - -- Sayısal tuş. `UInt64`. Tanımlanan `` etiket veya kullanma `PRIMARY KEY` kelime. -- Kompozit anahtar. Farklı türde değerler kümesi. Etiket definedinde tanımlı `` veya `PRIMARY KEY` kelime. - -Bir xml yapısı şunları içerebilir `` veya ``. DDL sorgusu tek içermelidir `PRIMARY KEY`. - -!!! warning "Uyarıcı" - Anahtarı bir öznitelik olarak tanımlamamalısınız. - -### Sayısal Tuş {#ext_dict-numeric-key} - -Tür: `UInt64`. - -Yapılandırma örneği: - -``` xml - - Id - -``` - -Yapılandırma alanları: - -- `name` – The name of the column with keys. - -DDL sorgusu için: - -``` sql -CREATE DICTIONARY ( - Id UInt64, - ... -) -PRIMARY KEY Id -... -``` - -- `PRIMARY KEY` – The name of the column with keys. - -### Kompozit Anahtar {#composite-key} - -Anahtar bir olabilir `tuple` her türlü alandan. Bu [düzen](external-dicts-dict-layout.md) bu durumda olmalıdır `complex_key_hashed` veya `complex_key_cache`. - -!!! tip "Uç" - Bileşik bir anahtar tek bir elemandan oluşabilir. Bu, örneğin bir dizeyi anahtar olarak kullanmayı mümkün kılar. - -Anahtar yapısı eleman ayarlanır ``. Anahtar alanlar sözlük ile aynı biçimde belirtilir [öznitelik](external-dicts-dict-structure.md). Örnek: - -``` xml - - - - field1 - String - - - field2 - UInt32 - - ... - -... -``` - -veya - -``` sql -CREATE DICTIONARY ( - field1 String, - field2 String - ... -) -PRIMARY KEY field1, field2 -... -``` - -Bir sorgu için `dictGet*` fonksiyon, bir tuple anahtar olarak geçirilir. Örnek: `dictGetString('dict_name', 'attr_name', tuple('string for field1', num_for_field2))`. - -## Öznitelik {#ext_dict_structure-attributes} - -Yapılandırma örneği: - -``` xml - - ... - - Name - ClickHouseDataType - - rand64() - true - true - true - - -``` - -veya - -``` sql -CREATE DICTIONARY somename ( - Name ClickHouseDataType DEFAULT '' EXPRESSION rand64() HIERARCHICAL INJECTIVE IS_OBJECT_ID -) -``` - -Yapılandırma alanları: - -| Etiket | Açıklama | Gerekli | -|------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| -| `name` | Sütun adı. | Evet | -| `type` | ClickHouse veri türü.
ClickHouse, sözlükten belirtilen veri türüne değer atmaya çalışır. Örneğin, MySQL için alan olabilir `TEXT`, `VARCHAR`, veya `BLOB` MySQL kaynak tablosunda, ancak şu şekilde yüklenebilir `String` Clickhouse'da.
[Nullable](../../../sql-reference/data-types/nullable.md) desteklenmiyor. | Evet | -| `null_value` | Varolan olmayan bir öğe için varsayılan değer.
Örnekte, boş bir dizedir. Kullanamazsınız `NULL` bu alanda. | Evet | -| `expression` | [İfade](../../syntax.md#syntax-expressions) bu ClickHouse değeri yürütür.
İfade, uzak SQL veritabanında bir sütun adı olabilir. Bu nedenle, uzak sütun için bir diğer ad oluşturmak için kullanabilirsiniz.

Varsayılan değer: ifade yok. | Hayır | -| `hierarchical` | Eğer `true`, öznitelik, geçerli anahtar için bir üst anahtarın değerini içerir. Görmek [Hiyerarşik Sözlükler](external-dicts-dict-hierarchical.md).

Varsayılan değer: `false`. | Hayır | -| `injective` | Olup olmadığını gösteren bayrak `id -> attribute` ima isge is [enjektif](https://en.wikipedia.org/wiki/Injective_function).
Eğer `true`, ClickHouse sonra otomatik olarak yerleştirebilirsiniz `GROUP BY` fık .ra ile ilgili istek dictionariesleriniz Genellikle bu tür taleplerin miktarını önemli ölçüde azaltır.

Varsayılan değer: `false`. | Hayır | -| `is_object_id` | Bir MongoDB belgesi için sorgunun yürütülüp yürütülmediğini gösteren bayrak `ObjectID`.

Varsayılan değer: `false`. | Hayır | - -## Ayrıca Bakınız {#see-also} - -- [Harici sözlüklerle çalışmak için işlevler](../../../sql-reference/functions/ext-dict-functions.md). - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_structure/) diff --git a/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md b/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md deleted file mode 100644 index a039f75ac44..00000000000 --- a/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 40 -toc_title: "Harici bir s\xF6zl\xFCk yap\u0131land\u0131rma" ---- - -# Harici bir sözlük yapılandırma {#dicts-external-dicts-dict} - -Sözlük xml dosyası kullanılarak yapılandırılmışsa, sözlük yapılandırması aşağıdaki yapıya sahiptir: - -``` xml - - dict_name - - - - - - - - - - - - - - - - - -``` - -İlgili [DDL-sorgu](../../statements/create.md#create-dictionary-query) aşağıdaki yapıya sahiptir: - -``` sql -CREATE DICTIONARY dict_name -( - ... -- attributes -) -PRIMARY KEY ... -- complex or single key configuration -SOURCE(...) -- Source configuration -LAYOUT(...) -- Memory layout configuration -LIFETIME(...) -- Lifetime of dictionary in memory -``` - -- `name` – The identifier that can be used to access the dictionary. Use the characters `[a-zA-Z0-9_\-]`. -- [kaynaklı](external-dicts-dict-sources.md) — Source of the dictionary. -- [düzen](external-dicts-dict-layout.md) — Dictionary layout in memory. -- [yapılı](external-dicts-dict-structure.md) — Structure of the dictionary . A key and attributes that can be retrieved by this key. -- [ömür](external-dicts-dict-lifetime.md) — Frequency of dictionary updates. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict/) diff --git a/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts.md b/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts.md deleted file mode 100644 index cfbbc1a1ad8..00000000000 --- a/docs/tr/sql-reference/dictionaries/external-dictionaries/external-dicts.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 39 -toc_title: "Genel A\xE7\u0131klama" ---- - -# Dış Söz Dictionarieslükler {#dicts-external-dicts} - -Çeşitli veri kaynaklarından kendi sözlükleri ekleyebilirsiniz. Bir sözlük için veri kaynağı, yerel bir metin veya yürütülebilir dosya, bir HTTP(s) kaynağı veya başka bir DBMS olabilir. Daha fazla bilgi için, bkz. “[Dış sözlükler için kaynaklar](external-dicts-dict-sources.md)”. - -ClickHouse: - -- Sözlükleri RAM'de tamamen veya kısmen saklar. -- Sözlükleri periyodik olarak günceller ve eksik değerleri dinamik olarak yükler. Başka bir deyişle, sözlükler dinamik olarak yüklenebilir. -- Xml dosyaları ile harici sözlükler oluşturmak için izin verir veya [DDL sorguları](../../statements/create.md#create-dictionary-query). - -Dış sözlüklerin yapılandırması bir veya daha fazla xml dosyasında bulunabilir. Yapılandırma yolu belirtilen [dictionaries_config](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_config) parametre. - -Sözlükler sunucu başlangıçta veya ilk kullanımda, bağlı olarak yüklenebilir [dictionaries_lazy_load](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_lazy_load) ayar. - -Bu [sözlükler](../../../operations/system-tables.md#system_tables-dictionaries) sistem tablosu sunucuda yapılandırılmış sözlükler hakkında bilgi içerir. Her sözlük için orada bulabilirsiniz: - -- Sözlük durumu. -- Yapılandırma parametreleri. -- Sözlük için ayrılan RAM miktarı veya sözlük başarıyla yüklendiğinden bu yana bir dizi sorgu gibi metrikler. - -Sözlük yapılandırma dosyası aşağıdaki biçime sahiptir: - -``` xml - - An optional element with any content. Ignored by the ClickHouse server. - - - /etc/metrika.xml - - - - - - - - -``` - -Yapabilirsin [yapılandırmak](external-dicts-dict.md) aynı dosyada sözlükler herhangi bir sayıda. - -[Sözlükler için DDL sorguları](../../statements/create.md#create-dictionary-query) sunucu yapılandırmasında herhangi bir ek kayıt gerektirmez. Tablolar veya görünümler gibi birinci sınıf varlıklar olarak sözlüklerle çalışmaya izin verirler. - -!!! attention "Dikkat" - Küçük bir sözlük için değerleri, bir `SELECT` sorgu (bkz. [dönüştürmek](../../../sql-reference/functions/other-functions.md) işlev). Bu işlevsellik harici sözlüklerle ilgili değildir. - -## Ayrıca Bakınız {#ext-dicts-see-also} - -- [Harici bir sözlük yapılandırma](external-dicts-dict.md) -- [Sözlükleri bellekte saklama](external-dicts-dict-layout.md) -- [Sözlük Güncellemeleri](external-dicts-dict-lifetime.md) -- [Dış Sözlüklerin kaynakları](external-dicts-dict-sources.md) -- [Sözlük anahtarı ve alanları](external-dicts-dict-structure.md) -- [Harici Sözlüklerle çalışmak için işlevler](../../../sql-reference/functions/ext-dict-functions.md) - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts/) diff --git a/docs/tr/sql-reference/dictionaries/external-dictionaries/index.md b/docs/tr/sql-reference/dictionaries/external-dictionaries/index.md deleted file mode 100644 index d0ce40069e4..00000000000 --- a/docs/tr/sql-reference/dictionaries/external-dictionaries/index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "D\u0131\u015F S\xF6z Dictionariesl\xFCkler" -toc_priority: 37 ---- - - diff --git a/docs/tr/sql-reference/dictionaries/index.md b/docs/tr/sql-reference/dictionaries/index.md deleted file mode 100644 index 68f3f7a97be..00000000000 --- a/docs/tr/sql-reference/dictionaries/index.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "S\xF6zl\xFCkler" -toc_priority: 35 -toc_title: "Giri\u015F" ---- - -# Sözlükler {#dictionaries} - -Bir sözlük bir haritalama (`key -> attributes`) bu referans listeleri çeşitli türleri için uygundur. - -ClickHouse, sorgularda kullanılabilecek sözlüklerle çalışmak için özel işlevleri destekler. Sözlükleri işlevlerle kullanmak daha kolay ve daha verimlidir. `JOIN` referans tabloları ile. - -[NULL](../../sql-reference/syntax.md#null-literal) değerler sözlükte saklanamaz. - -ClickHouse destekler: - -- [Dahili sözlükler](internal-dicts.md#internal_dicts) bir özel ile [fonksiyonlar kümesi](../../sql-reference/functions/ym-dict-functions.md). -- [Eklenti (harici) söz dictionarieslükler](external-dictionaries/external-dicts.md#dicts-external-dicts) ile bir [fonksiyonlar kümesi](../../sql-reference/functions/ext-dict-functions.md). - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/) diff --git a/docs/tr/sql-reference/dictionaries/internal-dicts.md b/docs/tr/sql-reference/dictionaries/internal-dicts.md deleted file mode 100644 index 05aa218ed9f..00000000000 --- a/docs/tr/sql-reference/dictionaries/internal-dicts.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 39 -toc_title: "\u0130\xE7 S\xF6z Dictionariesl\xFCkler" ---- - -# İç Söz Dictionarieslükler {#internal_dicts} - -ClickHouse, bir geobase ile çalışmak için yerleşik bir özellik içerir. - -Bu size sağlar: - -- Adını istediğiniz dilde almak için bölgenin kimliğini kullanın. -- Bir şehir, bölge, federal bölge, ülke veya kıtanın kimliğini almak için bölgenin kimliğini kullanın. -- Bir bölgenin başka bir bölgenin parçası olup olmadığını kontrol edin. -- Ana bölgeler zinciri alın. - -Tüm fonksiyonları destek “translocality,” aynı anda bölge mülkiyeti farklı bakış açıları kullanma yeteneği. Daha fazla bilgi için bölüme bakın “Functions for working with Yandex.Metrica dictionaries”. - -İç sözlükler varsayılan pakette devre dışı bırakılır. -Bunları etkinleştirmek için, parametreleri uncomment `path_to_regions_hierarchy_file` ve `path_to_regions_names_files` sunucu yapılandırma dosyasında. - -Geobase metin dosyalarından yüklenir. - -Place the `regions_hierarchy*.txt` dosyaları içine `path_to_regions_hierarchy_file` dizin. Bu yapılandırma parametresi, `regions_hierarchy.txt` dosya (varsayılan bölgesel hiyerarşi) ve diğer dosyalar (`regions_hierarchy_ua.txt`) aynı dizinde bulunmalıdır. - -Koy... `regions_names_*.txt` dosyalar içinde `path_to_regions_names_files` dizin. - -Bu dosyaları kendiniz de oluşturabilirsiniz. Dosya biçimi aşağıdaki gibidir: - -`regions_hierarchy*.txt`: TabSeparated (başlık yok), sütunlar: - -- bölge kimliği (`UInt32`) -- üst bölge kimliği (`UInt32`) -- bölge tipi (`UInt8`): 1 - kıta, 3-ülke, 4-federal bölge, 5-bölge, 6-şehir; diğer türlerin değerleri yoktur -- nüfuslu (`UInt32`) — optional column - -`regions_names_*.txt`: TabSeparated (başlık yok), sütunlar: - -- bölge kimliği (`UInt32`) -- bölge adı (`String`) — Can't contain tabs or line feeds, even escaped ones. - -RAM'de depolamak için düz bir dizi kullanılır. Bu nedenle, IDs bir milyondan fazla olmamalıdır. - -Sözlükler sunucuyu yeniden başlatmadan güncellenebilir. Ancak, kullanılabilir sözlükler kümesi güncelleştirilmez. -Güncellemeler için dosya değiştirme süreleri kontrol edilir. Bir dosya değiştiyse, sözlük güncelleştirilir. -Değişiklikleri kontrol etmek için Aralık `builtin_dictionaries_reload_interval` parametre. -Sözlük güncelleştirmeleri (ilk kullanımda yükleme dışında) sorguları engellemez. Güncelleştirmeler sırasında, sorgular sözlüklerin eski sürümlerini kullanır. Güncelleştirme sırasında bir hata oluşursa, hata sunucu günlüğüne yazılır ve sorgular sözlüklerin eski sürümünü kullanmaya devam eder. - -Sözlükleri geobase ile periyodik olarak güncellemenizi öneririz. Bir güncelleme sırasında yeni dosyalar oluşturun ve bunları ayrı bir konuma yazın. Her şey hazır olduğunda, bunları sunucu tarafından kullanılan dosyalara yeniden adlandırın. - -OS tanımlayıcıları ve Yandex ile çalışmak için işlevler de vardır.Metrica arama motorları, ancak kullanılmamalıdır. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/internal_dicts/) diff --git a/docs/tr/sql-reference/functions/arithmetic-functions.md b/docs/tr/sql-reference/functions/arithmetic-functions.md deleted file mode 100644 index eaab128cc36..00000000000 --- a/docs/tr/sql-reference/functions/arithmetic-functions.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 35 -toc_title: Aritmetik ---- - -# Aritmetik Fonksiyonlar {#arithmetic-functions} - -Tüm aritmetik işlevler için, sonuç türü, böyle bir tür varsa, sonucun sığdığı en küçük sayı türü olarak hesaplanır. Minimum, bit sayısına, imzalanıp imzalanmadığına ve yüzüp yüzmediğine bağlı olarak aynı anda alınır. Yeterli bit yoksa, en yüksek bit türü alınır. - -Örnek: - -``` sql -SELECT toTypeName(0), toTypeName(0 + 0), toTypeName(0 + 0 + 0), toTypeName(0 + 0 + 0 + 0) -``` - -``` text -┌─toTypeName(0)─┬─toTypeName(plus(0, 0))─┬─toTypeName(plus(plus(0, 0), 0))─┬─toTypeName(plus(plus(plus(0, 0), 0), 0))─┐ -│ UInt8 │ UInt16 │ UInt32 │ UInt64 │ -└───────────────┴────────────────────────┴─────────────────────────────────┴──────────────────────────────────────────┘ -``` - -Aritmetik fonksiyonlar, uint8, Uİnt16, Uİnt32, Uint64, Int8, Int16, Int32, Int64, Float32 veya Float64 türlerinden herhangi bir çift için çalışır. - -Taşma, C++ile aynı şekilde üretilir. - -## artı (a, b), A + B operatörü {#plusa-b-a-b-operator} - -Sayıların toplamını hesaplar. -Ayrıca bir tarih veya tarih ve Saat ile tamsayı numaraları ekleyebilirsiniz. Bir tarih durumunda, bir tamsayı eklemek, karşılık gelen gün sayısını eklemek anlamına gelir. Zamanla bir tarih için, karşılık gelen saniye sayısını eklemek anlamına gelir. - -## eksi (a, b), A - B operatörü {#minusa-b-a-b-operator} - -Farkı hesaplar. Sonuç her zaman imzalanır. - -You can also calculate integer numbers from a date or date with time. The idea is the same – see above for ‘plus’. - -## çarp operatorma (a, b), A \* B operatörü {#multiplya-b-a-b-operator} - -Sayıların ürününü hesaplar. - -## böl (a, b), A / B operatörü {#dividea-b-a-b-operator} - -Sayıların bölümünü hesaplar. Sonuç türü her zaman bir kayan nokta türüdür. -Tam sayı bölümü değildir. Tamsayı bölümü için, ‘intDiv’ İşlev. -Sıfıra bölerek zaman olsun ‘inf’, ‘-inf’, veya ‘nan’. - -## ıntdiv(a, b) {#intdiva-b} - -Sayıların bölümünü hesaplar. Tamsayılara bölünür, yuvarlanır (mutlak değere göre). -Sıfıra bölünürken veya en az negatif sayıyı eksi bir ile bölürken bir istisna atılır. - -## ıntdivorzero(a, b) {#intdivorzeroa-b} - -Farklıdır ‘intDiv’ bu, sıfıra bölünürken veya en az bir negatif sayıyı eksi bir ile bölerek sıfır döndürür. - -## modulo (a, b), A % B operatörü {#moduloa-b-a-b-operator} - -Bölünmeden sonra kalan hesaplar. -Bağımsız değişkenler kayan nokta sayılarıysa, ondalık bölümü bırakarak tamsayılara önceden dönüştürülürler. -Kalan C++ile aynı anlamda alınır. Kesik bölme negatif sayılar için kullanılır. -Sıfıra bölünürken veya en az negatif sayıyı eksi bir ile bölürken bir istisna atılır. - -## moduloOrZero(a, b) {#moduloorzeroa-b} - -Farklıdır ‘modulo’ bölen sıfır olduğunda sıfır döndürür. - -## negate (a), - bir operatör {#negatea-a-operator} - -Ters işareti ile bir sayı hesaplar. Sonuç her zaman imzalanır. - -## abs (a) {#arithm_func-abs} - -\(A\) sayısının mutlak değerini hesaplar. Yani, \< 0 ise,- A döndürür. imzasız türler için hiçbir şey yapmaz. İmzalı tamsayı türleri için imzasız bir sayı döndürür. - -## gcd (a, b) {#gcda-b} - -Sayıların en büyük ortak böleni döndürür. -Sıfıra bölünürken veya en az negatif sayıyı eksi bir ile bölürken bir istisna atılır. - -## lcm(a, b) {#lcma-b} - -Sayıların en az ortak katını döndürür. -Sıfıra bölünürken veya en az negatif sayıyı eksi bir ile bölürken bir istisna atılır. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/arithmetic_functions/) diff --git a/docs/tr/sql-reference/functions/array-functions.md b/docs/tr/sql-reference/functions/array-functions.md deleted file mode 100644 index e1887af03bc..00000000000 --- a/docs/tr/sql-reference/functions/array-functions.md +++ /dev/null @@ -1,1061 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 46 -toc_title: "Dizilerle \xE7al\u0131\u015Fma" ---- - -# Dizilerle çalışmak için işlevler {#functions-for-working-with-arrays} - -## boş {#function-empty} - -Boş bir dizi için 1 veya boş olmayan bir dizi için 0 döndürür. -Sonuç türü Uint8'dir. -İşlev ayrıca dizeler için de çalışır. - -## notEmpty {#function-notempty} - -Boş bir dizi için 0 veya boş olmayan bir dizi için 1 döndürür. -Sonuç türü Uint8'dir. -İşlev ayrıca dizeler için de çalışır. - -## uzunluk {#array_functions-length} - -Dizideki öğe sayısını döndürür. -Sonuç türü Uint64'tür. -İşlev ayrıca dizeler için de çalışır. - -## emptyArrayUİnt8, emptyArrayUİnt16, emptyArrayUİnt32, emptyArrayUİnt64 {#emptyarrayuint8-emptyarrayuint16-emptyarrayuint32-emptyarrayuint64} - -## emptyArrayİnt8, emptyArrayİnt16, emptyArrayİnt32, emptyArrayİnt64 {#emptyarrayint8-emptyarrayint16-emptyarrayint32-emptyarrayint64} - -## emptyArrayFloat32, emptyArrayFloat64 {#emptyarrayfloat32-emptyarrayfloat64} - -## emptyArrayDate, emptyArrayDateTime {#emptyarraydate-emptyarraydatetime} - -## emptyArrayString {#emptyarraystring} - -Sıfır bağımsız değişkeni kabul eder ve uygun türde boş bir dizi döndürür. - -## emptyArrayToSingle {#emptyarraytosingle} - -Boş bir dizi kabul eder ve varsayılan değere eşit bir tek öğe dizisi döndürür. - -## Aralık (bitiş), Aralık(başlangıç, bitiş \[, adım\]) {#rangeend-rangestart-end-step} - -1 Adım-başından sonuna kadar sayıların bir dizi döndürür. -Eğer argüman `start` belirtilmemiş, varsayılan olarak 0. -Eğer argüman `step` belirtilmemiş, varsayılan olarak 1. -Neredeyse pythonic gibi davranışlar `range`. Ancak fark, tüm argümanların tipinin olması gerektiğidir `UInt` şiir. -Bir veri bloğunda toplam uzunluğu 100.000.000'den fazla öğe olan diziler oluşturulursa, bir istisna atılır. - -## array(x1, …), operator \[x1, …\] {#arrayx1-operator-x1} - -İşlev bağımsız değişkenlerinden bir dizi oluşturur. -Bağımsız değişkenler sabit olmalı ve en küçük ortak türe sahip türlere sahip olmalıdır. En az bir argüman geçirilmelidir, çünkü aksi takdirde hangi tür dizinin oluşturulacağı açık değildir. Yani, boş bir dizi oluşturmak için bu işlevi kullanamazsınız (bunu yapmak için, ‘emptyArray\*’ fonksiyon yukarıda açıklandığı. -Ret anur anns an ‘Array(T)’ sonucu yazın, nerede ‘T’ geçirilen bağımsız değişkenlerin en küçük ortak türüdür. - -## arrayConcat {#arrayconcat} - -Argüman olarak geçirilen dizileri birleştirir. - -``` sql -arrayConcat(arrays) -``` - -**Parametre** - -- `arrays` – Arbitrary number of arguments of [Dizi](../../sql-reference/data-types/array.md) tür. - **Örnek** - - - -``` sql -SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res -``` - -``` text -┌─res───────────┐ -│ [1,2,3,4,5,6] │ -└───────────────┘ -``` - -## arrayElement (arr, n), operatör arr\[n\] {#arrayelementarr-n-operator-arrn} - -İnd theex ile eleman alın `n` diz theiden `arr`. `n` herhangi bir tamsayı türü olmalıdır. -Bir dizideki dizinler birinden başlar. -Negatif dizinler desteklenir. Bu durumda, sondan numaralandırılmış ilgili elemanı seçer. Mesela, `arr[-1]` dizideki son öğedir. - -Dizin bir dizinin sınırlarının dışına düşerse, bazı varsayılan değer döndürür (sayılar için 0, dizeler için boş bir dize vb.).), sabit olmayan bir dizi ve sabit bir dizin 0 olan durum hariç (bu durumda bir hata olacaktır `Array indices are 1-based`). - -## has (arr, elem) {#hasarr-elem} - -Olup olmadığını denetler ‘arr’ dizi var ‘elem’ öğe. -Öğe dizide değilse 0 veya varsa 1 değerini döndürür. - -`NULL` değer olarak iş islenir. - -``` sql -SELECT has([1, 2, NULL], NULL) -``` - -``` text -┌─has([1, 2, NULL], NULL)─┐ -│ 1 │ -└─────────────────────────┘ -``` - -## hasAll {#hasall} - -Bir dizi başka bir alt kümesi olup olmadığını denetler. - -``` sql -hasAll(set, subset) -``` - -**Parametre** - -- `set` – Array of any type with a set of elements. -- `subset` – Array of any type with elements that should be tested to be a subset of `set`. - -**Dönüş değerleri** - -- `1`, eğer `set` tüm öğeleri içerir `subset`. -- `0`, başka. - -**Tuhaf özellikler** - -- Boş bir dizi, herhangi bir dizinin bir alt kümesidir. -- `Null` bir değer olarak işlenir. -- Her iki dizideki değerlerin sırası önemli değil. - -**Örnekler** - -`SELECT hasAll([], [])` döner 1. - -`SELECT hasAll([1, Null], [Null])` döner 1. - -`SELECT hasAll([1.0, 2, 3, 4], [1, 3])` döner 1. - -`SELECT hasAll(['a', 'b'], ['a'])` döner 1. - -`SELECT hasAll([1], ['a'])` 0 döndürür. - -`SELECT hasAll([[1, 2], [3, 4]], [[1, 2], [3, 5]])` 0 döndürür. - -## hasAny {#hasany} - -İki dizinin bazı öğelerle kesiştiği olup olmadığını kontrol eder. - -``` sql -hasAny(array1, array2) -``` - -**Parametre** - -- `array1` – Array of any type with a set of elements. -- `array2` – Array of any type with a set of elements. - -**Dönüş değerleri** - -- `1`, eğer `array1` ve `array2` en azından benzer bir öğeye sahip olun. -- `0`, başka. - -**Tuhaf özellikler** - -- `Null` bir değer olarak işlenir. -- Her iki dizideki değerlerin sırası önemli değil. - -**Örnekler** - -`SELECT hasAny([1], [])` dönüşler `0`. - -`SELECT hasAny([Null], [Null, 1])` dönüşler `1`. - -`SELECT hasAny([-128, 1., 512], [1])` dönüşler `1`. - -`SELECT hasAny([[1, 2], [3, 4]], ['a', 'c'])` dönüşler `0`. - -`SELECT hasAll([[1, 2], [3, 4]], [[1, 2], [1, 2]])` dönüşler `1`. - -## ındexof(arr, x) {#indexofarr-x} - -İlk dizini döndürür ‘x’ dizide ise öğe (1'den başlayarak) veya değilse 0. - -Örnek: - -``` sql -SELECT indexOf([1, 3, NULL, NULL], NULL) -``` - -``` text -┌─indexOf([1, 3, NULL, NULL], NULL)─┐ -│ 3 │ -└───────────────────────────────────┘ -``` - -Elem setents set to `NULL` normal değerler olarak ele alınır. - -## countEqual(arr, x) {#countequalarr-x} - -X eşit dizideki öğelerin sayısını döndürür. Arraycount eşdeğer (elem - \> elem = x, arr). - -`NULL` öğeler ayrı değerler olarak işlenir. - -Örnek: - -``` sql -SELECT countEqual([1, 2, NULL, NULL], NULL) -``` - -``` text -┌─countEqual([1, 2, NULL, NULL], NULL)─┐ -│ 2 │ -└──────────────────────────────────────┘ -``` - -## arrayEnumerate(arr) {#array_functions-arrayenumerate} - -Returns the array \[1, 2, 3, …, length (arr) \] - -Bu işlev normalde ARRAY JOIN ile kullanılır. ARRAY JOİN uyguladıktan sonra her dizi için sadece bir kez bir şey saymaya izin verir. Örnek: - -``` sql -SELECT - count() AS Reaches, - countIf(num = 1) AS Hits -FROM test.hits -ARRAY JOIN - GoalsReached, - arrayEnumerate(GoalsReached) AS num -WHERE CounterID = 160656 -LIMIT 10 -``` - -``` text -┌─Reaches─┬──Hits─┐ -│ 95606 │ 31406 │ -└─────────┴───────┘ -``` - -Bu örnekte, Reaches dönüşümlerin sayısıdır (ARRAY JOİN uygulandıktan sonra alınan dizeler) ve İsabetler sayfa görüntüleme sayısıdır (ARRAY JOİN önce dizeler). Bu özel durumda, aynı sonucu daha kolay bir şekilde alabilirsiniz: - -``` sql -SELECT - sum(length(GoalsReached)) AS Reaches, - count() AS Hits -FROM test.hits -WHERE (CounterID = 160656) AND notEmpty(GoalsReached) -``` - -``` text -┌─Reaches─┬──Hits─┐ -│ 95606 │ 31406 │ -└─────────┴───────┘ -``` - -Bu fonksiyon aynı zamanda yüksek mertebeden fonksiyonlarda da kullanılabilir. Örneğin, bir koşulla eşleşen öğeler için dizi dizinleri almak için kullanabilirsiniz. - -## arrayEnumerateUniq(arr, …) {#arrayenumerateuniqarr} - -Kaynak diziyle aynı boyutta bir dizi döndürür ve her öğe için aynı değere sahip öğeler arasında konumunun ne olduğunu gösterir. -Örneğin: arrayEnumerateUniq(\[10, 20, 10, 30\]) = \[1, 1, 2, 1\]. - -Bu işlev, dizi birleştirme ve dizi öğelerinin toplanmasını kullanırken kullanışlıdır. -Örnek: - -``` sql -SELECT - Goals.ID AS GoalID, - sum(Sign) AS Reaches, - sumIf(Sign, num = 1) AS Visits -FROM test.visits -ARRAY JOIN - Goals, - arrayEnumerateUniq(Goals.ID) AS num -WHERE CounterID = 160656 -GROUP BY GoalID -ORDER BY Reaches DESC -LIMIT 10 -``` - -``` text -┌──GoalID─┬─Reaches─┬─Visits─┐ -│ 53225 │ 3214 │ 1097 │ -│ 2825062 │ 3188 │ 1097 │ -│ 56600 │ 2803 │ 488 │ -│ 1989037 │ 2401 │ 365 │ -│ 2830064 │ 2396 │ 910 │ -│ 1113562 │ 2372 │ 373 │ -│ 3270895 │ 2262 │ 812 │ -│ 1084657 │ 2262 │ 345 │ -│ 56599 │ 2260 │ 799 │ -│ 3271094 │ 2256 │ 812 │ -└─────────┴─────────┴────────┘ -``` - -Bu örnekte, her hedef kimliğinin dönüşüm sayısı (hedefler iç içe geçmiş veri yapısındaki her öğe, bir dönüşüm olarak adlandırdığımız ulaşılan bir hedeftir) ve oturum sayısı Hesaplaması vardır. ARRAY JOİN olmadan, oturum sayısını sum(Sign) olarak sayardık. Ancak bu özel durumda, satırlar iç içe geçmiş hedefler yapısıyla çarpıldı, bu nedenle her oturumu bir kez saymak için arrayenumerateuniq değerine bir koşul uyguluyoruz(Goals.ID) fonksiyonu. - -Arrayenumerateuniq işlevi, bağımsız değişkenlerle aynı boyutta birden çok dizi alabilir. Bu durumda, tüm dizilerde aynı konumlardaki elemanların tuplesleri için benzersizlik düşünülür. - -``` sql -SELECT arrayEnumerateUniq([1, 1, 1, 2, 2, 2], [1, 1, 2, 1, 1, 2]) AS res -``` - -``` text -┌─res───────────┐ -│ [1,2,1,1,2,1] │ -└───────────────┘ -``` - -Bu, iç içe geçmiş bir veri yapısı ve bu yapıdaki birden çok öğe arasında daha fazla toplama ile dizi birleşimini kullanırken gereklidir. - -## arrayPopBack {#arraypopback} - -Son öğeyi diziden kaldırır. - -``` sql -arrayPopBack(array) -``` - -**Parametre** - -- `array` – Array. - -**Örnek** - -``` sql -SELECT arrayPopBack([1, 2, 3]) AS res -``` - -``` text -┌─res───┐ -│ [1,2] │ -└───────┘ -``` - -## arrayPopFront {#arraypopfront} - -İlk öğeyi diziden kaldırır. - -``` sql -arrayPopFront(array) -``` - -**Parametre** - -- `array` – Array. - -**Örnek** - -``` sql -SELECT arrayPopFront([1, 2, 3]) AS res -``` - -``` text -┌─res───┐ -│ [2,3] │ -└───────┘ -``` - -## arrayPushBack {#arraypushback} - -Dizinin sonuna bir öğe ekler. - -``` sql -arrayPushBack(array, single_value) -``` - -**Parametre** - -- `array` – Array. -- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` dizinin veri türü için yazın. Clickhouse'daki veri türleri hakkında daha fazla bilgi için bkz. “[Veri türleri](../../sql-reference/data-types/index.md#data_types)”. Olabilir `NULL`. Fonksiyon bir ekler `NULL` bir dizi için öğe ve dizi öğeleri türü dönüştürür `Nullable`. - -**Örnek** - -``` sql -SELECT arrayPushBack(['a'], 'b') AS res -``` - -``` text -┌─res───────┐ -│ ['a','b'] │ -└───────────┘ -``` - -## arrayPushFront {#arraypushfront} - -Dizinin başına bir öğe ekler. - -``` sql -arrayPushFront(array, single_value) -``` - -**Parametre** - -- `array` – Array. -- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` dizinin veri türü için yazın. Clickhouse'daki veri türleri hakkında daha fazla bilgi için bkz. “[Veri türleri](../../sql-reference/data-types/index.md#data_types)”. Olabilir `NULL`. Fonksiyon bir ekler `NULL` bir dizi için öğe ve dizi öğeleri türü dönüştürür `Nullable`. - -**Örnek** - -``` sql -SELECT arrayPushFront(['b'], 'a') AS res -``` - -``` text -┌─res───────┐ -│ ['a','b'] │ -└───────────┘ -``` - -## arrayResize {#arrayresize} - -Dizinin uzunluğunu değiştirir. - -``` sql -arrayResize(array, size[, extender]) -``` - -**Parametre:** - -- `array` — Array. -- `size` — Required length of the array. - - Eğer `size` dizinin orijinal boyutundan daha az, dizi sağdan kesilir. -- Eğer `size` dizinin başlangıç boyutundan daha büyük, dizi sağa uzatılır `extender` dizi öğelerinin veri türü için değerler veya varsayılan değerler. -- `extender` — Value for extending an array. Can be `NULL`. - -**Döndürülen değer:** - -Bir dizi uzunluk `size`. - -**Arama örnekleri** - -``` sql -SELECT arrayResize([1], 3) -``` - -``` text -┌─arrayResize([1], 3)─┐ -│ [1,0,0] │ -└─────────────────────┘ -``` - -``` sql -SELECT arrayResize([1], 3, NULL) -``` - -``` text -┌─arrayResize([1], 3, NULL)─┐ -│ [1,NULL,NULL] │ -└───────────────────────────┘ -``` - -## arraySlice {#arrayslice} - -Dizinin bir dilimini döndürür. - -``` sql -arraySlice(array, offset[, length]) -``` - -**Parametre** - -- `array` – Array of data. -- `offset` – Indent from the edge of the array. A positive value indicates an offset on the left, and a negative value is an indent on the right. Numbering of the array items begins with 1. -- `length` - Gerekli dilimin uzunluğu. Negatif bir değer belirtirseniz, işlev açık bir dilim döndürür `[offset, array_length - length)`. Değeri atlarsanız, işlev dilimi döndürür `[offset, the_end_of_array]`. - -**Örnek** - -``` sql -SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res -``` - -``` text -┌─res────────┐ -│ [2,NULL,4] │ -└────────────┘ -``` - -Ar arrayray elem toents set to `NULL` normal değerler olarak ele alınır. - -## arraySort(\[func,\] arr, …) {#array_functions-sort} - -Elemanları sıralar `arr` artan düzende dizi. Eğer... `func` fonksiyonu belirtilir, sıralama düzeni sonucu belirlenir `func` fonksiyon dizinin elemanlarına uygulanır. Eğer `func` birden fazla argüman kabul eder, `arraySort` fonksiyon argümanları birkaç diziler geçirilir `func` karşılık gelir. Ayrıntılı örnekler sonunda gösterilir `arraySort` açıklama. - -Tamsayı değerleri sıralama örneği: - -``` sql -SELECT arraySort([1, 3, 3, 0]); -``` - -``` text -┌─arraySort([1, 3, 3, 0])─┐ -│ [0,1,3,3] │ -└─────────────────────────┘ -``` - -Dize değerleri sıralama örneği: - -``` sql -SELECT arraySort(['hello', 'world', '!']); -``` - -``` text -┌─arraySort(['hello', 'world', '!'])─┐ -│ ['!','hello','world'] │ -└────────────────────────────────────┘ -``` - -Aşağıdaki sıralama sırasını göz önünde bulundurun `NULL`, `NaN` ve `Inf` değerler: - -``` sql -SELECT arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]); -``` - -``` text -┌─arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf])─┐ -│ [-inf,-4,1,2,3,inf,nan,nan,NULL,NULL] │ -└───────────────────────────────────────────────────────────┘ -``` - -- `-Inf` değerler dizide ilk sırada yer alır. -- `NULL` değerler dizideki son değerlerdir. -- `NaN` değerler hemen önce `NULL`. -- `Inf` değerler hemen önce `NaN`. - -Not thate that `arraySort` is a [yüksek sipariş fonksiyonu](higher-order-functions.md). Bir lambda işlevini ilk argüman olarak iletebilirsiniz. Bu durumda, sıralama sırası, dizinin elemanlarına uygulanan lambda işlevinin sonucu ile belirlenir. - -Aşağıdaki örneği ele alalım: - -``` sql -SELECT arraySort((x) -> -x, [1, 2, 3]) as res; -``` - -``` text -┌─res─────┐ -│ [3,2,1] │ -└─────────┘ -``` - -For each element of the source array, the lambda function returns the sorting key, that is, \[1 –\> -1, 2 –\> -2, 3 –\> -3\]. Since the `arraySort` fonksiyon tuşları artan sırayla sıralar, sonuç \[3, 2, 1\]. Böylece, `(x) –> -x` lambda fonksiyonu setleri [azalan düzen](#array_functions-reverse-sort) bir sıralama içinde. - -Lambda işlevi birden çok bağımsız değişken kabul edebilir. Bu durumda, geçmek gerekir `arraySort` işlev lambda işlevinin argümanlarının karşılık geleceği aynı uzunlukta birkaç dizi. Elde edilen dizi ilk giriş dizisinden elemanlardan oluşacaktır; bir sonraki giriş dizilerinden elemanlar sıralama anahtarlarını belirtir. Mesela: - -``` sql -SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res; -``` - -``` text -┌─res────────────────┐ -│ ['world', 'hello'] │ -└────────────────────┘ -``` - -Burada, ikinci dizide (\[2, 1\]) geçirilen öğeler, kaynak diziden karşılık gelen öğe için bir sıralama anahtarı tanımlar (\[‘hello’, ‘world’\]), bu, \[‘hello’ –\> 2, ‘world’ –\> 1\]. Since the lambda function doesn't use `x`, kaynak dizinin gerçek değerleri sonuçtaki sırayı etkilemez. Böyle, ‘hello’ sonuçtaki ikinci eleman olacak ve ‘world’ ilk olacak. - -Diğer örnekler aşağıda gösterilmiştir. - -``` sql -SELECT arraySort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res; -``` - -``` text -┌─res─────┐ -│ [2,1,0] │ -└─────────┘ -``` - -``` sql -SELECT arraySort((x, y) -> -y, [0, 1, 2], [1, 2, 3]) as res; -``` - -``` text -┌─res─────┐ -│ [2,1,0] │ -└─────────┘ -``` - -!!! note "Not" - Sıralama verimliliğini artırmak için, [Schwartzian dönüşümü](https://en.wikipedia.org/wiki/Schwartzian_transform) kullanılır. - -## arrayReverseSort(\[func,\] arr, …) {#array_functions-reverse-sort} - -Elemanları sıralar `arr` azalan sırayla dizi. Eğer... `func` fonksiyon belirtilir, `arr` sonucuna göre sıra islanır. `func` işlev dizinin öğelerine uygulanır ve sonra sıralanmış dizi tersine çevrilir. Eğer `func` birden fazla argüman kabul eder, `arrayReverseSort` fonksiyon argümanları birkaç diziler geçirilir `func` karşılık gelir. Ayrıntılı örnekler sonunda gösterilir `arrayReverseSort` açıklama. - -Tamsayı değerleri sıralama örneği: - -``` sql -SELECT arrayReverseSort([1, 3, 3, 0]); -``` - -``` text -┌─arrayReverseSort([1, 3, 3, 0])─┐ -│ [3,3,1,0] │ -└────────────────────────────────┘ -``` - -Dize değerleri sıralama örneği: - -``` sql -SELECT arrayReverseSort(['hello', 'world', '!']); -``` - -``` text -┌─arrayReverseSort(['hello', 'world', '!'])─┐ -│ ['world','hello','!'] │ -└───────────────────────────────────────────┘ -``` - -Aşağıdaki sıralama sırasını göz önünde bulundurun `NULL`, `NaN` ve `Inf` değerler: - -``` sql -SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]) as res; -``` - -``` text -┌─res───────────────────────────────────┐ -│ [inf,3,2,1,-4,-inf,nan,nan,NULL,NULL] │ -└───────────────────────────────────────┘ -``` - -- `Inf` değerler dizide ilk sırada yer alır. -- `NULL` değerler dizideki son değerlerdir. -- `NaN` değerler hemen önce `NULL`. -- `-Inf` değerler hemen önce `NaN`. - -Not `arrayReverseSort` is a [yüksek sipariş fonksiyonu](higher-order-functions.md). Bir lambda işlevini ilk argüman olarak iletebilirsiniz. Örnek aşağıda gösterilmiştir. - -``` sql -SELECT arrayReverseSort((x) -> -x, [1, 2, 3]) as res; -``` - -``` text -┌─res─────┐ -│ [1,2,3] │ -└─────────┘ -``` - -Dizi aşağıdaki şekilde sıralanır: - -1. İlk başta, kaynak dizi (\[1, 2, 3\]), dizinin elemanlarına uygulanan lambda işlevinin sonucuna göre sıralanır. Sonuç bir dizidir \[3, 2, 1\]. -2. Önceki adımda elde edilen dizi tersine çevrilir. Yani, nihai sonuç \[1, 2, 3\]. - -Lambda işlevi birden çok bağımsız değişken kabul edebilir. Bu durumda, geçmek gerekir `arrayReverseSort` işlev lambda işlevinin argümanlarının karşılık geleceği aynı uzunlukta birkaç dizi. Elde edilen dizi ilk giriş dizisinden elemanlardan oluşacaktır; bir sonraki giriş dizilerinden elemanlar sıralama anahtarlarını belirtir. Mesela: - -``` sql -SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res; -``` - -``` text -┌─res───────────────┐ -│ ['hello','world'] │ -└───────────────────┘ -``` - -Bu örnekte, dizi aşağıdaki şekilde sıralanır: - -1. İlk başta, kaynak dizi (\[‘hello’, ‘world’\]) dizilerin elemanlarına uygulanan lambda işlevinin sonucuna göre sıralanır. İkinci dizide geçirilen öğeler (\[2, 1\]), kaynak diziden karşılık gelen öğeler için sıralama anahtarlarını tanımlar. Sonuç bir dizidir \[‘world’, ‘hello’\]. -2. Önceki adımda sıralanmış dizi tersine çevrilir. Yani, nihai sonuç \[‘hello’, ‘world’\]. - -Diğer örnekler aşağıda gösterilmiştir. - -``` sql -SELECT arrayReverseSort((x, y) -> y, [4, 3, 5], ['a', 'b', 'c']) AS res; -``` - -``` text -┌─res─────┐ -│ [5,3,4] │ -└─────────┘ -``` - -``` sql -SELECT arrayReverseSort((x, y) -> -y, [4, 3, 5], [1, 2, 3]) AS res; -``` - -``` text -┌─res─────┐ -│ [4,3,5] │ -└─────────┘ -``` - -## arrayUniq(arr, …) {#arrayuniqarr} - -Bir bağımsız değişken geçirilirse, dizideki farklı öğelerin sayısını sayar. -Birden çok bağımsız değişken geçirilirse, birden çok dizideki karşılık gelen konumlardaki farklı öğe kümelerinin sayısını sayar. - -Bir dizideki benzersiz öğelerin bir listesini almak istiyorsanız, arrayreduce kullanabilirsiniz(‘groupUniqArray’, arr). - -## arrayJoin(arr) {#array-functions-join} - -Özel bir işlev. Bölümüne bakınız [“ArrayJoin function”](array-join.md#functions_arrayjoin). - -## arrayDifference {#arraydifference} - -Bitişik dizi öğeleri arasındaki farkı hesaplar. İlk öğenin 0 olacağı bir dizi döndürür, ikincisi arasındaki farktır `a[1] - a[0]`, etc. The type of elements in the resulting array is determined by the type inference rules for subtraction (e.g. `UInt8` - `UInt8` = `Int16`). - -**Sözdizimi** - -``` sql -arrayDifference(array) -``` - -**Parametre** - -- `array` – [Dizi](https://clickhouse.tech/docs/en/data_types/array/). - -**Döndürülen değerler** - -Bitişik öğeler arasındaki farklar dizisini döndürür. - -Tür: [Uİnt\*](https://clickhouse.tech/docs/en/data_types/int_uint/#uint-ranges), [Tamsayı\*](https://clickhouse.tech/docs/en/data_types/int_uint/#int-ranges), [Yüzdürmek\*](https://clickhouse.tech/docs/en/data_types/float/). - -**Örnek** - -Sorgu: - -``` sql -SELECT arrayDifference([1, 2, 3, 4]) -``` - -Sonuç: - -``` text -┌─arrayDifference([1, 2, 3, 4])─┐ -│ [0,1,1,1] │ -└───────────────────────────────┘ -``` - -Sonuç türü Int64 nedeniyle taşma örneği: - -Sorgu: - -``` sql -SELECT arrayDifference([0, 10000000000000000000]) -``` - -Sonuç: - -``` text -┌─arrayDifference([0, 10000000000000000000])─┐ -│ [0,-8446744073709551616] │ -└────────────────────────────────────────────┘ -``` - -## arrayDistinct {#arraydistinct} - -Bir dizi alır, yalnızca farklı öğeleri içeren bir dizi döndürür. - -**Sözdizimi** - -``` sql -arrayDistinct(array) -``` - -**Parametre** - -- `array` – [Dizi](https://clickhouse.tech/docs/en/data_types/array/). - -**Döndürülen değerler** - -Farklı öğeleri içeren bir dizi döndürür. - -**Örnek** - -Sorgu: - -``` sql -SELECT arrayDistinct([1, 2, 2, 3, 1]) -``` - -Sonuç: - -``` text -┌─arrayDistinct([1, 2, 2, 3, 1])─┐ -│ [1,2,3] │ -└────────────────────────────────┘ -``` - -## arrayEnumerateDense(arr) {#array_functions-arrayenumeratedense} - -Kaynak diziyle aynı boyutta bir dizi döndürür ve her öğenin kaynak dizide ilk olarak nerede göründüğünü gösterir. - -Örnek: - -``` sql -SELECT arrayEnumerateDense([10, 20, 10, 30]) -``` - -``` text -┌─arrayEnumerateDense([10, 20, 10, 30])─┐ -│ [1,2,1,3] │ -└───────────────────────────────────────┘ -``` - -## arrayıntersect(arr) {#array-functions-arrayintersect} - -Birden çok dizi alır, tüm kaynak dizilerde bulunan öğeleri içeren bir dizi döndürür. Elde edilen dizideki öğeler sırası ilk dizideki ile aynıdır. - -Örnek: - -``` sql -SELECT - arrayIntersect([1, 2], [1, 3], [2, 3]) AS no_intersect, - arrayIntersect([1, 2], [1, 3], [1, 4]) AS intersect -``` - -``` text -┌─no_intersect─┬─intersect─┐ -│ [] │ [1] │ -└──────────────┴───────────┘ -``` - -## arrayReduce {#arrayreduce} - -Dizi öğelerine bir toplama işlevi uygular ve sonucunu döndürür. Toplama işlevinin adı, tek tırnak içinde bir dize olarak geçirilir `'max'`, `'sum'`. Parametrik toplama işlevleri kullanıldığında, parametre parantez içinde işlev adından sonra gösterilir `'uniqUpTo(6)'`. - -**Sözdizimi** - -``` sql -arrayReduce(agg_func, arr1, arr2, ..., arrN) -``` - -**Parametre** - -- `agg_func` — The name of an aggregate function which should be a constant [dize](../../sql-reference/data-types/string.md). -- `arr` — Any number of [dizi](../../sql-reference/data-types/array.md) sütunları toplama işlevinin parametreleri olarak yazın. - -**Döndürülen değer** - -**Örnek** - -``` sql -SELECT arrayReduce('max', [1, 2, 3]) -``` - -``` text -┌─arrayReduce('max', [1, 2, 3])─┐ -│ 3 │ -└───────────────────────────────┘ -``` - -Bir toplama işlevi birden çok bağımsız değişken alırsa, bu işlev aynı boyuttaki birden çok diziye uygulanmalıdır. - -``` sql -SELECT arrayReduce('maxIf', [3, 5], [1, 0]) -``` - -``` text -┌─arrayReduce('maxIf', [3, 5], [1, 0])─┐ -│ 3 │ -└──────────────────────────────────────┘ -``` - -Parametrik toplama fonksiyonu ile örnek: - -``` sql -SELECT arrayReduce('uniqUpTo(3)', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) -``` - -``` text -┌─arrayReduce('uniqUpTo(3)', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])─┐ -│ 4 │ -└─────────────────────────────────────────────────────────────┘ -``` - -## arrayReduceİnRanges {#arrayreduceinranges} - -Belirli aralıklardaki dizi öğelerine bir toplama işlevi uygular ve her aralığa karşılık gelen sonucu içeren bir dizi döndürür. Fonksiyon aynı sonucu birden fazla olarak döndürür `arrayReduce(agg_func, arraySlice(arr1, index, length), ...)`. - -**Sözdizimi** - -``` sql -arrayReduceInRanges(agg_func, ranges, arr1, arr2, ..., arrN) -``` - -**Parametre** - -- `agg_func` — The name of an aggregate function which should be a constant [dize](../../sql-reference/data-types/string.md). -- `ranges` — The ranges to aggretate which should be an [dizi](../../sql-reference/data-types/array.md) -den [Demetler](../../sql-reference/data-types/tuple.md) indeks ve her aralığın uzunluğunu içeren. -- `arr` — Any number of [dizi](../../sql-reference/data-types/array.md) sütunları toplama işlevinin parametreleri olarak yazın. - -**Döndürülen değer** - -**Örnek** - -``` sql -SELECT arrayReduceInRanges( - 'sum', - [(1, 5), (2, 3), (3, 4), (4, 4)], - [1000000, 200000, 30000, 4000, 500, 60, 7] -) AS res -``` - -``` text -┌─res─────────────────────────┐ -│ [1234500,234000,34560,4567] │ -└─────────────────────────────┘ -``` - -## arrayReverse(arr) {#arrayreverse} - -Öğeleri ters sırada içeren orijinal diziyle aynı boyutta bir dizi döndürür. - -Örnek: - -``` sql -SELECT arrayReverse([1, 2, 3]) -``` - -``` text -┌─arrayReverse([1, 2, 3])─┐ -│ [3,2,1] │ -└─────────────────────────┘ -``` - -## ters (arr) {#array-functions-reverse} - -Eşanlamlı [“arrayReverse”](#arrayreverse) - -## arrayFlatten {#arrayflatten} - -Bir dizi diziyi düz bir diziye dönüştürür. - -İşlev: - -- İç içe geçmiş dizilerin herhangi bir derinliği için geçerlidir. -- Zaten düz olan dizileri değiştirmez. - -Düzleştirilmiş dizi, tüm kaynak dizilerdeki tüm öğeleri içerir. - -**Sözdizimi** - -``` sql -flatten(array_of_arrays) -``` - -Takma ad: `flatten`. - -**Parametre** - -- `array_of_arrays` — [Dizi](../../sql-reference/data-types/array.md) dizilerin. Mesela, `[[1,2,3], [4,5]]`. - -**Örnekler** - -``` sql -SELECT flatten([[[1]], [[2], [3]]]) -``` - -``` text -┌─flatten(array(array([1]), array([2], [3])))─┐ -│ [1,2,3] │ -└─────────────────────────────────────────────┘ -``` - -## arrayCompact {#arraycompact} - -Ardışık yinelenen öğeleri bir diziden kaldırır. Sonuç değerlerinin sırası, kaynak dizideki sıraya göre belirlenir. - -**Sözdizimi** - -``` sql -arrayCompact(arr) -``` - -**Parametre** - -`arr` — The [dizi](../../sql-reference/data-types/array.md) incelemek. - -**Döndürülen değer** - -Yinelenen olmadan dizi. - -Tür: `Array`. - -**Örnek** - -Sorgu: - -``` sql -SELECT arrayCompact([1, 1, nan, nan, 2, 3, 3, 3]) -``` - -Sonuç: - -``` text -┌─arrayCompact([1, 1, nan, nan, 2, 3, 3, 3])─┐ -│ [1,nan,nan,2,3] │ -└────────────────────────────────────────────┘ -``` - -## arrayZip {#arrayzip} - -Birden çok diziyi tek bir dizide birleştirir. Elde edilen dizi, listelenen bağımsız değişken sırasına göre gruplandırılmış kaynak dizilerin karşılık gelen öğelerini içerir. - -**Sözdizimi** - -``` sql -arrayZip(arr1, arr2, ..., arrN) -``` - -**Parametre** - -- `arrN` — [Dizi](../data-types/array.md). - -İşlev, farklı türde herhangi bir dizi alabilir. Tüm giriş dizileri eşit boyutta olmalıdır. - -**Döndürülen değer** - -- Gruplandırılmış kaynak dizilerden öğelerle dizi [Demetler](../data-types/tuple.md). Veri türleri tuple giriş dizileri türleri ile aynıdır ve diziler geçirilir aynı sırada. - -Tür: [Dizi](../data-types/array.md). - -**Örnek** - -Sorgu: - -``` sql -SELECT arrayZip(['a', 'b', 'c'], [5, 2, 1]) -``` - -Sonuç: - -``` text -┌─arrayZip(['a', 'b', 'c'], [5, 2, 1])─┐ -│ [('a',5),('b',2),('c',1)] │ -└──────────────────────────────────────┘ -``` - -## arrayAUC {#arrayauc} - -Auc'yi hesaplayın (makine öğreniminde bir kavram olan eğrinin altındaki alan, daha fazla ayrıntıya bakın: https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve). - -**Sözdizimi** - -``` sql -arrayAUC(arr_scores, arr_labels) -``` - -**Parametre** -- `arr_scores` — scores prediction model gives. -- `arr_labels` — labels of samples, usually 1 for positive sample and 0 for negtive sample. - -**Döndürülen değer** -Float64 türü ile AUC değerini döndürür. - -**Örnek** -Sorgu: - -``` sql -select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) -``` - -Sonuç: - -``` text -┌─arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1])─┐ -│ 0.75 │ -└────────────────────────────────────────---──┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/array_functions/) diff --git a/docs/tr/sql-reference/functions/array-join.md b/docs/tr/sql-reference/functions/array-join.md deleted file mode 100644 index 1d097ec510f..00000000000 --- a/docs/tr/sql-reference/functions/array-join.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 61 -toc_title: arrayJoin ---- - -# arrayjoin işlevi {#functions_arrayjoin} - -Bu çok sıradışı bir işlevdir. - -Normal işlevler bir dizi satırı değiştirmez, ancak her satırdaki değerleri değiştirir (harita). -Toplama işlevleri bir dizi satırı sıkıştırır (katlayın veya azaltın). -Bu ‘arrayJoin’ işlev her satırı alır ve bir dizi satır oluşturur (açılır). - -Bu işlev bir diziyi bağımsız değişken olarak alır ve kaynak satırı dizideki öğe sayısı için birden çok satıra yayar. -Sütunlardaki tüm değerler, bu işlevin uygulandığı sütundaki değerler dışında kopyalanır; karşılık gelen dizi değeri ile değiştirilir. - -Bir sorgu birden çok kullanabilirsiniz `arrayJoin` işlevler. Bu durumda, dönüşüm birden çok kez gerçekleştirilir. - -Daha geniş olanaklar sağlayan SELECT sorgusunda dizi birleştirme sözdizimini not alın. - -Örnek: - -``` sql -SELECT arrayJoin([1, 2, 3] AS src) AS dst, 'Hello', src -``` - -``` text -┌─dst─┬─\'Hello\'─┬─src─────┐ -│ 1 │ Hello │ [1,2,3] │ -│ 2 │ Hello │ [1,2,3] │ -│ 3 │ Hello │ [1,2,3] │ -└─────┴───────────┴─────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/array_join/) diff --git a/docs/tr/sql-reference/functions/bit-functions.md b/docs/tr/sql-reference/functions/bit-functions.md deleted file mode 100644 index 09e81202ac3..00000000000 --- a/docs/tr/sql-reference/functions/bit-functions.md +++ /dev/null @@ -1,255 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 48 -toc_title: Bitlik ---- - -# Bit Fonksiyonları {#bit-functions} - -Bit işlevleri, uint8, Uİnt16, Uİnt32, Uint64, Int8, Int16, Int32, Int64, Float32 veya Float64 türlerinden herhangi bir çift için çalışır. - -Sonuç türü, bağımsız değişkenlerinin maksimum bitlerine eşit bit içeren bir tamsayıdır. Bağımsız değişkenlerden en az biri imzalanırsa, sonuç imzalı bir sayıdır. Bir bağımsız değişken bir kayan noktalı sayı ise, Int64 için cast. - -## bıtor(a, b) {#bitanda-b} - -## bitOr(a, b) {#bitora-b} - -## bitXor(a, b) {#bitxora-b} - -## bitNot (a) {#bitnota} - -## bitShiftLeft(a, b) {#bitshiftlefta-b} - -## bitShiftRight(a, b) {#bitshiftrighta-b} - -## bitRotateLeft(a, b) {#bitrotatelefta-b} - -## bitRotateRight(a, b) {#bitrotaterighta-b} - -## bitTest {#bittest} - -Herhangi bir tamsayı alır ve dönüştürür [ikili form](https://en.wikipedia.org/wiki/Binary_number), belirtilen konumda bir bit değerini döndürür. Geri sayım sağdan sola 0 başlar. - -**Sözdizimi** - -``` sql -SELECT bitTest(number, index) -``` - -**Parametre** - -- `number` – integer number. -- `index` – position of bit. - -**Döndürülen değerler** - -Belirtilen konumda bit değeri döndürür. - -Tür: `UInt8`. - -**Örnek** - -Örneğin, taban-2 (ikili) sayı sistemindeki 43 sayısı 101011'dir. - -Sorgu: - -``` sql -SELECT bitTest(43, 1) -``` - -Sonuç: - -``` text -┌─bitTest(43, 1)─┐ -│ 1 │ -└────────────────┘ -``` - -Başka bir örnek: - -Sorgu: - -``` sql -SELECT bitTest(43, 2) -``` - -Sonuç: - -``` text -┌─bitTest(43, 2)─┐ -│ 0 │ -└────────────────┘ -``` - -## bitTestAll {#bittestall} - -Sonucu döndürür [mantıksal conjuction](https://en.wikipedia.org/wiki/Logical_conjunction) Verilen pozisyonlarda tüm bitlerin (ve operatörü). Geri sayım sağdan sola 0 başlar. - -Bitsel işlemler için conjuction: - -0 AND 0 = 0 - -0 AND 1 = 0 - -1 AND 0 = 0 - -1 AND 1 = 1 - -**Sözdizimi** - -``` sql -SELECT bitTestAll(number, index1, index2, index3, index4, ...) -``` - -**Parametre** - -- `number` – integer number. -- `index1`, `index2`, `index3`, `index4` – positions of bit. For example, for set of positions (`index1`, `index2`, `index3`, `index4`) doğru ise ve sadece tüm pozisyon trueları doğru ise (`index1` ⋀ `index2`, ⋀ `index3` ⋀ `index4`). - -**Döndürülen değerler** - -Mantıksal conjuction sonucunu döndürür. - -Tür: `UInt8`. - -**Örnek** - -Örneğin, taban-2 (ikili) sayı sistemindeki 43 sayısı 101011'dir. - -Sorgu: - -``` sql -SELECT bitTestAll(43, 0, 1, 3, 5) -``` - -Sonuç: - -``` text -┌─bitTestAll(43, 0, 1, 3, 5)─┐ -│ 1 │ -└────────────────────────────┘ -``` - -Başka bir örnek: - -Sorgu: - -``` sql -SELECT bitTestAll(43, 0, 1, 3, 5, 2) -``` - -Sonuç: - -``` text -┌─bitTestAll(43, 0, 1, 3, 5, 2)─┐ -│ 0 │ -└───────────────────────────────┘ -``` - -## bitTestAny {#bittestany} - -Sonucu döndürür [mantıksal ayrılma](https://en.wikipedia.org/wiki/Logical_disjunction) Verilen konumlardaki tüm bitlerin (veya operatör). Geri sayım sağdan sola 0 başlar. - -Bitsel işlemler için ayrılma: - -0 OR 0 = 0 - -0 OR 1 = 1 - -1 OR 0 = 1 - -1 OR 1 = 1 - -**Sözdizimi** - -``` sql -SELECT bitTestAny(number, index1, index2, index3, index4, ...) -``` - -**Parametre** - -- `number` – integer number. -- `index1`, `index2`, `index3`, `index4` – positions of bit. - -**Döndürülen değerler** - -Mantıksal disjuction sonucunu döndürür. - -Tür: `UInt8`. - -**Örnek** - -Örneğin, taban-2 (ikili) sayı sistemindeki 43 sayısı 101011'dir. - -Sorgu: - -``` sql -SELECT bitTestAny(43, 0, 2) -``` - -Sonuç: - -``` text -┌─bitTestAny(43, 0, 2)─┐ -│ 1 │ -└──────────────────────┘ -``` - -Başka bir örnek: - -Sorgu: - -``` sql -SELECT bitTestAny(43, 4, 2) -``` - -Sonuç: - -``` text -┌─bitTestAny(43, 4, 2)─┐ -│ 0 │ -└──────────────────────┘ -``` - -## bitCount {#bitcount} - -Bir sayının ikili gösteriminde birine ayarlanmış bit sayısını hesaplar. - -**Sözdizimi** - -``` sql -bitCount(x) -``` - -**Parametre** - -- `x` — [Tamsayı](../../sql-reference/data-types/int-uint.md) veya [kayan nokta](../../sql-reference/data-types/float.md) numara. İşlev, bellekteki değer gösterimini kullanır. Kayan noktalı sayıları desteklemeye izin verir. - -**Döndürülen değer** - -- Giriş numarasında birine ayarlanmış bit sayısı. - -İşlev, giriş değerini daha büyük bir türe dönüştürmez ([işaret uzantısı](https://en.wikipedia.org/wiki/Sign_extension)). Bu yüzden, örneğin , `bitCount(toUInt8(-1)) = 8`. - -Tür: `UInt8`. - -**Örnek** - -Örneğin 333 sayısını alın. İkili gösterimi: 0000000101001101. - -Sorgu: - -``` sql -SELECT bitCount(333) -``` - -Sonuç: - -``` text -┌─bitCount(333)─┐ -│ 5 │ -└───────────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/bit_functions/) diff --git a/docs/tr/sql-reference/functions/bitmap-functions.md b/docs/tr/sql-reference/functions/bitmap-functions.md deleted file mode 100644 index 8e51218da16..00000000000 --- a/docs/tr/sql-reference/functions/bitmap-functions.md +++ /dev/null @@ -1,496 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 49 -toc_title: "E\u015Flem" ---- - -# Bitmap İşlevleri {#bitmap-functions} - -Bitmap işlevleri iki bit eşlemler nesne değeri hesaplama için çalışmak, yeni bitmap veya kardinality formül hesaplama, and, or, xor ve not, vb gibi kullanırken döndürmektir. - -Bitmap nesnesi için 2 çeşit inşaat yöntemi vardır. Biri-State ile toplama işlevi groupBitmap tarafından inşa edilecek, diğeri Array nesnesi tarafından inşa edilecek. Ayrıca bitmap nesnesini dizi nesnesine dönüştürmektir. - -Roaringbitmap, Bitmap nesnelerinin gerçek depolanması sırasında bir veri yapısına sarılır. Önemlilik 32'den küçük veya eşit olduğunda, Set objet kullanır. Kardinality 32'den büyük olduğunda, roaringbitmap nesnesi kullanır. Bu nedenle düşük kardinalite kümesinin depolanması daha hızlıdır. - -RoaringBitmap hakkında daha fazla bilgi için bkz: [CRoaring](https://github.com/RoaringBitmap/CRoaring). - -## bitmapBuild {#bitmap_functions-bitmapbuild} - -İmzasız tamsayı dizisinden bir bit eşlem oluşturun. - -``` sql -bitmapBuild(array) -``` - -**Parametre** - -- `array` – unsigned integer array. - -**Örnek** - -``` sql -SELECT bitmapBuild([1, 2, 3, 4, 5]) AS res, toTypeName(res) -``` - -``` text -┌─res─┬─toTypeName(bitmapBuild([1, 2, 3, 4, 5]))─────┐ -│  │ AggregateFunction(groupBitmap, UInt8) │ -└─────┴──────────────────────────────────────────────┘ -``` - -## bitmapToArray {#bitmaptoarray} - -Bitmap'i tamsayı dizisine dönüştürün. - -``` sql -bitmapToArray(bitmap) -``` - -**Parametre** - -- `bitmap` – bitmap object. - -**Örnek** - -``` sql -SELECT bitmapToArray(bitmapBuild([1, 2, 3, 4, 5])) AS res -``` - -``` text -┌─res─────────┐ -│ [1,2,3,4,5] │ -└─────────────┘ -``` - -## bitmapsubsetınrange {#bitmap-functions-bitmapsubsetinrange} - -Belirtilen aralıktaki alt kümesi döndürür (range_end içermez). - -``` sql -bitmapSubsetInRange(bitmap, range_start, range_end) -``` - -**Parametre** - -- `bitmap` – [Bitmap nesnesi](#bitmap_functions-bitmapbuild). -- `range_start` – range start point. Type: [Uİnt32](../../sql-reference/data-types/int-uint.md). -- `range_end` – range end point(excluded). Type: [Uİnt32](../../sql-reference/data-types/int-uint.md). - -**Örnek** - -``` sql -SELECT bitmapToArray(bitmapSubsetInRange(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res -``` - -``` text -┌─res───────────────┐ -│ [30,31,32,33,100] │ -└───────────────────┘ -``` - -## bitmapSubsetLimit {#bitmapsubsetlimit} - -Arasında alınan n öğeleri ile bitmap bir alt kümesi oluşturur `range_start` ve `cardinality_limit`. - -**Sözdizimi** - -``` sql -bitmapSubsetLimit(bitmap, range_start, cardinality_limit) -``` - -**Parametre** - -- `bitmap` – [Bitmap nesnesi](#bitmap_functions-bitmapbuild). -- `range_start` – The subset starting point. Type: [Uİnt32](../../sql-reference/data-types/int-uint.md). -- `cardinality_limit` – The subset cardinality upper limit. Type: [Uİnt32](../../sql-reference/data-types/int-uint.md). - -**Döndürülen değer** - -Alt. - -Tür: `Bitmap object`. - -**Örnek** - -Sorgu: - -``` sql -SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res -``` - -Sonuç: - -``` text -┌─res───────────────────────┐ -│ [30,31,32,33,100,200,500] │ -└───────────────────────────┘ -``` - -## bitmapContains {#bitmap_functions-bitmapcontains} - -Bit eşlem bir öğe içerip içermediğini denetler. - -``` sql -bitmapContains(haystack, needle) -``` - -**Parametre** - -- `haystack` – [Bitmap nesnesi](#bitmap_functions-bitmapbuild), fonksiyon arar nerede. -- `needle` – Value that the function searches. Type: [Uİnt32](../../sql-reference/data-types/int-uint.md). - -**Döndürülen değerler** - -- 0 — If `haystack` içermez `needle`. -- 1 — If `haystack` içeriyor `needle`. - -Tür: `UInt8`. - -**Örnek** - -``` sql -SELECT bitmapContains(bitmapBuild([1,5,7,9]), toUInt32(9)) AS res -``` - -``` text -┌─res─┐ -│ 1 │ -└─────┘ -``` - -## bitmapHasAny {#bitmaphasany} - -İki bit eşlemin bazı öğelerle kesiştiği olup olmadığını kontrol eder. - -``` sql -bitmapHasAny(bitmap1, bitmap2) -``` - -Eğer eminseniz `bitmap2` kesinlikle bir öğe içerir, kullanmayı düşünün [bitmapContains](#bitmap_functions-bitmapcontains) İşlev. Daha verimli çalışır. - -**Parametre** - -- `bitmap*` – bitmap object. - -**Dönüş değerleri** - -- `1`, eğer `bitmap1` ve `bitmap2` en azından benzer bir öğeye sahip olun. -- `0`, başka. - -**Örnek** - -``` sql -SELECT bitmapHasAny(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res -``` - -``` text -┌─res─┐ -│ 1 │ -└─────┘ -``` - -## bitmapHasAll {#bitmaphasall} - -Benzer `hasAll(array, array)` ilk bit eşlem, ikincisinin tüm öğelerini içeriyorsa, 1 değerini döndürür, aksi halde 0. -İkinci bağımsız değişken boş bir bit eşlem ise, 1 döndürür. - -``` sql -bitmapHasAll(bitmap,bitmap) -``` - -**Parametre** - -- `bitmap` – bitmap object. - -**Örnek** - -``` sql -SELECT bitmapHasAll(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res -``` - -``` text -┌─res─┐ -│ 0 │ -└─────┘ -``` - -## bitmapCardinality {#bitmapcardinality} - -Retrun bit eşlem kardinalite türü Uİnt64. - -``` sql -bitmapCardinality(bitmap) -``` - -**Parametre** - -- `bitmap` – bitmap object. - -**Örnek** - -``` sql -SELECT bitmapCardinality(bitmapBuild([1, 2, 3, 4, 5])) AS res -``` - -``` text -┌─res─┐ -│ 5 │ -└─────┘ -``` - -## bitmapMin {#bitmapmin} - -Kümedeki uint64 türünün en küçük değerini yeniden çalıştırın, küme boşsa UİNT32_MAX. - - bitmapMin(bitmap) - -**Parametre** - -- `bitmap` – bitmap object. - -**Örnek** - -``` sql -SELECT bitmapMin(bitmapBuild([1, 2, 3, 4, 5])) AS res -``` - - ┌─res─┐ - │ 1 │ - └─────┘ - -## bitmapMax {#bitmapmax} - -Küme boşsa, kümedeki uint64 türünün en büyük değerini 0 olarak yeniden çalıştırın. - - bitmapMax(bitmap) - -**Parametre** - -- `bitmap` – bitmap object. - -**Örnek** - -``` sql -SELECT bitmapMax(bitmapBuild([1, 2, 3, 4, 5])) AS res -``` - - ┌─res─┐ - │ 5 │ - └─────┘ - -## bitmapTransform {#bitmaptransform} - -Bitmap'teki bir değer dizisini başka bir değer dizisine dönüştürün, sonuç yeni bir bitmap'tir. - - bitmapTransform(bitmap, from_array, to_array) - -**Parametre** - -- `bitmap` – bitmap object. -- `from_array` – UInt32 array. For idx in range \[0, from_array.size()), if bitmap contains from_array\[idx\], then replace it with to_array\[idx\]. Note that the result depends on array ordering if there are common elements between from_array and to_array. -- `to_array` – UInt32 array, its size shall be the same to from_array. - -**Örnek** - -``` sql -SELECT bitmapToArray(bitmapTransform(bitmapBuild([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), cast([5,999,2] as Array(UInt32)), cast([2,888,20] as Array(UInt32)))) AS res -``` - - ┌─res───────────────────┐ - │ [1,3,4,6,7,8,9,10,20] │ - └───────────────────────┘ - -## bitmapAnd {#bitmapand} - -İki bitmap ve hesaplama, sonuç yeni bir bitmap'tir. - -``` sql -bitmapAnd(bitmap,bitmap) -``` - -**Parametre** - -- `bitmap` – bitmap object. - -**Örnek** - -``` sql -SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res -``` - -``` text -┌─res─┐ -│ [3] │ -└─────┘ -``` - -## bitmapOr {#bitmapor} - -İki bitmap veya hesaplama, sonuç yeni bir bitmap'tir. - -``` sql -bitmapOr(bitmap,bitmap) -``` - -**Parametre** - -- `bitmap` – bitmap object. - -**Örnek** - -``` sql -SELECT bitmapToArray(bitmapOr(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res -``` - -``` text -┌─res─────────┐ -│ [1,2,3,4,5] │ -└─────────────┘ -``` - -## bitmapXor {#bitmapxor} - -İki bitmap XOR hesaplama, sonuç yeni bir bitmap. - -``` sql -bitmapXor(bitmap,bitmap) -``` - -**Parametre** - -- `bitmap` – bitmap object. - -**Örnek** - -``` sql -SELECT bitmapToArray(bitmapXor(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res -``` - -``` text -┌─res───────┐ -│ [1,2,4,5] │ -└───────────┘ -``` - -## bitmapAndnot {#bitmapandnot} - -İki bit eşlem andnot hesaplama, sonuç yeni bir bit eşlem. - -``` sql -bitmapAndnot(bitmap,bitmap) -``` - -**Parametre** - -- `bitmap` – bitmap object. - -**Örnek** - -``` sql -SELECT bitmapToArray(bitmapAndnot(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res -``` - -``` text -┌─res───┐ -│ [1,2] │ -└───────┘ -``` - -## bitmapAndCardinality {#bitmapandcardinality} - -İki bitmap ve hesaplama, uint64 türünün kardinalliğini döndürür. - -``` sql -bitmapAndCardinality(bitmap,bitmap) -``` - -**Parametre** - -- `bitmap` – bitmap object. - -**Örnek** - -``` sql -SELECT bitmapAndCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; -``` - -``` text -┌─res─┐ -│ 1 │ -└─────┘ -``` - -## bitmapOrCardinality {#bitmaporcardinality} - -İki bitmap veya hesaplama, uint64 türünün kardinalliğini döndürür. - -``` sql -bitmapOrCardinality(bitmap,bitmap) -``` - -**Parametre** - -- `bitmap` – bitmap object. - -**Örnek** - -``` sql -SELECT bitmapOrCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; -``` - -``` text -┌─res─┐ -│ 5 │ -└─────┘ -``` - -## bitmapXorCardinality {#bitmapxorcardinality} - -İki bitmap XOR hesaplama, uint64 türünün kardinalliğini döndürür. - -``` sql -bitmapXorCardinality(bitmap,bitmap) -``` - -**Parametre** - -- `bitmap` – bitmap object. - -**Örnek** - -``` sql -SELECT bitmapXorCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; -``` - -``` text -┌─res─┐ -│ 4 │ -└─────┘ -``` - -## bitmapAndnotCardinality {#bitmapandnotcardinality} - -İki bitmap andnot hesaplama, uint64 türünün kardinalliğini döndürür. - -``` sql -bitmapAndnotCardinality(bitmap,bitmap) -``` - -**Parametre** - -- `bitmap` – bitmap object. - -**Örnek** - -``` sql -SELECT bitmapAndnotCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; -``` - -``` text -┌─res─┐ -│ 2 │ -└─────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/bitmap_functions/) diff --git a/docs/tr/sql-reference/functions/comparison-functions.md b/docs/tr/sql-reference/functions/comparison-functions.md deleted file mode 100644 index 43d0dddb6e3..00000000000 --- a/docs/tr/sql-reference/functions/comparison-functions.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 36 -toc_title: "Kar\u015F\u0131la\u015Ft\u0131rma" ---- - -# Karşılaştırma Fonksiyonları {#comparison-functions} - -Karşılaştırma işlevleri her zaman 0 veya 1 (Uint8) döndürür. - -Aşağıdaki türler karşılaştırılabilir: - -- şiir -- dizeler ve sabit dizeler -- tarihliler -- tarihleri ile saatleri - -her grup içinde, ancak farklı gruplar arasında değil. - -Örneğin, bir tarihi bir dizeyle karşılaştıramazsınız. Bir tarih dizesi dönüştürmek, ya da tam tersi bir işlev kullanmak zorunda. - -Dizeler baytlarla karşılaştırılır. Daha kısa bir dize, onunla başlayan ve en az bir karakter daha içeren tüm dizelerden daha küçüktür. - -## eşittir, a = b VE a = = B operatörü {#function-equals} - -## notEquals, a ! operatör = b VE A \<\> b {#function-notequals} - -## l ,ess, \< operat \ operatör {#function-greater} - -## lessOrEquals, \< = operatör {#function-lessorequals} - -## greaterOrEquals, \> = operatör {#function-greaterorequals} - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/comparison_functions/) diff --git a/docs/tr/sql-reference/functions/conditional-functions.md b/docs/tr/sql-reference/functions/conditional-functions.md deleted file mode 100644 index c1f6c7a9880..00000000000 --- a/docs/tr/sql-reference/functions/conditional-functions.md +++ /dev/null @@ -1,207 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 43 -toc_title: "Ko\u015Fullu " ---- - -# Koşullu Fonksiyonlar {#conditional-functions} - -## eğer {#if} - -Koşullu dallanmayı kontrol eder. Çoğu sistemin aksine, ClickHouse her zaman her iki ifadeyi de değerlendirir `then` ve `else`. - -**Sözdizimi** - -``` sql -SELECT if(cond, then, else) -``` - -Eğer durum `cond` sıfır olmayan bir değere değerlendirir, ifadenin sonucunu döndürür `then` ve ifad andenin sonucu `else` varsa, atlanır. Eğer... `cond` sıfır veya `NULL` fakat daha sonra sonucu `then` ifade atlanır ve sonucu `else` Ifade, varsa, döndürülür. - -**Parametre** - -- `cond` – The condition for evaluation that can be zero or not. The type is UInt8, Nullable(UInt8) or NULL. -- `then` - Koşul karşılanırsa dönmek için ifade. -- `else` - Koşul karşılanmazsa dönmek için ifade. - -**Döndürülen değerler** - -İşlev yürütür `then` ve `else` ifadeler ve koşulun olup olmadığına bağlı olarak sonucunu döndürür `cond` sıfır ya da değil. - -**Örnek** - -Sorgu: - -``` sql -SELECT if(1, plus(2, 2), plus(2, 6)) -``` - -Sonuç: - -``` text -┌─plus(2, 2)─┐ -│ 4 │ -└────────────┘ -``` - -Sorgu: - -``` sql -SELECT if(0, plus(2, 2), plus(2, 6)) -``` - -Sonuç: - -``` text -┌─plus(2, 6)─┐ -│ 8 │ -└────────────┘ -``` - -- `then` ve `else` en düşük ortak türe sahip olmalıdır. - -**Örnek:** - -Bunu al `LEFT_RIGHT` Tablo: - -``` sql -SELECT * -FROM LEFT_RIGHT - -┌─left─┬─right─┐ -│ ᴺᵁᴸᴸ │ 4 │ -│ 1 │ 3 │ -│ 2 │ 2 │ -│ 3 │ 1 │ -│ 4 │ ᴺᵁᴸᴸ │ -└──────┴───────┘ -``` - -Aşağıdaki sorgu karşılaştırır `left` ve `right` değerler: - -``` sql -SELECT - left, - right, - if(left < right, 'left is smaller than right', 'right is greater or equal than left') AS is_smaller -FROM LEFT_RIGHT -WHERE isNotNull(left) AND isNotNull(right) - -┌─left─┬─right─┬─is_smaller──────────────────────────┐ -│ 1 │ 3 │ left is smaller than right │ -│ 2 │ 2 │ right is greater or equal than left │ -│ 3 │ 1 │ right is greater or equal than left │ -└──────┴───────┴─────────────────────────────────────┘ -``` - -Not: `NULL` bu örnekte değerler kullanılmaz, kontrol edin [Koşullardaki boş değerler](#null-values-in-conditionals) bölme. - -## Üçlü Operatör {#ternary-operator} - -Aynı gibi çalışıyor. `if` İşlev. - -Sözdizimi: `cond ? then : else` - -Dönüşler `then` eğer... `cond` true (sıfırdan büyük) olarak değerlendirir, aksi takdirde döndürür `else`. - -- `cond` türü olmalıdır `UInt8`, ve `then` ve `else` en düşük ortak türe sahip olmalıdır. - -- `then` ve `else` olabilir `NULL` - -**Ayrıca bakınız** - -- [ifNotFinite](other-functions.md#ifnotfinite). - -## multiİf {#multiif} - -Yaz allowsmanızı sağlar [CASE](../operators/index.md#operator_case) operatör sorguda daha kompakt. - -Sözdizimi: `multiIf(cond_1, then_1, cond_2, then_2, ..., else)` - -**Parametre:** - -- `cond_N` — The condition for the function to return `then_N`. -- `then_N` — The result of the function when executed. -- `else` — The result of the function if none of the conditions is met. - -İşlev kabul eder `2N+1` parametre. - -**Döndürülen değerler** - -İşlev, değerlerden birini döndürür `then_N` veya `else` bu koşullara bağlı olarak `cond_N`. - -**Örnek** - -Yine kullanarak `LEFT_RIGHT` Tablo. - -``` sql -SELECT - left, - right, - multiIf(left < right, 'left is smaller', left > right, 'left is greater', left = right, 'Both equal', 'Null value') AS result -FROM LEFT_RIGHT - -┌─left─┬─right─┬─result──────────┐ -│ ᴺᵁᴸᴸ │ 4 │ Null value │ -│ 1 │ 3 │ left is smaller │ -│ 2 │ 2 │ Both equal │ -│ 3 │ 1 │ left is greater │ -│ 4 │ ᴺᵁᴸᴸ │ Null value │ -└──────┴───────┴─────────────────┘ -``` - -## Koşullu Sonuçları Doğrudan Kullanma {#using-conditional-results-directly} - -Koşullar her zaman sonuç `0`, `1` veya `NULL`. Böylece koşullu sonuçları doğrudan bu şekilde kullanabilirsiniz: - -``` sql -SELECT left < right AS is_small -FROM LEFT_RIGHT - -┌─is_small─┐ -│ ᴺᵁᴸᴸ │ -│ 1 │ -│ 0 │ -│ 0 │ -│ ᴺᵁᴸᴸ │ -└──────────┘ -``` - -## Koşullardaki boş değerler {#null-values-in-conditionals} - -Ne zaman `NULL` değerler koşullarla ilgilidir, sonuç da olacaktır `NULL`. - -``` sql -SELECT - NULL < 1, - 2 < NULL, - NULL < NULL, - NULL = NULL - -┌─less(NULL, 1)─┬─less(2, NULL)─┬─less(NULL, NULL)─┬─equals(NULL, NULL)─┐ -│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ -└───────────────┴───────────────┴──────────────────┴────────────────────┘ -``` - -Bu nedenle, sorgularınızı türleri dikkatli bir şekilde oluşturmalısınız `Nullable`. - -Aşağıdaki örnek, eşittir koşulu eklemek başarısız tarafından bu gösterir `multiIf`. - -``` sql -SELECT - left, - right, - multiIf(left < right, 'left is smaller', left > right, 'right is smaller', 'Both equal') AS faulty_result -FROM LEFT_RIGHT - -┌─left─┬─right─┬─faulty_result────┐ -│ ᴺᵁᴸᴸ │ 4 │ Both equal │ -│ 1 │ 3 │ left is smaller │ -│ 2 │ 2 │ Both equal │ -│ 3 │ 1 │ right is smaller │ -│ 4 │ ᴺᵁᴸᴸ │ Both equal │ -└──────┴───────┴──────────────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/conditional_functions/) diff --git a/docs/tr/sql-reference/functions/date-time-functions.md b/docs/tr/sql-reference/functions/date-time-functions.md deleted file mode 100644 index 7dc082c3670..00000000000 --- a/docs/tr/sql-reference/functions/date-time-functions.md +++ /dev/null @@ -1,450 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 39 -toc_title: "Tarih ve Saatlerle \xE7al\u0131\u015Fma" ---- - -# Tarih ve Saatlerle çalışmak için işlevler {#functions-for-working-with-dates-and-times} - -Saat dilimleri için destek - -Saat dilimi için mantıksal kullanımı olan tarih ve Saat ile çalışmak için tüm işlevler, ikinci bir isteğe bağlı saat dilimi bağımsız değişkeni kabul edebilir. Örnek: Asya / Yekaterinburg. Bu durumda, yerel (varsayılan) yerine belirtilen saat dilimini kullanırlar. - -``` sql -SELECT - toDateTime('2016-06-15 23:00:00') AS time, - toDate(time) AS date_local, - toDate(time, 'Asia/Yekaterinburg') AS date_yekat, - toString(time, 'US/Samoa') AS time_samoa -``` - -``` text -┌────────────────time─┬─date_local─┬─date_yekat─┬─time_samoa──────────┐ -│ 2016-06-15 23:00:00 │ 2016-06-15 │ 2016-06-16 │ 2016-06-15 09:00:00 │ -└─────────────────────┴────────────┴────────────┴─────────────────────┘ -``` - -UTC'DEN saat sayısına göre farklı olan yalnızca saat dilimleri desteklenir. - -## toTimeZone {#totimezone} - -Saat veya tarih ve saati belirtilen saat dilimine dönüştürün. - -## toYear {#toyear} - -Bir tarihi veya tarihi zamanla yıl numarasını (AD) içeren bir Uınt16 numarasına dönüştürür. - -## toQuarter {#toquarter} - -Bir tarihi veya tarihi zaman ile çeyrek sayısını içeren bir Uİnt8 numarasına dönüştürür. - -## toMonth {#tomonth} - -Bir tarih veya tarih ile saati, ay numarasını (1-12) içeren bir Uİnt8 numarasına dönüştürür. - -## bugünyıl {#todayofyear} - -Bir tarih veya tarih ile saat, yılın gün sayısını (1-366) içeren bir Uınt16 numarasına dönüştürür. - -## bugünay {#todayofmonth} - -Bir tarih veya tarih ile saat, Ayın gün sayısını (1-31) içeren bir Uınt8 numarasına dönüştürür. - -## bugünhafta {#todayofweek} - -Bir tarih veya tarih ile saat, haftanın gününün sayısını içeren bir Uınt8 numarasına dönüştürür (Pazartesi 1 ve pazar 7'dir). - -## toHour {#tohour} - -Saatli bir tarihi, 24 saatlik süre (0-23) saat sayısını içeren bir Uınt8 numarasına dönüştürür. -This function assumes that if clocks are moved ahead, it is by one hour and occurs at 2 a.m., and if clocks are moved back, it is by one hour and occurs at 3 a.m. (which is not always true – even in Moscow the clocks were twice changed at a different time). - -## toMinute {#tominute} - -Saatli bir tarihi, saatin dakika sayısını (0-59) içeren bir Uınt8 numarasına dönüştürür. - -## toSecond {#tosecond} - -Dakika (0-59) ikinci sayısını içeren bir uınt8 numarasına zaman ile bir tarih dönüştürür. -Sıçrama saniye hesaba değildir. - -## toUnixTimestamp {#to-unix-timestamp} - -DateTime argümanı için: değeri dahili sayısal gösterimine dönüştürür (Unıx Zaman Damgası). -String argümanı için: datetime'ı dizeden saat dilimine göre ayrıştırın (isteğe bağlı ikinci argüman, sunucu zaman dilimi varsayılan olarak kullanılır) ve karşılık gelen unıx zaman damgasını döndürür. -Tarih argümanı için: davranış belirtilmemiş. - -**Sözdizimi** - -``` sql -toUnixTimestamp(datetime) -toUnixTimestamp(str, [timezone]) -``` - -**Döndürülen değer** - -- Unix zaman damgasını döndürür. - -Tür: `UInt32`. - -**Örnek** - -Sorgu: - -``` sql -SELECT toUnixTimestamp('2017-11-05 08:07:47', 'Asia/Tokyo') AS unix_timestamp -``` - -Sonuç: - -``` text -┌─unix_timestamp─┐ -│ 1509836867 │ -└────────────────┘ -``` - -## toStartOfYear {#tostartofyear} - -Yılın ilk gününe kadar bir tarih veya tarih aşağı yuvarlar. -Tarihi döndürür. - -## toStartOfİSOYear {#tostartofisoyear} - -ISO yılın ilk gününe kadar bir tarih veya tarih aşağı yuvarlar. -Tarihi döndürür. - -## toStartOfQuarter {#tostartofquarter} - -Çeyrek ilk güne kadar bir tarih veya tarih aşağı yuvarlar. -Çeyreğin ilk günü 1 Ocak, 1 Nisan, 1 Temmuz veya 1 ekim'dir. -Tarihi döndürür. - -## toStartOfMonth {#tostartofmonth} - -Ayın ilk gününe kadar bir tarih veya tarih aşağı yuvarlar. -Tarihi döndürür. - -!!! attention "Dikkat" - Yanlış tarihleri ayrıştırma davranışı uygulamaya özeldir. ClickHouse sıfır tarihi döndürebilir, bir istisna atabilir veya yapabilir “natural” taşmak. - -## toMonday {#tomonday} - -En yakın Pazartesi günü bir tarih veya tarih aşağı yuvarlar. -Tarihi döndürür. - -## toStartOfWeek(t \[, mod\]) {#tostartofweektmode} - -Modu ile en yakın pazar veya Pazartesi zaman bir tarih veya tarih aşağı yuvarlar. -Tarihi döndürür. -Mod bağımsız değişkeni, toWeek () için mod bağımsız değişkeni gibi çalışır. Tek bağımsız değişken sözdizimi için 0 mod değeri kullanılır. - -## toStartOfDay {#tostartofday} - -Günün başlangıcına kadar bir tarih aşağı yuvarlar. - -## toStartOfHour {#tostartofhour} - -Saat başlangıcına kadar bir tarih aşağı yuvarlar. - -## toStartOfMinute {#tostartofminute} - -Dakikanın başlangıcına kadar bir tarih aşağı yuvarlar. - -## toStartOfFiveMinute {#tostartoffiveminute} - -Beş dakikalık aralığın başlangıcına kadar bir tarih aşağı yuvarlar. - -## toStartOfTenMinutes {#tostartoftenminutes} - -On dakikalık aralığın başlangıcına kadar bir tarih aşağı yuvarlar. - -## toStartOfFifteenMinutes {#tostartoffifteenminutes} - -On beş dakikalık aralığın başlangıcına kadar tarih aşağı yuvarlar. - -## toStartOfİnterval (time_or_data, Aralık x birimi \[, time_zone\]) {#tostartofintervaltime-or-data-interval-x-unit-time-zone} - -Bu, diğer işlevlerin bir genellemesidir `toStartOf*`. Mesela, -`toStartOfInterval(t, INTERVAL 1 year)` aynı döndürür `toStartOfYear(t)`, -`toStartOfInterval(t, INTERVAL 1 month)` aynı döndürür `toStartOfMonth(t)`, -`toStartOfInterval(t, INTERVAL 1 day)` aynı döndürür `toStartOfDay(t)`, -`toStartOfInterval(t, INTERVAL 15 minute)` aynı döndürür `toStartOfFifteenMinutes(t)` vb. - -## toTime {#totime} - -Belirli bir sabit tarihe zaman ile bir tarih dönüştürür, zaman korurken. - -## toRelativeYearNum {#torelativeyearnum} - -Geçmişte belirli bir sabit noktadan başlayarak, yıl sayısına saat veya tarih ile bir tarih dönüştürür. - -## toRelativeQuarterNum {#torelativequarternum} - -Geçmişte belirli bir sabit noktadan başlayarak, çeyrek sayısına saat veya tarih ile bir tarih dönüştürür. - -## toRelativeMonthNum {#torelativemonthnum} - -Geçmişte belirli bir sabit noktadan başlayarak, Ayın sayısına saat veya tarih ile bir tarih dönüştürür. - -## toRelativeWeekNum {#torelativeweeknum} - -Geçmişte belirli bir sabit noktadan başlayarak, haftanın sayısına saat veya tarih ile bir tarih dönüştürür. - -## toRelativeDayNum {#torelativedaynum} - -Geçmişte belirli bir sabit noktadan başlayarak, günün sayısına saat veya tarih ile bir tarih dönüştürür. - -## toRelativeHourNum {#torelativehournum} - -Geçmişte belirli bir sabit noktadan başlayarak, saat veya tarih ile bir tarih saat sayısına dönüştürür. - -## toRelativeMinuteNum {#torelativeminutenum} - -Geçmişte belirli bir sabit noktadan başlayarak, dakika sayısına saat veya tarih ile bir tarih dönüştürür. - -## toRelativeSecondNum {#torelativesecondnum} - -Geçmişte belirli bir sabit noktadan başlayarak, ikinci sayısına saat veya tarih ile bir tarih dönüştürür. - -## toİSOYear {#toisoyear} - -ISO yıl numarasını içeren bir uınt16 numarasına bir tarih veya tarih zaman dönüştürür. - -## toİSOWeek {#toisoweek} - -ISO hafta numarasını içeren bir uınt8 numarasına bir tarih veya tarih zaman dönüştürür. - -## toWeek (tarih \[, mod\]) {#toweekdatemode} - -Bu işlev, date veya datetime için hafta numarasını döndürür. ToWeek () ' in iki bağımsız değişkenli formu, haftanın pazar veya Pazartesi günü başlayıp başlamadığını ve dönüş değerinin 0 ile 53 arasında mı yoksa 1 ile 53 arasında mı olması gerektiğini belirlemenizi sağlar. Mod bağımsız değişkeni atlanırsa, varsayılan mod 0'dır. -`toISOWeek()`eşdeğer bir uyumluluk işlevidir `toWeek(date,3)`. -Aşağıdaki tabloda mod bağımsız değişkeni nasıl çalıştığını açıklar. - -| Modu | Haftanın ilk günü | Aralık | Week 1 is the first week … | -|------|-------------------|--------|----------------------------------| -| 0 | Pazar | 0-53 | bu yıl bir pazar günü ile | -| 1 | Pazartesi | 0-53 | bu yıl 4 veya daha fazla gün ile | -| 2 | Pazar | 1-53 | bu yıl bir pazar günü ile | -| 3 | Pazartesi | 1-53 | bu yıl 4 veya daha fazla gün ile | -| 4 | Pazar | 0-53 | bu yıl 4 veya daha fazla gün ile | -| 5 | Pazartesi | 0-53 | bu yıl bir Pazartesi ile | -| 6 | Pazar | 1-53 | bu yıl 4 veya daha fazla gün ile | -| 7 | Pazartesi | 1-53 | bu yıl bir Pazartesi ile | -| 8 | Pazar | 1-53 | 1 Ocak içerir | -| 9 | Pazartesi | 1-53 | 1 Ocak içerir | - -Bir anlamı olan mod değerleri için “with 4 or more days this year,” haftalar ISO 8601: 1988'e göre numaralandırılmıştır: - -- 1 Ocak içeren haftanın yeni yılda 4 veya daha fazla günü varsa, 1. haftadır. - -- Aksi takdirde, bir önceki yılın son haftasıdır ve bir sonraki hafta 1. haftadır. - -Bir anlamı olan mod değerleri için “contains January 1”, 1 Ocak haftanın 1.haft .asıdır. Haftanın yeni yılda kaç gün içerdiği önemli değil, sadece bir gün içerse bile. - -``` sql -toWeek(date, [, mode][, Timezone]) -``` - -**Parametre** - -- `date` – Date or DateTime. -- `mode` – Optional parameter, Range of values is \[0,9\], default is 0. -- `Timezone` – Optional parameter, it behaves like any other conversion function. - -**Örnek** - -``` sql -SELECT toDate('2016-12-27') AS date, toWeek(date) AS week0, toWeek(date,1) AS week1, toWeek(date,9) AS week9; -``` - -``` text -┌───────date─┬─week0─┬─week1─┬─week9─┐ -│ 2016-12-27 │ 52 │ 52 │ 1 │ -└────────────┴───────┴───────┴───────┘ -``` - -## toYearWeek (tarih \[, mod\]) {#toyearweekdatemode} - -Bir tarih için yıl ve hafta döndürür. Sonuçtaki yıl, yılın ilk ve son haftası için tarih argümanındaki yıldan farklı olabilir. - -Mod bağımsız değişkeni, toWeek () için mod bağımsız değişkeni gibi çalışır. Tek bağımsız değişken sözdizimi için 0 mod değeri kullanılır. - -`toISOYear()`eşdeğer bir uyumluluk işlevidir `intDiv(toYearWeek(date,3),100)`. - -**Örnek** - -``` sql -SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(date,1) AS yearWeek1, toYearWeek(date,9) AS yearWeek9; -``` - -``` text -┌───────date─┬─yearWeek0─┬─yearWeek1─┬─yearWeek9─┐ -│ 2016-12-27 │ 201652 │ 201652 │ 201701 │ -└────────────┴───────────┴───────────┴───────────┘ -``` - -## şimdi {#now} - -Sıfır bağımsız değişkeni kabul eder ve geçerli saati istek yürütme anlarından birinde döndürür. -Bu işlev, isteğin tamamlanması uzun zaman alsa bile bir sabit döndürür. - -## bugünkü {#today} - -Sıfır bağımsız değişkeni kabul eder ve geçerli tarihi, istek yürütme anlarından birinde döndürür. -Olarak aynı ‘toDate(now())’. - -## dün {#yesterday} - -Sıfır bağımsız değişkeni kabul eder ve istek yürütme anlarından birinde dünün tarihini döndürür. -Olarak aynı ‘today() - 1’. - -## zaman dilimi {#timeslot} - -Yarım saat için zaman yuvarlar. -Bu fonksiyon (kayıt olmak için özeldir.Metrica, yarım saat, bir izleme etiketi, tek bir kullanıcının ardışık sayfa görüntülemelerini, zaman içinde bu miktardan kesinlikle daha fazla farklılık gösteriyorsa, bir oturumu iki oturuma bölmek için minimum zaman miktarıdır. Bu, ilgili oturumda bulunan sayfa görüntülemelerini aramak için tuples (etiket kimliği, kullanıcı kimliği ve zaman dilimi) kullanılabileceği anlamına gelir. - -## toYYYYMM {#toyyyymm} - -Bir tarih veya tarih ile saat, yıl ve ay numarasını (YYYY \* 100 + MM) içeren bir Uınt32 numarasına dönüştürür. - -## toYYYYMMDD {#toyyyymmdd} - -Bir tarih veya tarih ile saat, yıl ve ay numarasını içeren bir Uınt32 numarasına dönüştürür (YYYY \* 10000 + MM \* 100 + DD). - -## toYYYYMMDDhhmmss {#toyyyymmddhhmmss} - -Bir tarihi veya tarihi, yıl ve ay numarasını içeren bir Uınt64 numarasına dönüştürür (YYYY \* 1000000 + MM \* 1000000 + DD \* 1000000 + hh \* 10000 + mm \* 100 + ss). - -## addYears, addMonths, addWeeks, addDays, addHours, addMinutes, addSeconds, addQuarters {#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters} - -İşlev, bir tarih/DateTime aralığına bir tarih/DateTime ekler ve ardından Tarih/Datetime'ı döndürür. Mesela: - -``` sql -WITH - toDate('2018-01-01') AS date, - toDateTime('2018-01-01 00:00:00') AS date_time -SELECT - addYears(date, 1) AS add_years_with_date, - addYears(date_time, 1) AS add_years_with_date_time -``` - -``` text -┌─add_years_with_date─┬─add_years_with_date_time─┐ -│ 2019-01-01 │ 2019-01-01 00:00:00 │ -└─────────────────────┴──────────────────────────┘ -``` - -## subtractYears, subtractMonths, subtractWeeks, subtractDays, subtractHours, subtractMinutes, subtractSeconds, subtractQuarters {#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters} - -Fonksiyon bir tarih/DateTime aralığını bir tarih/DateTime olarak çıkarır ve ardından Tarih/Datetime'ı döndürür. Mesela: - -``` sql -WITH - toDate('2019-01-01') AS date, - toDateTime('2019-01-01 00:00:00') AS date_time -SELECT - subtractYears(date, 1) AS subtract_years_with_date, - subtractYears(date_time, 1) AS subtract_years_with_date_time -``` - -``` text -┌─subtract_years_with_date─┬─subtract_years_with_date_time─┐ -│ 2018-01-01 │ 2018-01-01 00:00:00 │ -└──────────────────────────┴───────────────────────────────┘ -``` - -## dateDiff {#datediff} - -İki Date veya DateTime değerleri arasındaki farkı döndürür. - -**Sözdizimi** - -``` sql -dateDiff('unit', startdate, enddate, [timezone]) -``` - -**Parametre** - -- `unit` — Time unit, in which the returned value is expressed. [Dize](../syntax.md#syntax-string-literal). - - Supported values: - - | unit | - | ---- | - |second | - |minute | - |hour | - |day | - |week | - |month | - |quarter | - |year | - -- `startdate` — The first time value to compare. [Tarihli](../../sql-reference/data-types/date.md) veya [DateTime](../../sql-reference/data-types/datetime.md). - -- `enddate` — The second time value to compare. [Tarihli](../../sql-reference/data-types/date.md) veya [DateTime](../../sql-reference/data-types/datetime.md). - -- `timezone` — Optional parameter. If specified, it is applied to both `startdate` ve `enddate`. Belirtilmemişse, saat dilimleri `startdate` ve `enddate` kullanılır. Aynı değilse, sonuç belirtilmemiştir. - -**Döndürülen değer** - -Arasındaki fark `startdate` ve `enddate` ifade edilen `unit`. - -Tür: `int`. - -**Örnek** - -Sorgu: - -``` sql -SELECT dateDiff('hour', toDateTime('2018-01-01 22:00:00'), toDateTime('2018-01-02 23:00:00')); -``` - -Sonuç: - -``` text -┌─dateDiff('hour', toDateTime('2018-01-01 22:00:00'), toDateTime('2018-01-02 23:00:00'))─┐ -│ 25 │ -└────────────────────────────────────────────────────────────────────────────────────────┘ -``` - -## (StartTime, Süresi,\[, Boyutu zaman yuvasının\]) {#timeslotsstarttime-duration-size} - -Başlayan bir zaman aralığı için ‘StartTime’ ve devam etmek için ‘Duration’ saniye, bu aralıktan aşağı yuvarlanan noktalardan oluşan zaman içinde bir dizi moment döndürür ‘Size’ saniyeler içinde. ‘Size’ isteğe bağlı bir parametredir: varsayılan olarak 1800 olarak ayarlanmış bir sabit Uİnt32. -Mesela, `timeSlots(toDateTime('2012-01-01 12:20:00'), 600) = [toDateTime('2012-01-01 12:00:00'), toDateTime('2012-01-01 12:30:00')]`. -Bu, ilgili oturumda sayfa görüntülemelerini aramak için gereklidir. - -## formatDateTime (saat, Biçim \[, Saat Dilimi\]) {#formatdatetime} - -Function formats a Time according given Format string. N.B.: Format is a constant expression, e.g. you can not have multiple formats for single result column. - -Biçim için desteklenen değiştiriciler: -(“Example” sütun, zaman için biçimlendirme sonucunu gösterir `2018-01-02 22:33:44`) - -| Değiştirici | Açıklama | Örnek | -|-------------|----------------------------------------------------------|------------| -| %C | yıl 100'e bölünür ve tamsayıya kesilir (00-99) | 20 | -| %d | Ayın günü, sıfır yastıklı (01-31) | 02 | -| %D | Kısa MM/DD/YY tarih, eşdeğer %m / %d / % y | 01/02/18 | -| %e | Ayın günü, boşluk dolgulu (1-31) | 2 | -| %F | kısa YYYY-AA-DD tarih, eşdeğer %Y-%m - %d | 2018-01-02 | -| %H | 24 saat formatında saat (00-23) | 22 | -| %I | 12h formatında saat (01-12) | 10 | -| %j | yılın günü (001-366) | 002 | -| %metre | ondalık sayı olarak ay (01-12) | 01 | -| %M | dakika (00-59) | 33 | -| %ve | new-line char (ac (ter (") | | -| %p | AM veya PM atama | PM | -| %R | 24-hour HH: MM Zaman, eşdeğer %H:%M | 22:33 | -| %S | ikinci (00-59) | 44 | -| %t | yatay-sekme karakteri (') | | -| %T | ISO 8601 saat biçimi (HH:MM:SS), eşdeğer %H:%M: % S | 22:33:44 | -| %u | ISO 8601 hafta içi sayı olarak Pazartesi olarak 1 (1-7) | 2 | -| %V | ISO 8601 hafta numarası (01-53) | 01 | -| %g | Pazar günü 0 (0-6) olarak ondalık sayı olarak hafta içi) | 2 | -| %y | Yıl, son iki basamak (00-99) | 18 | -| %Y | Yıllık | 2018 | -| %% | im | % | - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/date_time_functions/) diff --git a/docs/tr/sql-reference/functions/encoding-functions.md b/docs/tr/sql-reference/functions/encoding-functions.md deleted file mode 100644 index b0ba76a6c32..00000000000 --- a/docs/tr/sql-reference/functions/encoding-functions.md +++ /dev/null @@ -1,175 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 52 -toc_title: "Kodlam\u0131\u015F" ---- - -# Kodlama Fonksiyonları {#encoding-functions} - -## kömürleşmek {#char} - -Geçirilen bağımsız değişkenlerin sayısı olarak uzunluğu olan dizeyi döndürür ve her bayt karşılık gelen bağımsız değişken değerine sahiptir. Sayısal türlerin birden çok bağımsız değişkeni kabul eder. Bağımsız değişken değeri uint8 veri türü aralığının dışındaysa, Olası yuvarlama ve taşma ile Uint8'e dönüştürülür. - -**Sözdizimi** - -``` sql -char(number_1, [number_2, ..., number_n]); -``` - -**Parametre** - -- `number_1, number_2, ..., number_n` — Numerical arguments interpreted as integers. Types: [Tamsayı](../../sql-reference/data-types/int-uint.md), [Yüzdürmek](../../sql-reference/data-types/float.md). - -**Döndürülen değer** - -- verilen bayt dizisi. - -Tür: `String`. - -**Örnek** - -Sorgu: - -``` sql -SELECT char(104.1, 101, 108.9, 108.9, 111) AS hello -``` - -Sonuç: - -``` text -┌─hello─┐ -│ hello │ -└───────┘ -``` - -Karşılık gelen baytları geçirerek bir rasgele kodlama dizesi oluşturabilirsiniz. İşte UTF-8 için örnek: - -Sorgu: - -``` sql -SELECT char(0xD0, 0xBF, 0xD1, 0x80, 0xD0, 0xB8, 0xD0, 0xB2, 0xD0, 0xB5, 0xD1, 0x82) AS hello; -``` - -Sonuç: - -``` text -┌─hello──┐ -│ привет │ -└────────┘ -``` - -Sorgu: - -``` sql -SELECT char(0xE4, 0xBD, 0xA0, 0xE5, 0xA5, 0xBD) AS hello; -``` - -Sonuç: - -``` text -┌─hello─┐ -│ 你好 │ -└───────┘ -``` - -## büyü {#hex} - -Bağımsız değişkenin onaltılık gösterimini içeren bir dize döndürür. - -**Sözdizimi** - -``` sql -hex(arg) -``` - -İşlev büyük harfler kullanıyor `A-F` ve herhangi bir önek kullanmamak (gibi `0x`) veya sonekler (gibi `h`). - -Tamsayı argümanları için, onaltılık basamak yazdırır (“nibbles”) en önemliden en önemlisine (big endian veya “human readable” sipariş). En önemli sıfır olmayan baytla başlar (önde gelen sıfır bayt atlanır), ancak önde gelen basamak sıfır olsa bile her baytın her iki basamağını da yazdırır. - -Örnek: - -**Örnek** - -Sorgu: - -``` sql -SELECT hex(1); -``` - -Sonuç: - -``` text -01 -``` - -Tip değerleri `Date` ve `DateTime` karşılık gelen tamsayılar olarak biçimlendirilir (tarih için çağdan bu yana geçen gün sayısı ve datetime için Unix zaman damgasının değeri). - -İçin `String` ve `FixedString`, tüm bayt sadece iki onaltılık sayı olarak kodlanır. Sıfır bayt ihmal edilmez. - -Kayan nokta ve ondalık türlerinin değerleri, bellekteki gösterimi olarak kodlanır. Küçük endian mimarisini desteklediğimiz için, bunlar küçük endian'da kodlanmıştır. Sıfır önde gelen / sondaki bayt ihmal edilmez. - -**Parametre** - -- `arg` — A value to convert to hexadecimal. Types: [Dize](../../sql-reference/data-types/string.md), [Uİnt](../../sql-reference/data-types/int-uint.md), [Yüzdürmek](../../sql-reference/data-types/float.md), [Ondalık](../../sql-reference/data-types/decimal.md), [Tarihli](../../sql-reference/data-types/date.md) veya [DateTime](../../sql-reference/data-types/datetime.md). - -**Döndürülen değer** - -- Bağımsız değişken onaltılık gösterimi ile bir dize. - -Tür: `String`. - -**Örnek** - -Sorgu: - -``` sql -SELECT hex(toFloat32(number)) as hex_presentation FROM numbers(15, 2); -``` - -Sonuç: - -``` text -┌─hex_presentation─┐ -│ 00007041 │ -│ 00008041 │ -└──────────────────┘ -``` - -Sorgu: - -``` sql -SELECT hex(toFloat64(number)) as hex_presentation FROM numbers(15, 2); -``` - -Sonuç: - -``` text -┌─hex_presentation─┐ -│ 0000000000002E40 │ -│ 0000000000003040 │ -└──────────────────┘ -``` - -## unhex (str) {#unhexstr} - -Onaltılık basamak herhangi bir sayıda içeren bir dize kabul eder ve karşılık gelen bayt içeren bir dize döndürür. Hem büyük hem de küçük harfleri destekler a-F. onaltılık basamak sayısı bile olmak zorunda değildir. Tek ise, son rakam 00-0F baytın en az önemli yarısı olarak yorumlanır. Bağımsız değişken dizesi onaltılık basamaklardan başka bir şey içeriyorsa, uygulama tanımlı bazı sonuçlar döndürülür (bir özel durum atılmaz). -Sonucu bir sayıya dönüştürmek istiyorsanız, ‘reverse’ ve ‘reinterpretAsType’ işlevler. - -## UUİDStringToNum (str) {#uuidstringtonumstr} - -Biçiminde 36 karakter içeren bir dize kabul eder `123e4567-e89b-12d3-a456-426655440000` ve bir fixedstring(16) bayt kümesi olarak döndürür. - -## UUİDNumToString (str) {#uuidnumtostringstr} - -FixedString(16) değerini kabul eder. Metin biçiminde 36 karakter içeren bir dize döndürür. - -## bitmaskToList (num) {#bitmasktolistnum} - -Bir tamsayı kabul eder. Özetlendiğinde kaynak sayısını toplayan iki güç listesini içeren bir dize döndürür. Artan düzende metin biçiminde boşluk bırakmadan virgülle ayrılırlar. - -## bitmaskToArray (num) {#bitmasktoarraynum} - -Bir tamsayı kabul eder. Özetlendiğinde kaynak sayısını toplayan iki güç listesini içeren bir uint64 sayı dizisi döndürür. Dizideki sayılar artan düzendedir. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/encoding_functions/) diff --git a/docs/tr/sql-reference/functions/ext-dict-functions.md b/docs/tr/sql-reference/functions/ext-dict-functions.md deleted file mode 100644 index 80f06040b8c..00000000000 --- a/docs/tr/sql-reference/functions/ext-dict-functions.md +++ /dev/null @@ -1,205 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 58 -toc_title: "Harici S\xF6zl\xFCklerle \xE7al\u0131\u015Fma" ---- - -# Harici Sözlüklerle çalışmak için işlevler {#ext_dict_functions} - -Dış sözlükleri bağlama ve yapılandırma hakkında bilgi için bkz. [Dış söz dictionarieslükler](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md). - -## dictGet {#dictget} - -Harici bir sözlükten bir değer alır. - -``` sql -dictGet('dict_name', 'attr_name', id_expr) -dictGetOrDefault('dict_name', 'attr_name', id_expr, default_value_expr) -``` - -**Parametre** - -- `dict_name` — Name of the dictionary. [String lit literal](../syntax.md#syntax-string-literal). -- `attr_name` — Name of the column of the dictionary. [String lit literal](../syntax.md#syntax-string-literal). -- `id_expr` — Key value. [İfade](../syntax.md#syntax-expressions) dönen bir [Uİnt64](../../sql-reference/data-types/int-uint.md) veya [Demet](../../sql-reference/data-types/tuple.md)- sözlük yapılandırmasına bağlı olarak değer yazın. -- `default_value_expr` — Value returned if the dictionary doesn't contain a row with the `id_expr` anahtar. [İfade](../syntax.md#syntax-expressions) veri türü için yapılandırılmış değeri döndürme `attr_name` nitelik. - -**Döndürülen değer** - -- ClickHouse özniteliği başarıyla ayrıştırırsa [öznitelik veri türü](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md#ext_dict_structure-attributes), fonksiyonlar karşılık gelen sözlük özniteliğinin değerini döndürür `id_expr`. - -- Anahtar yoksa, karşılık gelen `id_expr`, söz thelükte, sonra: - - - `dictGet` returns the content of the `` element specified for the attribute in the dictionary configuration. - - `dictGetOrDefault` returns the value passed as the `default_value_expr` parameter. - -Clickhouse, özniteliğin değerini ayrıştıramazsa veya değer öznitelik veri türüyle eşleşmiyorsa bir özel durum atar. - -**Örnek** - -Metin dosyası oluşturma `ext-dict-text.csv` aşağıdakileri içeren: - -``` text -1,1 -2,2 -``` - -İlk sütun `id` ikinci sütun `c1`. - -Dış sözlüğü yapılandırma: - -``` xml - - - ext-dict-test - - - /path-to/ext-dict-test.csv - CSV - - - - - - - - id - - - c1 - UInt32 - - - - 0 - - -``` - -Sorguyu gerçekleştir: - -``` sql -SELECT - dictGetOrDefault('ext-dict-test', 'c1', number + 1, toUInt32(number * 10)) AS val, - toTypeName(val) AS type -FROM system.numbers -LIMIT 3 -``` - -``` text -┌─val─┬─type───┐ -│ 1 │ UInt32 │ -│ 2 │ UInt32 │ -│ 20 │ UInt32 │ -└─────┴────────┘ -``` - -**Ayrıca Bakınız** - -- [Dış Söz Dictionarieslükler](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) - -## dictHas {#dicthas} - -Bir anahtar sözlükte mevcut olup olmadığını denetler. - -``` sql -dictHas('dict_name', id_expr) -``` - -**Parametre** - -- `dict_name` — Name of the dictionary. [String lit literal](../syntax.md#syntax-string-literal). -- `id_expr` — Key value. [İfade](../syntax.md#syntax-expressions) dönen bir [Uİnt64](../../sql-reference/data-types/int-uint.md)- tip değeri. - -**Döndürülen değer** - -- 0, anahtar yoksa. -- 1, bir anahtar varsa. - -Tür: `UInt8`. - -## dictGetHierarchy {#dictgethierarchy} - -Bir anahtarın tüm ebeveynlerini içeren bir dizi oluşturur. [hiyerarş dictionaryik sözlük](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md). - -**Sözdizimi** - -``` sql -dictGetHierarchy('dict_name', key) -``` - -**Parametre** - -- `dict_name` — Name of the dictionary. [String lit literal](../syntax.md#syntax-string-literal). -- `key` — Key value. [İfade](../syntax.md#syntax-expressions) dönen bir [Uİnt64](../../sql-reference/data-types/int-uint.md)- tip değeri. - -**Döndürülen değer** - -- Anahtar için ebeveynler. - -Tür: [Dizi (Uİnt64)](../../sql-reference/data-types/array.md). - -## dictİsİn {#dictisin} - -Sözlükteki tüm hiyerarşik zincir boyunca bir anahtarın atasını kontrol eder. - -``` sql -dictIsIn('dict_name', child_id_expr, ancestor_id_expr) -``` - -**Parametre** - -- `dict_name` — Name of the dictionary. [String lit literal](../syntax.md#syntax-string-literal). -- `child_id_expr` — Key to be checked. [İfade](../syntax.md#syntax-expressions) dönen bir [Uİnt64](../../sql-reference/data-types/int-uint.md)- tip değeri. -- `ancestor_id_expr` — Alleged ancestor of the `child_id_expr` anahtar. [İfade](../syntax.md#syntax-expressions) dönen bir [Uİnt64](../../sql-reference/data-types/int-uint.md)- tip değeri. - -**Döndürülen değer** - -- 0, eğer `child_id_expr` bir çocuk değil mi `ancestor_id_expr`. -- 1, Eğer `child_id_expr` bir çocuk `ancestor_id_expr` veya eğer `child_id_expr` is an `ancestor_id_expr`. - -Tür: `UInt8`. - -## Diğer Fonksiyonlar {#ext_dict_functions-other} - -ClickHouse sözlük yapılandırma ne olursa olsun belirli bir veri türü için sözlük öznitelik değerlerini dönüştürmek özel işlevleri destekler. - -İşlevler: - -- `dictGetInt8`, `dictGetInt16`, `dictGetInt32`, `dictGetInt64` -- `dictGetUInt8`, `dictGetUInt16`, `dictGetUInt32`, `dictGetUInt64` -- `dictGetFloat32`, `dictGetFloat64` -- `dictGetDate` -- `dictGetDateTime` -- `dictGetUUID` -- `dictGetString` - -Tüm bu işlevler `OrDefault` değişiklik. Mesela, `dictGetDateOrDefault`. - -Sözdizimi: - -``` sql -dictGet[Type]('dict_name', 'attr_name', id_expr) -dictGet[Type]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) -``` - -**Parametre** - -- `dict_name` — Name of the dictionary. [String lit literal](../syntax.md#syntax-string-literal). -- `attr_name` — Name of the column of the dictionary. [String lit literal](../syntax.md#syntax-string-literal). -- `id_expr` — Key value. [İfade](../syntax.md#syntax-expressions) dönen bir [Uİnt64](../../sql-reference/data-types/int-uint.md)- tip değeri. -- `default_value_expr` — Value which is returned if the dictionary doesn't contain a row with the `id_expr` anahtar. [İfade](../syntax.md#syntax-expressions) veri türü için yapılandırılmış bir değer döndürme `attr_name` nitelik. - -**Döndürülen değer** - -- ClickHouse özniteliği başarıyla ayrıştırırsa [öznitelik veri türü](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md#ext_dict_structure-attributes), fonksiyonlar karşılık gelen sözlük özniteliğinin değerini döndürür `id_expr`. - -- İsten nomiyorsa `id_expr` söz thelükte o zaman: - - - `dictGet[Type]` returns the content of the `` element specified for the attribute in the dictionary configuration. - - `dictGet[Type]OrDefault` returns the value passed as the `default_value_expr` parameter. - -Clickhouse, özniteliğin değerini ayrıştıramazsa veya değer öznitelik veri türüyle eşleşmiyorsa bir özel durum atar. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/ext_dict_functions/) diff --git a/docs/tr/sql-reference/functions/functions-for-nulls.md b/docs/tr/sql-reference/functions/functions-for-nulls.md deleted file mode 100644 index 19e6f32dfd5..00000000000 --- a/docs/tr/sql-reference/functions/functions-for-nulls.md +++ /dev/null @@ -1,312 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 63 -toc_title: "Null arg\xFCmanlarla \xE7al\u0131\u015Fma" ---- - -# Null Agregalarla çalışmak için işlevler {#functions-for-working-with-nullable-aggregates} - -## isNull {#isnull} - -Bağımsız değişken olup olmadığını denetler [NULL](../../sql-reference/syntax.md#null-literal). - -``` sql -isNull(x) -``` - -**Parametre** - -- `x` — A value with a non-compound data type. - -**Döndürülen değer** - -- `1` eğer `x` oluyor `NULL`. -- `0` eğer `x` değildir `NULL`. - -**Örnek** - -Giriş tablosu - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 2 │ 3 │ -└───┴──────┘ -``` - -Sorgu - -``` sql -SELECT x FROM t_null WHERE isNull(y) -``` - -``` text -┌─x─┐ -│ 1 │ -└───┘ -``` - -## isNotNull {#isnotnull} - -Bağımsız değişken olup olmadığını denetler [NULL](../../sql-reference/syntax.md#null-literal). - -``` sql -isNotNull(x) -``` - -**Parametre:** - -- `x` — A value with a non-compound data type. - -**Döndürülen değer** - -- `0` eğer `x` oluyor `NULL`. -- `1` eğer `x` değildir `NULL`. - -**Örnek** - -Giriş tablosu - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 2 │ 3 │ -└───┴──────┘ -``` - -Sorgu - -``` sql -SELECT x FROM t_null WHERE isNotNull(y) -``` - -``` text -┌─x─┐ -│ 2 │ -└───┘ -``` - -## birleşmek {#coalesce} - -Olup olmadığını soldan sağa denetler `NULL` argümanlar geçti ve ilk olmayan döndürür-`NULL` tartışma. - -``` sql -coalesce(x,...) -``` - -**Parametre:** - -- Bileşik olmayan tipte herhangi bir sayıda parametre. Tüm parametreler veri türüne göre uyumlu olmalıdır. - -**Döndürülen değerler** - -- İlk sigara-`NULL` tartışma. -- `NULL`, eğer tüm argümanlar `NULL`. - -**Örnek** - -Bir müşteriyle iletişim kurmak için birden çok yol belirtebilecek kişilerin listesini düşünün. - -``` text -┌─name─────┬─mail─┬─phone─────┬──icq─┐ -│ client 1 │ ᴺᵁᴸᴸ │ 123-45-67 │ 123 │ -│ client 2 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ -└──────────┴──────┴───────────┴──────┘ -``` - -Bu `mail` ve `phone` alanlar String tip ofindedir, ancak `icq` Fi fieldeld is `UInt32`, bu yüzden dönüştürülmesi gerekiyor `String`. - -Müşteri için ilk kullanılabilir iletişim yöntemini kişi listesinden alın: - -``` sql -SELECT coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook -``` - -``` text -┌─name─────┬─coalesce(mail, phone, CAST(icq, 'Nullable(String)'))─┐ -│ client 1 │ 123-45-67 │ -│ client 2 │ ᴺᵁᴸᴸ │ -└──────────┴──────────────────────────────────────────────────────┘ -``` - -## ifNull {#ifnull} - -Ana bağımsız değişken ise alternatif bir değer döndürür `NULL`. - -``` sql -ifNull(x,alt) -``` - -**Parametre:** - -- `x` — The value to check for `NULL`. -- `alt` — The value that the function returns if `x` oluyor `NULL`. - -**Döndürülen değerler** - -- Değer `x`, eğer `x` değildir `NULL`. -- Değer `alt`, eğer `x` oluyor `NULL`. - -**Örnek** - -``` sql -SELECT ifNull('a', 'b') -``` - -``` text -┌─ifNull('a', 'b')─┐ -│ a │ -└──────────────────┘ -``` - -``` sql -SELECT ifNull(NULL, 'b') -``` - -``` text -┌─ifNull(NULL, 'b')─┐ -│ b │ -└───────────────────┘ -``` - -## nullİf {#nullif} - -Dönüşler `NULL` argümanlar eşitse. - -``` sql -nullIf(x, y) -``` - -**Parametre:** - -`x`, `y` — Values for comparison. They must be compatible types, or ClickHouse will generate an exception. - -**Döndürülen değerler** - -- `NULL`, argümanlar eşitse. -- Bu `x` bağımsız değişkenler eşit değilse, değer. - -**Örnek** - -``` sql -SELECT nullIf(1, 1) -``` - -``` text -┌─nullIf(1, 1)─┐ -│ ᴺᵁᴸᴸ │ -└──────────────┘ -``` - -``` sql -SELECT nullIf(1, 2) -``` - -``` text -┌─nullIf(1, 2)─┐ -│ 1 │ -└──────────────┘ -``` - -## assumeNotNull {#assumenotnull} - -Bir tür değeri ile sonuçlanır [Nullable](../../sql-reference/data-types/nullable.md) bir sigara için- `Nullable` eğer değer değil `NULL`. - -``` sql -assumeNotNull(x) -``` - -**Parametre:** - -- `x` — The original value. - -**Döndürülen değerler** - -- Olmayan orijinal değeri-`Nullable` tipi, değilse `NULL`. -- Olmayan için varsayılan değer-`Nullable` özgün değer ise yazın `NULL`. - -**Örnek** - -Düşünün `t_null` Tablo. - -``` sql -SHOW CREATE TABLE t_null -``` - -``` text -┌─statement─────────────────────────────────────────────────────────────────┐ -│ CREATE TABLE default.t_null ( x Int8, y Nullable(Int8)) ENGINE = TinyLog │ -└───────────────────────────────────────────────────────────────────────────┘ -``` - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 2 │ 3 │ -└───┴──────┘ -``` - -Uygula `assumeNotNull` fonksiyonu için `y` sütun. - -``` sql -SELECT assumeNotNull(y) FROM t_null -``` - -``` text -┌─assumeNotNull(y)─┐ -│ 0 │ -│ 3 │ -└──────────────────┘ -``` - -``` sql -SELECT toTypeName(assumeNotNull(y)) FROM t_null -``` - -``` text -┌─toTypeName(assumeNotNull(y))─┐ -│ Int8 │ -│ Int8 │ -└──────────────────────────────┘ -``` - -## toNullable {#tonullable} - -Bağımsız değişken türünü dönüştürür `Nullable`. - -``` sql -toNullable(x) -``` - -**Parametre:** - -- `x` — The value of any non-compound type. - -**Döndürülen değer** - -- Bir ile giriş değeri `Nullable` tür. - -**Örnek** - -``` sql -SELECT toTypeName(10) -``` - -``` text -┌─toTypeName(10)─┐ -│ UInt8 │ -└────────────────┘ -``` - -``` sql -SELECT toTypeName(toNullable(10)) -``` - -``` text -┌─toTypeName(toNullable(10))─┐ -│ Nullable(UInt8) │ -└────────────────────────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/functions_for_nulls/) diff --git a/docs/tr/sql-reference/functions/geo.md b/docs/tr/sql-reference/functions/geo.md deleted file mode 100644 index 39ffd44270c..00000000000 --- a/docs/tr/sql-reference/functions/geo.md +++ /dev/null @@ -1,510 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 62 -toc_title: "Co\u011Frafi koordinatlar ile \xE7al\u0131\u015Fma" ---- - -# Coğrafi Koordinatlarla çalışmak için fonksiyonlar {#functions-for-working-with-geographical-coordinates} - -## greatCircleDistance {#greatcircledistance} - -Dünya yüzeyindeki iki nokta arasındaki mesafeyi kullanarak hesaplayın [büyük daire formülü](https://en.wikipedia.org/wiki/Great-circle_distance). - -``` sql -greatCircleDistance(lon1Deg, lat1Deg, lon2Deg, lat2Deg) -``` - -**Giriş parametreleri** - -- `lon1Deg` — Longitude of the first point in degrees. Range: `[-180°, 180°]`. -- `lat1Deg` — Latitude of the first point in degrees. Range: `[-90°, 90°]`. -- `lon2Deg` — Longitude of the second point in degrees. Range: `[-180°, 180°]`. -- `lat2Deg` — Latitude of the second point in degrees. Range: `[-90°, 90°]`. - -Pozitif değerler Kuzey enlemi ve Doğu boylamına karşılık gelir ve negatif değerler Güney enlemi ve Batı boylamına karşılık gelir. - -**Döndürülen değer** - -Dünya yüzeyindeki iki nokta arasındaki mesafe, metre cinsinden. - -Girdi parametre değerleri aralığın dışına düştüğünde bir özel durum oluşturur. - -**Örnek** - -``` sql -SELECT greatCircleDistance(55.755831, 37.617673, -55.755831, -37.617673) -``` - -``` text -┌─greatCircleDistance(55.755831, 37.617673, -55.755831, -37.617673)─┐ -│ 14132374.194975413 │ -└───────────────────────────────────────────────────────────────────┘ -``` - -## pointİnEllipses {#pointinellipses} - -Noktanın elipslerden en az birine ait olup olmadığını kontrol eder. -Koordinatlar kartezyen koordinat sisteminde geometriktir. - -``` sql -pointInEllipses(x, y, x₀, y₀, a₀, b₀,...,xₙ, yₙ, aₙ, bₙ) -``` - -**Giriş parametreleri** - -- `x, y` — Coordinates of a point on the plane. -- `xᵢ, yᵢ` — Coordinates of the center of the `i`-inci üç nokta. -- `aᵢ, bᵢ` — Axes of the `i`- x, y koordinatları birimlerinde üç nokta. - -Giriş parametreleri olmalıdır `2+4⋅n`, nere `n` elips sayısıdır. - -**Döndürülen değerler** - -`1` nokta elipslerden en az birinin içindeyse; `0`hayır değil. - -**Örnek** - -``` sql -SELECT pointInEllipses(10., 10., 10., 9.1, 1., 0.9999) -``` - -``` text -┌─pointInEllipses(10., 10., 10., 9.1, 1., 0.9999)─┐ -│ 1 │ -└─────────────────────────────────────────────────┘ -``` - -## pointİnPolygon {#pointinpolygon} - -Noktanın düzlemdeki poligona ait olup olmadığını kontrol eder. - -``` sql -pointInPolygon((x, y), [(a, b), (c, d) ...], ...) -``` - -**Giriş değerleri** - -- `(x, y)` — Coordinates of a point on the plane. Data type — [Demet](../../sql-reference/data-types/tuple.md) — A tuple of two numbers. -- `[(a, b), (c, d) ...]` — Polygon vertices. Data type — [Dizi](../../sql-reference/data-types/array.md). Her köşe bir çift koordinat ile temsil edilir `(a, b)`. Köşeler saat yönünde veya saat yönünün tersine sırayla belirtilmelidir. Minimum köşe sayısı 3'tür. Çokgen sabit olmalıdır. -- Fonksiyon ayrıca delikli çokgenleri de destekler (bölümleri keser). Bu durumda, işlevin ek argümanlarını kullanarak kesilen bölümleri tanımlayan çokgenler ekleyin. İşlev, basit olmayan bağlı çokgenleri desteklemez. - -**Döndürülen değerler** - -`1` nokta çokgenin içinde ise, `0` hayır değil. -Nokta çokgen sınırında ise, işlev 0 veya 1 döndürebilir. - -**Örnek** - -``` sql -SELECT pointInPolygon((3., 3.), [(6, 0), (8, 4), (5, 8), (0, 2)]) AS res -``` - -``` text -┌─res─┐ -│ 1 │ -└─────┘ -``` - -## geohashEncode {#geohashencode} - -Enlem ve boylamı bir geohash-string olarak kodlar, Lütfen bakınız (http://geohash.org/, https://en.wikipedia.org/wiki/Geohash). - -``` sql -geohashEncode(longitude, latitude, [precision]) -``` - -**Giriş değerleri** - -- boylam-kodlamak istediğiniz koordinatın boylam kısmı. Aralık floatingta yüz floatingen`[-180°, 180°]` -- latitude-kodlamak istediğiniz koordinatın enlem kısmı. Aralık floatingta yüz floatingen `[-90°, 90°]` -- hassas-isteğe bağlı, elde edilen kodlanmış dizenin uzunluğu, varsayılan olarak `12`. Aralıktaki tamsayı `[1, 12]`. Herhangi bir değer daha az `1` veya daha büyük `12` sessizce dönüştürülür `12`. - -**Döndürülen değerler** - -- alfanümerik `String` kodlanmış koordinat (base32-kodlama alfabesinin değiştirilmiş versiyonu kullanılır). - -**Örnek** - -``` sql -SELECT geohashEncode(-5.60302734375, 42.593994140625, 0) AS res -``` - -``` text -┌─res──────────┐ -│ ezs42d000000 │ -└──────────────┘ -``` - -## geohashDecode {#geohashdecode} - -Herhangi bir geohash kodlu dizeyi boylam ve enlem olarak çözer. - -**Giriş değerleri** - -- kodlanmış dize-geohash kodlanmış dize. - -**Döndürülen değerler** - -- (boylam, enlem) - 2-tuple `Float64` boylam ve enlem değerleri. - -**Örnek** - -``` sql -SELECT geohashDecode('ezs42') AS res -``` - -``` text -┌─res─────────────────────────────┐ -│ (-5.60302734375,42.60498046875) │ -└─────────────────────────────────┘ -``` - -## geoToH3 {#geotoh3} - -Dönüşler [H3](https://uber.github.io/h3/#/documentation/overview/introduction) nokta Endeksi `(lon, lat)` belirtilen çözünürlük ile. - -[H3](https://uber.github.io/h3/#/documentation/overview/introduction) Dünya yüzeyinin altıgen fayanslara bile bölündüğü coğrafi bir indeksleme sistemidir. Bu sistem hiyerarşiktir, örn. üst seviyedeki her altıgen yedi hatta daha küçük olanlara bölünebilir. - -Bu endeks öncelikle kovalama yerleri ve diğer coğrafi manipülasyonlar için kullanılır. - -**Sözdizimi** - -``` sql -geoToH3(lon, lat, resolution) -``` - -**Parametre** - -- `lon` — Longitude. Type: [Float64](../../sql-reference/data-types/float.md). -- `lat` — Latitude. Type: [Float64](../../sql-reference/data-types/float.md). -- `resolution` — Index resolution. Range: `[0, 15]`. Tür: [Uİnt8](../../sql-reference/data-types/int-uint.md). - -**Döndürülen değerler** - -- Altıgen dizin numarası. -- Hata durumunda 0. - -Tür: `UInt64`. - -**Örnek** - -Sorgu: - -``` sql -SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index -``` - -Sonuç: - -``` text -┌────────────h3Index─┐ -│ 644325524701193974 │ -└────────────────────┘ -``` - -## geohashesİnBox {#geohashesinbox} - -Verilen kutunun içine giren ve verilen kutunun sınırlarını kesişen, temel olarak diziye düzleştirilmiş bir 2D ızgarası olan bir dizi geohash kodlu dizeler dizisi döndürür. - -**Giriş değerleri** - -- longitude_min - min boylam, aralıkta kayan değer `[-180°, 180°]` -- latitude_min - min enlem, aralıkta kayan değer `[-90°, 90°]` -- longitude_max-maksimum boylam, aralıkta kayan değer `[-180°, 180°]` -- latitude_max-maksimum enlem, aralıkta kayan değer `[-90°, 90°]` -- hassas-geohash hassas, `UInt8` Aralık inta `[1, 12]` - -Lütfen tüm koordinat parametrelerinin aynı tipte olması gerektiğini unutmayın: `Float32` veya `Float64`. - -**Döndürülen değerler** - -- verilen alanı kapsayan geohash kutularının hassas uzun dizeleri dizisi, öğelerin sırasına güvenmemelisiniz. -- \[\]- eğer boş dizi *dakika* değerleri *enlem* ve *Boylam* karşılık gelenden daha az değil *maksimum* değerler. - -Ortaya çıkan dizi 10'000' 000 ürün uzunluğundaysa, işlevin bir istisna atacağını lütfen unutmayın. - -**Örnek** - -``` sql -SELECT geohashesInBox(24.48, 40.56, 24.785, 40.81, 4) AS thasos -``` - -``` text -┌─thasos──────────────────────────────────────┐ -│ ['sx1q','sx1r','sx32','sx1w','sx1x','sx38'] │ -└─────────────────────────────────────────────┘ -``` - -## h3GetBaseCell {#h3getbasecell} - -Dizin temel hücre numarasını döndürür. - -**Sözdizimi** - -``` sql -h3GetBaseCell(index) -``` - -**Parametre** - -- `index` — Hexagon index number. Type: [Uİnt64](../../sql-reference/data-types/int-uint.md). - -**Döndürülen değerler** - -- Altıgen baz hücre numarası. Tür: [Uİnt8](../../sql-reference/data-types/int-uint.md). - -**Örnek** - -Sorgu: - -``` sql -SELECT h3GetBaseCell(612916788725809151) as basecell -``` - -Sonuç: - -``` text -┌─basecell─┐ -│ 12 │ -└──────────┘ -``` - -## h3HexAreaM2 {#h3hexaream2} - -Verilen çözünürlükte metrekare ortalama altıgen alan. - -**Sözdizimi** - -``` sql -h3HexAreaM2(resolution) -``` - -**Parametre** - -- `resolution` — Index resolution. Range: `[0, 15]`. Tür: [Uİnt8](../../sql-reference/data-types/int-uint.md). - -**Döndürülen değerler** - -- Area in m². Type: [Float64](../../sql-reference/data-types/float.md). - -**Örnek** - -Sorgu: - -``` sql -SELECT h3HexAreaM2(13) as area -``` - -Sonuç: - -``` text -┌─area─┐ -│ 43.9 │ -└──────┘ -``` - -## h3İndexesAreNeighbors {#h3indexesareneighbors} - -Sağlanan H3indexlerin komşu olup olmadığını döndürür. - -**Sözdizimi** - -``` sql -h3IndexesAreNeighbors(index1, index2) -``` - -**Parametre** - -- `index1` — Hexagon index number. Type: [Uİnt64](../../sql-reference/data-types/int-uint.md). -- `index2` — Hexagon index number. Type: [Uİnt64](../../sql-reference/data-types/int-uint.md). - -**Döndürülen değerler** - -- Dönüşler `1` dizinler komşu ise, `0` başka. Tür: [Uİnt8](../../sql-reference/data-types/int-uint.md). - -**Örnek** - -Sorgu: - -``` sql -SELECT h3IndexesAreNeighbors(617420388351344639, 617420388352655359) AS n -``` - -Sonuç: - -``` text -┌─n─┐ -│ 1 │ -└───┘ -``` - -## h3ToChildren {#h3tochildren} - -Verilen dizinin alt dizinlerini içeren bir dizi döndürür. - -**Sözdizimi** - -``` sql -h3ToChildren(index, resolution) -``` - -**Parametre** - -- `index` — Hexagon index number. Type: [Uİnt64](../../sql-reference/data-types/int-uint.md). -- `resolution` — Index resolution. Range: `[0, 15]`. Tür: [Uİnt8](../../sql-reference/data-types/int-uint.md). - -**Döndürülen değerler** - -- Alt H3 dizinleri ile dizi. Dizi türü: [Uİnt64](../../sql-reference/data-types/int-uint.md). - -**Örnek** - -Sorgu: - -``` sql -SELECT h3ToChildren(599405990164561919, 6) AS children -``` - -Sonuç: - -``` text -┌─children───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ -│ [603909588852408319,603909588986626047,603909589120843775,603909589255061503,603909589389279231,603909589523496959,603909589657714687] │ -└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ -``` - -## h3ToParent {#h3toparent} - -Verilen dizini içeren üst (kaba) dizini döndürür. - -**Sözdizimi** - -``` sql -h3ToParent(index, resolution) -``` - -**Parametre** - -- `index` — Hexagon index number. Type: [Uİnt64](../../sql-reference/data-types/int-uint.md). -- `resolution` — Index resolution. Range: `[0, 15]`. Tür: [Uİnt8](../../sql-reference/data-types/int-uint.md). - -**Döndürülen değerler** - -- Ana H3 Endeksi. Tür: [Uİnt64](../../sql-reference/data-types/int-uint.md). - -**Örnek** - -Sorgu: - -``` sql -SELECT h3ToParent(599405990164561919, 3) as parent -``` - -Sonuç: - -``` text -┌─────────────parent─┐ -│ 590398848891879423 │ -└────────────────────┘ -``` - -## h3ToString {#h3tostring} - -Dizinin H3ındex gösterimini dize gösterimine dönüştürür. - -``` sql -h3ToString(index) -``` - -**Parametre** - -- `index` — Hexagon index number. Type: [Uİnt64](../../sql-reference/data-types/int-uint.md). - -**Döndürülen değerler** - -- H3 dizininin dize gösterimi. Tür: [Dize](../../sql-reference/data-types/string.md). - -**Örnek** - -Sorgu: - -``` sql -SELECT h3ToString(617420388352917503) as h3_string -``` - -Sonuç: - -``` text -┌─h3_string───────┐ -│ 89184926cdbffff │ -└─────────────────┘ -``` - -## stringToH3 {#stringtoh3} - -Dize gösterimini H3ındex (Uİnt64) gösterimine dönüştürür. - -``` sql -stringToH3(index_str) -``` - -**Parametre** - -- `index_str` — String representation of the H3 index. Type: [Dize](../../sql-reference/data-types/string.md). - -**Döndürülen değerler** - -- Altıgen dizin numarası. Hata 0 döndürür. Tür: [Uİnt64](../../sql-reference/data-types/int-uint.md). - -**Örnek** - -Sorgu: - -``` sql -SELECT stringToH3('89184926cc3ffff') as index -``` - -Sonuç: - -``` text -┌──────────────index─┐ -│ 617420388351344639 │ -└────────────────────┘ -``` - -## h3GetResolution {#h3getresolution} - -Dizin çözünürlüğünü döndürür. - -**Sözdizimi** - -``` sql -h3GetResolution(index) -``` - -**Parametre** - -- `index` — Hexagon index number. Type: [Uİnt64](../../sql-reference/data-types/int-uint.md). - -**Döndürülen değerler** - -- Dizin çözünürlüğü. Aralık: `[0, 15]`. Tür: [Uİnt8](../../sql-reference/data-types/int-uint.md). - -**Örnek** - -Sorgu: - -``` sql -SELECT h3GetResolution(617420388352917503) as res -``` - -Sonuç: - -``` text -┌─res─┐ -│ 9 │ -└─────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/geo/) diff --git a/docs/tr/sql-reference/functions/hash-functions.md b/docs/tr/sql-reference/functions/hash-functions.md deleted file mode 100644 index 021ffa72ca1..00000000000 --- a/docs/tr/sql-reference/functions/hash-functions.md +++ /dev/null @@ -1,484 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 50 -toc_title: Karma ---- - -# Karma Fonksiyonlar {#hash-functions} - -Hash fonksiyonları elementlerin deterministik sözde rastgele karıştırma için kullanılabilir. - -## halfMD5 {#hash-functions-halfmd5} - -[Yorumluyor](../../sql-reference/functions/type-conversion-functions.md#type_conversion_functions-reinterpretAsString) tüm giriş parametrelerini dizeler olarak hesaplar ve [MD5](https://en.wikipedia.org/wiki/MD5) her biri için karma değeri. Sonra karmaları birleştirir, elde edilen dizenin karmasının ilk 8 baytını alır ve bunları şöyle yorumlar `UInt64` büyük endian bayt sırasına göre. - -``` sql -halfMD5(par1, ...) -``` - -İşlev nispeten yavaştır (işlemci çekirdeği başına saniyede 5 milyon kısa dizge). -Kullanmayı düşünün [sifash64](#hash_functions-siphash64) bunun yerine işlev. - -**Parametre** - -Fonksiyon, değişken sayıda giriş parametresi alır. Parametreler herhangi biri olabilir [desteklenen veri türleri](../../sql-reference/data-types/index.md). - -**Döndürülen Değer** - -A [Uİnt64](../../sql-reference/data-types/int-uint.md) veri türü karma değeri. - -**Örnek** - -``` sql -SELECT halfMD5(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS halfMD5hash, toTypeName(halfMD5hash) AS type -``` - -``` text -┌────────halfMD5hash─┬─type───┐ -│ 186182704141653334 │ UInt64 │ -└────────────────────┴────────┘ -``` - -## MD5 {#hash_functions-md5} - -MD5 bir dizeden hesaplar ve elde edilen bayt kümesini FixedString(16) olarak döndürür. -Özellikle MD5'E ihtiyacınız yoksa, ancak iyi bir şifreleme 128 bit karmasına ihtiyacınız varsa, ‘sipHash128’ bunun yerine işlev. -Md5sum yardımcı programı tarafından çıktı ile aynı sonucu elde etmek istiyorsanız, lower(hex(MD5(s))) kullanın. - -## sifash64 {#hash_functions-siphash64} - -64-bit üretir [Sifash](https://131002.net/siphash/) karma değeri. - -``` sql -sipHash64(par1,...) -``` - -Bu bir şifreleme karma işlevidir. En az üç kat daha hızlı çalışır [MD5](#hash_functions-md5) İşlev. - -İşlev [yorumluyor](../../sql-reference/functions/type-conversion-functions.md#type_conversion_functions-reinterpretAsString) tüm giriş parametreleri dizeleri olarak ve bunların her biri için karma değerini hesaplar. Sonra aşağıdaki algoritma ile karmaları birleştirir: - -1. Tüm giriş parametrelerini karma yaptıktan sonra, işlev karma dizisini alır. -2. Fonksiyon birinci ve ikinci öğeleri alır ve bunların dizisi için bir karma hesaplar. -3. Daha sonra işlev, önceki adımda hesaplanan karma değeri ve ilk karma dizinin üçüncü öğesini alır ve bunların dizisi için bir karma hesaplar. -4. Önceki adım, ilk karma dizinin kalan tüm öğeleri için tekrarlanır. - -**Parametre** - -Fonksiyon, değişken sayıda giriş parametresi alır. Parametreler herhangi biri olabilir [desteklenen veri türleri](../../sql-reference/data-types/index.md). - -**Döndürülen Değer** - -A [Uİnt64](../../sql-reference/data-types/int-uint.md) veri türü karma değeri. - -**Örnek** - -``` sql -SELECT sipHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS SipHash, toTypeName(SipHash) AS type -``` - -``` text -┌──────────────SipHash─┬─type───┐ -│ 13726873534472839665 │ UInt64 │ -└──────────────────────┴────────┘ -``` - -## sifash128 {#hash_functions-siphash128} - -Bir dizeden Sifash hesaplar. -Bir dize türü bağımsız değişkeni kabul eder. Fixedstring(16) Döndürür. -Sifash64'ten farklıdır, çünkü son xor katlama durumu sadece 128 bit'e kadar yapılır. - -## cityHash64 {#cityhash64} - -64-bit üretir [CityHash](https://github.com/google/cityhash) karma değeri. - -``` sql -cityHash64(par1,...) -``` - -Bu hızlı olmayan şifreleme karma işlevidir. Dize parametreleri için CityHash algoritmasını ve diğer veri türleriyle parametreler için uygulamaya özgü hızlı kriptografik olmayan karma işlevini kullanır. İşlev, nihai sonuçları almak için CityHash birleştiricisini kullanır. - -**Parametre** - -Fonksiyon, değişken sayıda giriş parametresi alır. Parametreler herhangi biri olabilir [desteklenen veri türleri](../../sql-reference/data-types/index.md). - -**Döndürülen Değer** - -A [Uİnt64](../../sql-reference/data-types/int-uint.md) veri türü karma değeri. - -**Örnekler** - -Çağrı örneği: - -``` sql -SELECT cityHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS CityHash, toTypeName(CityHash) AS type -``` - -``` text -┌─────────────CityHash─┬─type───┐ -│ 12072650598913549138 │ UInt64 │ -└──────────────────────┴────────┘ -``` - -Aşağıdaki örnek, tüm tablonun sağlama toplamının satır sırasına kadar doğrulukla nasıl hesaplanacağını gösterir: - -``` sql -SELECT groupBitXor(cityHash64(*)) FROM table -``` - -## intHash32 {#inthash32} - -Herhangi bir tamsayı türünden 32 bit karma kodu hesaplar. -Bu, sayılar için ortalama kalitenin nispeten hızlı bir kriptografik olmayan karma işlevidir. - -## intHash64 {#inthash64} - -Herhangi bir tamsayı türünden 64 bit karma kodu hesaplar. -Inthash32'den daha hızlı çalışır. Ortalama kalite. - -## SHA1 {#sha1} - -## SHA224 {#sha224} - -## SHA256 {#sha256} - -Bir dizeden SHA-1, SHA-224 veya SHA-256 hesaplar ve elde edilen bayt kümesini FixedString(20), FixedString(28) veya FixedString(32) olarak döndürür. -İşlev oldukça yavaş çalışır (SHA-1, işlemci çekirdeği başına saniyede yaklaşık 5 milyon kısa dizgiyi işler, SHA-224 ve SHA-256 ise yaklaşık 2.2 milyon işlem yapar). -Bu işlevi yalnızca belirli bir karma işleve ihtiyacınız olduğunda ve bunu seçemediğinizde kullanmanızı öneririz. -Bu gibi durumlarda bile, SELECTS'TE uygulamak yerine, tabloya eklerken işlev çevrimdışı ve ön hesaplama değerlerini uygulamanızı öneririz. - -## URLHash(url \[, N\]) {#urlhashurl-n} - -Bir tür normalleştirme kullanarak bir URL'den elde edilen bir dize için hızlı, iyi kalitede olmayan şifreleme karma işlevi. -`URLHash(s)` – Calculates a hash from a string without one of the trailing symbols `/`,`?` veya `#` sonunda, varsa. -`URLHash(s, N)` – Calculates a hash from a string up to the N level in the URL hierarchy, without one of the trailing symbols `/`,`?` veya `#` sonunda, varsa. -Düzeyleri URLHierarchy aynıdır. Bu fonksiyon (kayıt olmak için özeldir.Metrica. - -## farmHash64 {#farmhash64} - -64-bit üretir [FarmHash](https://github.com/google/farmhash) karma değeri. - -``` sql -farmHash64(par1, ...) -``` - -Fonksiyonu kullanır `Hash64` tüm yöntem [mevcut yöntemler](https://github.com/google/farmhash/blob/master/src/farmhash.h). - -**Parametre** - -Fonksiyon, değişken sayıda giriş parametresi alır. Parametreler herhangi biri olabilir [desteklenen veri türleri](../../sql-reference/data-types/index.md). - -**Döndürülen Değer** - -A [Uİnt64](../../sql-reference/data-types/int-uint.md) veri türü karma değeri. - -**Örnek** - -``` sql -SELECT farmHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS FarmHash, toTypeName(FarmHash) AS type -``` - -``` text -┌─────────────FarmHash─┬─type───┐ -│ 17790458267262532859 │ UInt64 │ -└──────────────────────┴────────┘ -``` - -## javaHash {#hash_functions-javahash} - -Hesaplıyor [JavaHash](http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add975b/src/share/classes/java/lang/String.java#l1452) bir ipten. Bu karma işlevi ne hızlı ne de iyi bir kaliteye sahip değildir. Bunu kullanmanın tek nedeni, bu algoritmanın zaten başka bir sistemde kullanılmasıdır ve tam olarak aynı sonucu hesaplamanız gerekir. - -**Sözdizimi** - -``` sql -SELECT javaHash(''); -``` - -**Döndürülen değer** - -A `Int32` veri türü karma değeri. - -**Örnek** - -Sorgu: - -``` sql -SELECT javaHash('Hello, world!'); -``` - -Sonuç: - -``` text -┌─javaHash('Hello, world!')─┐ -│ -1880044555 │ -└───────────────────────────┘ -``` - -## javaHashUTF16LE {#javahashutf16le} - -Hesaplıyor [JavaHash](http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add975b/src/share/classes/java/lang/String.java#l1452) bir dizeden, UTF-16LE kodlamasında bir dizeyi temsil eden bayt içerdiğini varsayarak. - -**Sözdizimi** - -``` sql -javaHashUTF16LE(stringUtf16le) -``` - -**Parametre** - -- `stringUtf16le` — a string in UTF-16LE encoding. - -**Döndürülen değer** - -A `Int32` veri türü karma değeri. - -**Örnek** - -UTF-16LE kodlanmış dize ile doğru sorgu. - -Sorgu: - -``` sql -SELECT javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le')) -``` - -Sonuç: - -``` text -┌─javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le'))─┐ -│ 3556498 │ -└──────────────────────────────────────────────────────────────┘ -``` - -## hiveHash {#hash-functions-hivehash} - -Hesaplıyor `HiveHash` bir ipten. - -``` sql -SELECT hiveHash(''); -``` - -Bu sadece [JavaHash](#hash_functions-javahash) sıfırlanmış işaret biti ile. Bu işlev kullanılır [Apache Kov Hanı](https://en.wikipedia.org/wiki/Apache_Hive) 3.0 öncesi sürümler için. Bu karma işlevi ne hızlı ne de iyi bir kaliteye sahip değildir. Bunu kullanmanın tek nedeni, bu algoritmanın zaten başka bir sistemde kullanılmasıdır ve tam olarak aynı sonucu hesaplamanız gerekir. - -**Döndürülen değer** - -A `Int32` veri türü karma değeri. - -Tür: `hiveHash`. - -**Örnek** - -Sorgu: - -``` sql -SELECT hiveHash('Hello, world!'); -``` - -Sonuç: - -``` text -┌─hiveHash('Hello, world!')─┐ -│ 267439093 │ -└───────────────────────────┘ -``` - -## metroHash64 {#metrohash64} - -64-bit üretir [MetroHash](http://www.jandrewrogers.com/2015/05/27/metrohash/) karma değeri. - -``` sql -metroHash64(par1, ...) -``` - -**Parametre** - -Fonksiyon, değişken sayıda giriş parametresi alır. Parametreler herhangi biri olabilir [desteklenen veri türleri](../../sql-reference/data-types/index.md). - -**Döndürülen Değer** - -A [Uİnt64](../../sql-reference/data-types/int-uint.md) veri türü karma değeri. - -**Örnek** - -``` sql -SELECT metroHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MetroHash, toTypeName(MetroHash) AS type -``` - -``` text -┌────────────MetroHash─┬─type───┐ -│ 14235658766382344533 │ UInt64 │ -└──────────────────────┴────────┘ -``` - -## jumpConsistentHash {#jumpconsistenthash} - -Bir Uint64 Formu jumpconsistenthash hesaplar. -İki bağımsız değişkeni kabul eder: bir uint64 tipi anahtar ve kova sayısı. Int32 Döndürür. -Daha fazla bilgi için bağlantıya bakın: [JumpConsistentHash](https://arxiv.org/pdf/1406.2294.pdf) - -## murmurHash2_32, murmurHash2_64 {#murmurhash2-32-murmurhash2-64} - -Üreten bir [MurmurHash2](https://github.com/aappleby/smhasher) karma değeri. - -``` sql -murmurHash2_32(par1, ...) -murmurHash2_64(par1, ...) -``` - -**Parametre** - -Her iki işlev de değişken sayıda giriş parametresi alır. Parametreler herhangi biri olabilir [desteklenen veri türleri](../../sql-reference/data-types/index.md). - -**Döndürülen Değer** - -- Bu `murmurHash2_32` fonksiyon hash değerini döndürür [Uİnt32](../../sql-reference/data-types/int-uint.md) veri türü. -- Bu `murmurHash2_64` fonksiyon hash değerini döndürür [Uİnt64](../../sql-reference/data-types/int-uint.md) veri türü. - -**Örnek** - -``` sql -SELECT murmurHash2_64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash2, toTypeName(MurmurHash2) AS type -``` - -``` text -┌──────────MurmurHash2─┬─type───┐ -│ 11832096901709403633 │ UInt64 │ -└──────────────────────┴────────┘ -``` - -## gccMurmurHash {#gccmurmurhash} - -64-bit hesaplar [MurmurHash2](https://github.com/aappleby/smhasher) aynı karma tohum kullanarak karma değeri [gcc](https://github.com/gcc-mirror/gcc/blob/41d6b10e96a1de98e90a7c0378437c3255814b16/libstdc%2B%2B-v3/include/bits/functional_hash.h#L191). CLang ve GCC yapıları arasında taşınabilir. - -**Sözdizimi** - -``` sql -gccMurmurHash(par1, ...); -``` - -**Parametre** - -- `par1, ...` — A variable number of parameters that can be any of the [desteklenen veri türleri](../../sql-reference/data-types/index.md#data_types). - -**Döndürülen değer** - -- Hesaplanan karma değeri. - -Tür: [Uİnt64](../../sql-reference/data-types/int-uint.md). - -**Örnek** - -Sorgu: - -``` sql -SELECT - gccMurmurHash(1, 2, 3) AS res1, - gccMurmurHash(('a', [1, 2, 3], 4, (4, ['foo', 'bar'], 1, (1, 2)))) AS res2 -``` - -Sonuç: - -``` text -┌─────────────────res1─┬────────────────res2─┐ -│ 12384823029245979431 │ 1188926775431157506 │ -└──────────────────────┴─────────────────────┘ -``` - -## murmurHash3_32, murmurHash3_64 {#murmurhash3-32-murmurhash3-64} - -Üreten bir [MurmurHash3](https://github.com/aappleby/smhasher) karma değeri. - -``` sql -murmurHash3_32(par1, ...) -murmurHash3_64(par1, ...) -``` - -**Parametre** - -Her iki işlev de değişken sayıda giriş parametresi alır. Parametreler herhangi biri olabilir [desteklenen veri türleri](../../sql-reference/data-types/index.md). - -**Döndürülen Değer** - -- Bu `murmurHash3_32` fonksiyon bir [Uİnt32](../../sql-reference/data-types/int-uint.md) veri türü karma değeri. -- Bu `murmurHash3_64` fonksiyon bir [Uİnt64](../../sql-reference/data-types/int-uint.md) veri türü karma değeri. - -**Örnek** - -``` sql -SELECT murmurHash3_32(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash3, toTypeName(MurmurHash3) AS type -``` - -``` text -┌─MurmurHash3─┬─type───┐ -│ 2152717 │ UInt32 │ -└─────────────┴────────┘ -``` - -## murmurHash3_128 {#murmurhash3-128} - -128-bit üretir [MurmurHash3](https://github.com/aappleby/smhasher) karma değeri. - -``` sql -murmurHash3_128( expr ) -``` - -**Parametre** - -- `expr` — [İfadeler](../syntax.md#syntax-expressions) dönen bir [Dize](../../sql-reference/data-types/string.md)- tip değeri. - -**Döndürülen Değer** - -A [FixedString (16)](../../sql-reference/data-types/fixedstring.md) veri türü karma değeri. - -**Örnek** - -``` sql -SELECT murmurHash3_128('example_string') AS MurmurHash3, toTypeName(MurmurHash3) AS type -``` - -``` text -┌─MurmurHash3──────┬─type────────────┐ -│ 6�1�4"S5KT�~~q │ FixedString(16) │ -└──────────────────┴─────────────────┘ -``` - -## xxHash32, xxHash64 {#hash-functions-xxhash32} - -Hesaplıyor `xxHash` bir ipten. İki tat, 32 ve 64 bit olarak önerilmiştir. - -``` sql -SELECT xxHash32(''); - -OR - -SELECT xxHash64(''); -``` - -**Döndürülen değer** - -A `Uint32` veya `Uint64` veri türü karma değeri. - -Tür: `xxHash`. - -**Örnek** - -Sorgu: - -``` sql -SELECT xxHash32('Hello, world!'); -``` - -Sonuç: - -``` text -┌─xxHash32('Hello, world!')─┐ -│ 834093149 │ -└───────────────────────────┘ -``` - -**Ayrıca Bakınız** - -- [xxHash](http://cyan4973.github.io/xxHash/). - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/hash_functions/) diff --git a/docs/tr/sql-reference/functions/higher-order-functions.md b/docs/tr/sql-reference/functions/higher-order-functions.md deleted file mode 100644 index 2901aa7d215..00000000000 --- a/docs/tr/sql-reference/functions/higher-order-functions.md +++ /dev/null @@ -1,264 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 57 -toc_title: "Y\xFCksek Sipari\u015F" ---- - -# Yüksek mertebeden fonksiyonlar {#higher-order-functions} - -## `->` operatör, lambda (params, expr) fonksiyonu {#operator-lambdaparams-expr-function} - -Allows describing a lambda function for passing to a higher-order function. The left side of the arrow has a formal parameter, which is any ID, or multiple formal parameters – any IDs in a tuple. The right side of the arrow has an expression that can use these formal parameters, as well as any table columns. - -Örnekler: `x -> 2 * x, str -> str != Referer.` - -Daha yüksek mertebeden işlevler yalnızca Lambda işlevlerini işlevsel argümanları olarak kabul edebilir. - -Birden çok bağımsız değişkeni kabul eden bir lambda işlevi, daha yüksek mertebeden bir işleve geçirilebilir. Bu durumda, yüksek mertebeden işlev, bu bağımsız değişkenlerin karşılık geleceği aynı uzunlukta birkaç diziden geçirilir. - -Gibi bazı işlevler için [arrayCount](#higher_order_functions-array-count) veya [arraySum](#higher_order_functions-array-count), ilk argüman (lambda işlevi) ihmal edilebilir. Bu durumda, aynı eşleme varsayılır. - -Aşağıdaki işlevler için bir lambda işlevi ihmal edilemez: - -- [arrayMap](#higher_order_functions-array-map) -- [arrayFilter](#higher_order_functions-array-filter) -- [arrayFill](#higher_order_functions-array-fill) -- [arrayReverseFill](#higher_order_functions-array-reverse-fill) -- [arraySplit](#higher_order_functions-array-split) -- [arrayReverseSplit](#higher_order_functions-array-reverse-split) -- [arrayFirst](#higher_order_functions-array-first) -- [arrayFirstİndex](#higher_order_functions-array-first-index) - -### arrayMap(func, arr1, …) {#higher_order_functions-array-map} - -Özgün uygulamadan elde edilen bir dizi döndürür `func` fonksiyon inunda her ele elementmana `arr` dizi. - -Örnekler: - -``` sql -SELECT arrayMap(x -> (x + 2), [1, 2, 3]) as res; -``` - -``` text -┌─res─────┐ -│ [3,4,5] │ -└─────────┘ -``` - -Aşağıdaki örnek, farklı dizilerden bir öğe kümesinin nasıl oluşturulacağını gösterir: - -``` sql -SELECT arrayMap((x, y) -> (x, y), [1, 2, 3], [4, 5, 6]) AS res -``` - -``` text -┌─res─────────────────┐ -│ [(1,4),(2,5),(3,6)] │ -└─────────────────────┘ -``` - -İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arrayMap` İşlev. - -### arrayFilter(func, arr1, …) {#higher_order_functions-array-filter} - -Yalnızca öğeleri içeren bir dizi döndürür `arr1` hangi için `func` 0'dan başka bir şey döndürür. - -Örnekler: - -``` sql -SELECT arrayFilter(x -> x LIKE '%World%', ['Hello', 'abc World']) AS res -``` - -``` text -┌─res───────────┐ -│ ['abc World'] │ -└───────────────┘ -``` - -``` sql -SELECT - arrayFilter( - (i, x) -> x LIKE '%World%', - arrayEnumerate(arr), - ['Hello', 'abc World'] AS arr) - AS res -``` - -``` text -┌─res─┐ -│ [2] │ -└─────┘ -``` - -İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arrayFilter` İşlev. - -### arrayFill(func, arr1, …) {#higher_order_functions-array-fill} - -Tarama yoluyla `arr1` ilk öğeden son öğeye ve değiştir `arr1[i]` tarafından `arr1[i - 1]` eğer `func` 0 döndürür. İlk eleman `arr1` değiştir notilm .eyecektir. - -Örnekler: - -``` sql -SELECT arrayFill(x -> not isNull(x), [1, null, 3, 11, 12, null, null, 5, 6, 14, null, null]) AS res -``` - -``` text -┌─res──────────────────────────────┐ -│ [1,1,3,11,12,12,12,5,6,14,14,14] │ -└──────────────────────────────────┘ -``` - -İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arrayFill` İşlev. - -### arrayReverseFill(func, arr1, …) {#higher_order_functions-array-reverse-fill} - -Tarama yoluyla `arr1` son öğeden ilk öğeye ve değiştir `arr1[i]` tarafından `arr1[i + 1]` eğer `func` 0 döndürür. The La lastst element of `arr1` değiştir notilm .eyecektir. - -Örnekler: - -``` sql -SELECT arrayReverseFill(x -> not isNull(x), [1, null, 3, 11, 12, null, null, 5, 6, 14, null, null]) AS res -``` - -``` text -┌─res────────────────────────────────┐ -│ [1,3,3,11,12,5,5,5,6,14,NULL,NULL] │ -└────────────────────────────────────┘ -``` - -İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arrayReverseFill` İşlev. - -### arraySplit(func, arr1, …) {#higher_order_functions-array-split} - -Bölme `arr1` birden fazla diziye. Ne zaman `func` 0'dan başka bir şey döndürür, dizi öğenin sol tarafında bölünecektir. Dizi ilk öğeden önce bölünmez. - -Örnekler: - -``` sql -SELECT arraySplit((x, y) -> y, [1, 2, 3, 4, 5], [1, 0, 0, 1, 0]) AS res -``` - -``` text -┌─res─────────────┐ -│ [[1,2,3],[4,5]] │ -└─────────────────┘ -``` - -İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arraySplit` İşlev. - -### arrayReverseSplit(func, arr1, …) {#higher_order_functions-array-reverse-split} - -Bölme `arr1` birden fazla diziye. Ne zaman `func` 0'dan başka bir şey döndürür, dizi öğenin sağ tarafında bölünecektir. Dizi son öğeden sonra bölünmez. - -Örnekler: - -``` sql -SELECT arrayReverseSplit((x, y) -> y, [1, 2, 3, 4, 5], [1, 0, 0, 1, 0]) AS res -``` - -``` text -┌─res───────────────┐ -│ [[1],[2,3,4],[5]] │ -└───────────────────┘ -``` - -İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arraySplit` İşlev. - -### arrayCount(\[func,\] arr1, …) {#higher_order_functions-array-count} - -Func 0'dan başka bir şey döndüren arr dizisindeki öğelerin sayısını döndürür. Eğer ‘func’ belirtilmemişse, dizideki sıfır olmayan öğelerin sayısını döndürür. - -### arrayExists(\[func,\] arr1, …) {#arrayexistsfunc-arr1} - -İçinde en az bir öğe varsa 1 değerini döndürür ‘arr’ hangi için ‘func’ 0'dan başka bir şey döndürür. Aksi takdirde, 0 döndürür. - -### arrayAll(\[func,\] arr1, …) {#arrayallfunc-arr1} - -Döner 1 Eğer ‘func’ içindeki tüm öğeler için 0'dan başka bir şey döndürür ‘arr’. Aksi takdirde, 0 döndürür. - -### arraySum(\[func,\] arr1, …) {#higher-order-functions-array-sum} - -Toplamını döndürür ‘func’ değerler. İşlev atlanırsa, sadece dizi öğelerinin toplamını döndürür. - -### arrayFirst(func, arr1, …) {#higher_order_functions-array-first} - -İlk öğeyi döndürür ‘arr1’ dizi hangi ‘func’ 0'dan başka bir şey döndürür. - -İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arrayFirst` İşlev. - -### arrayFirstIndex(func, arr1, …) {#higher_order_functions-array-first-index} - -İlk öğenin dizinini döndürür ‘arr1’ dizi hangi ‘func’ 0'dan başka bir şey döndürür. - -İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arrayFirstIndex` İşlev. - -### arrayCumSum(\[func,\] arr1, …) {#arraycumsumfunc-arr1} - -Kaynak dizideki öğelerin kısmi toplamlarının bir dizisini döndürür (çalışan bir toplam). Eğer... `func` işlev belirtilir, daha sonra dizi öğelerinin değerleri toplanmadan önce bu işlev tarafından dönüştürülür. - -Örnek: - -``` sql -SELECT arrayCumSum([1, 1, 1, 1]) AS res -``` - -``` text -┌─res──────────┐ -│ [1, 2, 3, 4] │ -└──────────────┘ -``` - -### arrayCumSumNonNegative(arr) {#arraycumsumnonnegativearr} - -Aynı olarak `arrayCumSum`, kaynak dizideki öğelerin kısmi toplamlarının bir dizisini döndürür (çalışan bir toplam). Farklı `arrayCumSum`, daha sonra döndürülen değer sıfırdan küçük bir değer içerdiğinde, değer sıfır ile değiştirilir ve sonraki hesaplama sıfır parametrelerle gerçekleştirilir. Mesela: - -``` sql -SELECT arrayCumSumNonNegative([1, 1, -4, 1]) AS res -``` - -``` text -┌─res───────┐ -│ [1,2,0,1] │ -└───────────┘ -``` - -### arraySort(\[func,\] arr1, …) {#arraysortfunc-arr1} - -Öğeleri sıralama sonucu bir dizi döndürür `arr1` artan düzende. Eğer... `func` fonksiyon belirtilir, sıralama sırası fonksiyonun sonucu ile belirlenir `func` dizi elemanlarına uygulanır (diziler) - -Bu [Schwartzian dönüşümü](https://en.wikipedia.org/wiki/Schwartzian_transform) sıralama verimliliğini artırmak için kullanılır. - -Örnek: - -``` sql -SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]); -``` - -``` text -┌─res────────────────┐ -│ ['world', 'hello'] │ -└────────────────────┘ -``` - -Hakkında daha fazla bilgi için `arraySort` yöntem, görmek [Dizilerle çalışmak için işlevler](array-functions.md#array_functions-sort) bölme. - -### arrayReverseSort(\[func,\] arr1, …) {#arrayreversesortfunc-arr1} - -Öğeleri sıralama sonucu bir dizi döndürür `arr1` azalan sırada. Eğer... `func` fonksiyon belirtilir, sıralama sırası fonksiyonun sonucu ile belirlenir `func` dizi (diziler) elemanlarına uygulanır. - -Örnek: - -``` sql -SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res; -``` - -``` text -┌─res───────────────┐ -│ ['hello','world'] │ -└───────────────────┘ -``` - -Hakkında daha fazla bilgi için `arrayReverseSort` yöntem, görmek [Dizilerle çalışmak için işlevler](array-functions.md#array_functions-reverse-sort) bölme. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/higher_order_functions/) diff --git a/docs/tr/sql-reference/functions/in-functions.md b/docs/tr/sql-reference/functions/in-functions.md deleted file mode 100644 index 18156952828..00000000000 --- a/docs/tr/sql-reference/functions/in-functions.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 60 -toc_title: "In \u0130\u015Flet theic implementingisinin uygulanmas\u0131" ---- - -# In operatörünü uygulamak için işlevler {#functions-for-implementing-the-in-operator} - -## içinde, notİn, globalİn, globalNotİn {#in-functions} - -Bölümüne bakınız [Operatör İNLERDE](../operators/in.md#select-in-operators). - -## tuple(x, y, …), operator (x, y, …) {#tuplex-y-operator-x-y} - -Birden çok sütun gruplama sağlayan bir işlev. -For columns with the types T1, T2, …, it returns a Tuple(T1, T2, …) type tuple containing these columns. There is no cost to execute the function. -Tuples normalde bir argüman için Ara değerler olarak kullanılır operatörler, veya lambda fonksiyonlarının resmi parametrelerin bir listesini oluşturmak için. Tuples bir masaya yazılamaz. - -## tupleElement (tuple, n), operatör x. N {#tupleelementtuple-n-operator-x-n} - -Bir tuple bir sütun alma sağlayan bir işlev. -‘N’ 1'den başlayarak sütun dizinidir. N sabit olmalıdır. ‘N’ bir sabit olması gerekir. ‘N’ tuple boyutundan daha büyük olmayan katı bir pozitif tamsayı olmalıdır. -İşlevi yürütmek için hiçbir maliyet yoktur. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/in_functions/) diff --git a/docs/tr/sql-reference/functions/index.md b/docs/tr/sql-reference/functions/index.md deleted file mode 100644 index 80776c9f933..00000000000 --- a/docs/tr/sql-reference/functions/index.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u0130\u015Flevler" -toc_priority: 32 -toc_title: "Giri\u015F" ---- - -# İşlevler {#functions} - -En az\* iki tür fonksiyon vardır-düzenli Fonksiyonlar (sadece denir “functions”) and aggregate functions. These are completely different concepts. Regular functions work as if they are applied to each row separately (for each row, the result of the function doesn't depend on the other rows). Aggregate functions accumulate a set of values from various rows (i.e. they depend on the entire set of rows). - -Bu bölümde düzenli işlevleri tartışıyoruz. Toplama işlevleri için bölüme bakın “Aggregate functions”. - -\* - Üçüncü bir işlev türü vardır ‘arrayJoin’ fonksiyon aittir; tablo fonksiyonları da ayrı ayrı belirtilebilir.\* - -## Güçlü Yazarak {#strong-typing} - -Standart SQL aksine, ClickHouse güçlü yazarak vardır. Başka bir deyişle, türler arasında örtük dönüşümler yapmaz. Her işlev belirli bir tür kümesi için çalışır. Bu, bazen tür dönüştürme işlevlerini kullanmanız gerektiği anlamına gelir. - -## Ortak Subexpression Eliminasyonu {#common-subexpression-elimination} - -Aynı AST (aynı kayıt veya sözdizimsel ayrıştırma aynı sonucu) olan bir sorgudaki tüm ifadeler aynı değerlere sahip olarak kabul edilir. Bu tür ifadeler bir kez birleştirilir ve yürütülür. Aynı alt sorgular da bu şekilde elimine edilir. - -## Sonuç türleri {#types-of-results} - -Tüm işlevler sonuç olarak tek bir dönüş döndürür (birkaç değer değil, sıfır değer değil). Sonuç türü genellikle değerlerle değil, yalnızca bağımsız değişken türleriyle tanımlanır. Özel durumlar tupleElement işlevi (a.n işleci) ve tofixedstring işlevidir. - -## Devamlılar {#constants} - -Basitlik için, bazı işlevler yalnızca bazı argümanlar için sabitlerle çalışabilir. Örneğin, LİKE operatörünün doğru argümanı sabit olmalıdır. -Hemen hemen tüm işlevler sabit argümanlar için bir sabit döndürür. İstisna, rasgele sayılar üreten işlevlerdir. -Bu ‘now’ işlev, farklı zamanlarda çalıştırılan sorgular için farklı değerler döndürür, ancak sonuç sabit olarak kabul edilir, çünkü sabitlik yalnızca tek bir sorguda önemlidir. -Sabit bir ifade de sabit olarak kabul edilir (örneğin, LİKE operatörünün sağ yarısı birden fazla sabitten oluşturulabilir). - -Fonksiyonlar sabit ve sabit olmayan argümanlar için farklı şekillerde uygulanabilir (farklı kod yürütülür). Ancak, bir sabit ve yalnızca aynı değeri içeren gerçek bir sütun için sonuçlar birbiriyle eşleşmelidir. - -## NULL işleme {#null-processing} - -Fonksiyonlar aşağıdaki davranışlara sahiptir: - -- İşlevin argümanlarından en az biri ise `NULL`, fonksiyon sonucu da `NULL`. -- Her işlevin açıklamasında ayrı ayrı belirtilen özel davranış. ClickHouse kaynak kodunda, bu işlevler `UseDefaultImplementationForNulls=false`. - -## Süreklilik {#constancy} - -Functions can't change the values of their arguments – any changes are returned as the result. Thus, the result of calculating separate functions does not depend on the order in which the functions are written in the query. - -## Hata İşleme {#error-handling} - -Veriler geçersizse bazı işlevler bir istisna oluşturabilir. Bu durumda, sorgu iptal edilir ve bir hata metni istemciye döndürülür. Dağıtılmış işlem için sunuculardan birinde bir özel durum oluştuğunda, diğer sunucular da sorguyu iptal etmeye çalışır. - -## Argüman ifadelerinin değerlendirilmesi {#evaluation-of-argument-expressions} - -Hemen hemen tüm programlama dillerinde, argümanlardan biri belirli operatörler için değerlendirilmeyebilir. Bu genellikle operatörler `&&`, `||`, ve `?:`. -Ancak Clickhouse'da, fonksiyonların (operatörler) argümanları her zaman değerlendirilir. Bunun nedeni, sütunların tüm bölümlerinin her satırı ayrı ayrı hesaplamak yerine bir kerede değerlendirilmesidir. - -## Dağıtılmış sorgu işleme işlevleri gerçekleştirme {#performing-functions-for-distributed-query-processing} - -Dağıtılmış sorgu işleme için, sorgu işlemenin mümkün olduğu kadar çok aşaması uzak sunucularda gerçekleştirilir ve aşamaların geri kalanı (Ara sonuçları ve bundan sonra her şeyi birleştirme) istek sahibi sunucuda gerçekleştirilir. - -Bu, işlevlerin farklı sunucularda gerçekleştirilebileceği anlamına gelir. -Örneğin, sorguda `SELECT f(sum(g(x))) FROM distributed_table GROUP BY h(y),` - -- eğer bir `distributed_table` en az iki parçaya sahiptir, fonksiyonlar ‘g’ ve ‘h’ uzak sunucularda gerçekleştirilir ve işlev ‘f’ ıstekçi sunucuda gerçekleştirilir. -- eğer bir `distributed_table` sadece bir parça var, tüm ‘f’, ‘g’, ve ‘h’ fonksiyonlar bu shard'ın sunucusunda gerçekleştirilir. - -Bir işlevin sonucu genellikle hangi sunucuda gerçekleştirildiğine bağlı değildir. Ancak, bazen bu önemlidir. -Örneğin, sözlüklerle çalışan işlevler, üzerinde çalışmakta oldukları sunucuda bulunan sözlüğü kullanır. -Başka bir örnek ise `hostName` yapmak için üzerinde çalıştığı sunucunun adını döndüren işlev `GROUP BY` sunucular tarafından bir `SELECT` sorgu. - -Eğer sorguda bir işlevi istemcisi sunucu üzerinde yapılır, ama uzak sunucularda bunu gerçekleştirmek için ihtiyacınız varsa, bir saramaz mısın ‘any’ toplama işlevi veya bir anahtara ekleyin `GROUP BY`. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/) diff --git a/docs/tr/sql-reference/functions/introspection.md b/docs/tr/sql-reference/functions/introspection.md deleted file mode 100644 index 9181463ae3f..00000000000 --- a/docs/tr/sql-reference/functions/introspection.md +++ /dev/null @@ -1,310 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 65 -toc_title: "\u0130\xE7g\xF6zlem" ---- - -# İç Gözlem Fonksiyonları {#introspection-functions} - -İç gözlem için bu bölümde açıklanan işlevleri kullanabilirsiniz [ELF](https://en.wikipedia.org/wiki/Executable_and_Linkable_Format) ve [DWARF](https://en.wikipedia.org/wiki/DWARF) sorgu profilleme için. - -!!! warning "Uyarıcı" - Bu işlevler yavaştır ve güvenlik konuları getirebilir. - -İç gözlem fonksiyonlarının düzgün çalışması için: - -- Yüklemek `clickhouse-common-static-dbg` paket. - -- Ayarla... [allow_introspection_functions](../../operations/settings/settings.md#settings-allow_introspection_functions) ayar 1. - - For security reasons introspection functions are disabled by default. - -ClickHouse için profiler raporları kaydeder [trace_log](../../operations/system-tables.md#system_tables-trace_log) sistem tablosu. Tablo ve profiler düzgün yapılandırıldığından emin olun. - -## addressToLine {#addresstoline} - -ClickHouse sunucu işleminin içindeki sanal bellek adresini dosya adına ve clickhouse kaynak kodundaki satır numarasına dönüştürür. - -Resmi ClickHouse paketleri kullanırsanız, yüklemeniz gerekir `clickhouse-common-static-dbg` paket. - -**Sözdizimi** - -``` sql -addressToLine(address_of_binary_instruction) -``` - -**Parametre** - -- `address_of_binary_instruction` ([Uİnt64](../../sql-reference/data-types/int-uint.md)) — Address of instruction in a running process. - -**Döndürülen değer** - -- Kaynak kodu dosya adı ve bu dosyadaki satır numarası iki nokta üst üste ile sınırlandırılmıştır. - - For example, `/build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:199`, where `199` is a line number. - -- Işlev hata ayıklama bilgilerini bulamadıysanız, bir ikili adı. - -- Adres geçerli değilse, boş dize. - -Tür: [Dize](../../sql-reference/data-types/string.md). - -**Örnek** - -İç gözlem işlevlerini etkinleştirme: - -``` sql -SET allow_introspection_functions=1 -``` - -İlk dizeyi seçme `trace_log` sistem tablosu: - -``` sql -SELECT * FROM system.trace_log LIMIT 1 \G -``` - -``` text -Row 1: -────── -event_date: 2019-11-19 -event_time: 2019-11-19 18:57:23 -revision: 54429 -timer_type: Real -thread_number: 48 -query_id: 421b6855-1858-45a5-8f37-f383409d6d72 -trace: [140658411141617,94784174532828,94784076370703,94784076372094,94784076361020,94784175007680,140658411116251,140658403895439] -``` - -Bu `trace` alan, örnekleme anında yığın izini içerir. - -Tek bir adres için kaynak kodu dosya adını ve satır numarasını alma: - -``` sql -SELECT addressToLine(94784076370703) \G -``` - -``` text -Row 1: -────── -addressToLine(94784076370703): /build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:199 -``` - -İşlevin tüm yığın izine uygulanması: - -``` sql -SELECT - arrayStringConcat(arrayMap(x -> addressToLine(x), trace), '\n') AS trace_source_code_lines -FROM system.trace_log -LIMIT 1 -\G -``` - -Bu [arrayMap](higher-order-functions.md#higher_order_functions-array-map) işlev, her bir elemanın işlenmesini sağlar `trace` ar arrayray by the `addressToLine` İşlev. Gördüğünüz bu işlemin sonucu `trace_source_code_lines` çıktı sütunu. - -``` text -Row 1: -────── -trace_source_code_lines: /lib/x86_64-linux-gnu/libpthread-2.27.so -/usr/lib/debug/usr/bin/clickhouse -/build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:199 -/build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.h:155 -/usr/include/c++/9/bits/atomic_base.h:551 -/usr/lib/debug/usr/bin/clickhouse -/lib/x86_64-linux-gnu/libpthread-2.27.so -/build/glibc-OTsEL5/glibc-2.27/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:97 -``` - -## addressToSymbol {#addresstosymbol} - -Clickhouse sunucu işlemi içindeki sanal bellek adresini ClickHouse nesne dosyalarından gelen simgeye dönüştürür. - -**Sözdizimi** - -``` sql -addressToSymbol(address_of_binary_instruction) -``` - -**Parametre** - -- `address_of_binary_instruction` ([Uİnt64](../../sql-reference/data-types/int-uint.md)) — Address of instruction in a running process. - -**Döndürülen değer** - -- ClickHouse nesne dosyalarından sembol. -- Adres geçerli değilse, boş dize. - -Tür: [Dize](../../sql-reference/data-types/string.md). - -**Örnek** - -İç gözlem işlevlerini etkinleştirme: - -``` sql -SET allow_introspection_functions=1 -``` - -İlk dizeyi seçme `trace_log` sistem tablosu: - -``` sql -SELECT * FROM system.trace_log LIMIT 1 \G -``` - -``` text -Row 1: -────── -event_date: 2019-11-20 -event_time: 2019-11-20 16:57:59 -revision: 54429 -timer_type: Real -thread_number: 48 -query_id: 724028bf-f550-45aa-910d-2af6212b94ac -trace: [94138803686098,94138815010911,94138815096522,94138815101224,94138815102091,94138814222988,94138806823642,94138814457211,94138806823642,94138814457211,94138806823642,94138806795179,94138806796144,94138753770094,94138753771646,94138753760572,94138852407232,140399185266395,140399178045583] -``` - -Bu `trace` alan, örnekleme anında yığın izini içerir. - -Tek bir adres için sembol alma: - -``` sql -SELECT addressToSymbol(94138803686098) \G -``` - -``` text -Row 1: -────── -addressToSymbol(94138803686098): _ZNK2DB24IAggregateFunctionHelperINS_20AggregateFunctionSumImmNS_24AggregateFunctionSumDataImEEEEE19addBatchSinglePlaceEmPcPPKNS_7IColumnEPNS_5ArenaE -``` - -İşlevin tüm yığın izine uygulanması: - -``` sql -SELECT - arrayStringConcat(arrayMap(x -> addressToSymbol(x), trace), '\n') AS trace_symbols -FROM system.trace_log -LIMIT 1 -\G -``` - -Bu [arrayMap](higher-order-functions.md#higher_order_functions-array-map) işlev, her bir elemanın işlenmesini sağlar `trace` ar arrayray by the `addressToSymbols` İşlev. Gördüğünüz bu işlemin sonucu `trace_symbols` çıktı sütunu. - -``` text -Row 1: -────── -trace_symbols: _ZNK2DB24IAggregateFunctionHelperINS_20AggregateFunctionSumImmNS_24AggregateFunctionSumDataImEEEEE19addBatchSinglePlaceEmPcPPKNS_7IColumnEPNS_5ArenaE -_ZNK2DB10Aggregator21executeWithoutKeyImplERPcmPNS0_28AggregateFunctionInstructionEPNS_5ArenaE -_ZN2DB10Aggregator14executeOnBlockESt6vectorIN3COWINS_7IColumnEE13immutable_ptrIS3_EESaIS6_EEmRNS_22AggregatedDataVariantsERS1_IPKS3_SaISC_EERS1_ISE_SaISE_EERb -_ZN2DB10Aggregator14executeOnBlockERKNS_5BlockERNS_22AggregatedDataVariantsERSt6vectorIPKNS_7IColumnESaIS9_EERS6_ISB_SaISB_EERb -_ZN2DB10Aggregator7executeERKSt10shared_ptrINS_17IBlockInputStreamEERNS_22AggregatedDataVariantsE -_ZN2DB27AggregatingBlockInputStream8readImplEv -_ZN2DB17IBlockInputStream4readEv -_ZN2DB26ExpressionBlockInputStream8readImplEv -_ZN2DB17IBlockInputStream4readEv -_ZN2DB26ExpressionBlockInputStream8readImplEv -_ZN2DB17IBlockInputStream4readEv -_ZN2DB28AsynchronousBlockInputStream9calculateEv -_ZNSt17_Function_handlerIFvvEZN2DB28AsynchronousBlockInputStream4nextEvEUlvE_E9_M_invokeERKSt9_Any_data -_ZN14ThreadPoolImplI20ThreadFromGlobalPoolE6workerESt14_List_iteratorIS0_E -_ZZN20ThreadFromGlobalPoolC4IZN14ThreadPoolImplIS_E12scheduleImplIvEET_St8functionIFvvEEiSt8optionalImEEUlvE1_JEEEOS4_DpOT0_ENKUlvE_clEv -_ZN14ThreadPoolImplISt6threadE6workerESt14_List_iteratorIS0_E -execute_native_thread_routine -start_thread -clone -``` - -## demangle {#demangle} - -Kullanarak alabileceğiniz bir sembolü dönüştürür [addressToSymbol](#addresstosymbol) C++ işlev adına işlev. - -**Sözdizimi** - -``` sql -demangle(symbol) -``` - -**Parametre** - -- `symbol` ([Dize](../../sql-reference/data-types/string.md)) — Symbol from an object file. - -**Döndürülen değer** - -- C++ işlevinin adı. -- Bir sembol geçerli değilse boş dize. - -Tür: [Dize](../../sql-reference/data-types/string.md). - -**Örnek** - -İç gözlem işlevlerini etkinleştirme: - -``` sql -SET allow_introspection_functions=1 -``` - -İlk dizeyi seçme `trace_log` sistem tablosu: - -``` sql -SELECT * FROM system.trace_log LIMIT 1 \G -``` - -``` text -Row 1: -────── -event_date: 2019-11-20 -event_time: 2019-11-20 16:57:59 -revision: 54429 -timer_type: Real -thread_number: 48 -query_id: 724028bf-f550-45aa-910d-2af6212b94ac -trace: [94138803686098,94138815010911,94138815096522,94138815101224,94138815102091,94138814222988,94138806823642,94138814457211,94138806823642,94138814457211,94138806823642,94138806795179,94138806796144,94138753770094,94138753771646,94138753760572,94138852407232,140399185266395,140399178045583] -``` - -Bu `trace` alan, örnekleme anında yığın izini içerir. - -Tek bir adres için bir işlev adı alma: - -``` sql -SELECT demangle(addressToSymbol(94138803686098)) \G -``` - -``` text -Row 1: -────── -demangle(addressToSymbol(94138803686098)): DB::IAggregateFunctionHelper > >::addBatchSinglePlace(unsigned long, char*, DB::IColumn const**, DB::Arena*) const -``` - -İşlevin tüm yığın izine uygulanması: - -``` sql -SELECT - arrayStringConcat(arrayMap(x -> demangle(addressToSymbol(x)), trace), '\n') AS trace_functions -FROM system.trace_log -LIMIT 1 -\G -``` - -Bu [arrayMap](higher-order-functions.md#higher_order_functions-array-map) işlev, her bir elemanın işlenmesini sağlar `trace` ar arrayray by the `demangle` İşlev. Gördüğünüz bu işlemin sonucu `trace_functions` çıktı sütunu. - -``` text -Row 1: -────── -trace_functions: DB::IAggregateFunctionHelper > >::addBatchSinglePlace(unsigned long, char*, DB::IColumn const**, DB::Arena*) const -DB::Aggregator::executeWithoutKeyImpl(char*&, unsigned long, DB::Aggregator::AggregateFunctionInstruction*, DB::Arena*) const -DB::Aggregator::executeOnBlock(std::vector::immutable_ptr, std::allocator::immutable_ptr > >, unsigned long, DB::AggregatedDataVariants&, std::vector >&, std::vector >, std::allocator > > >&, bool&) -DB::Aggregator::executeOnBlock(DB::Block const&, DB::AggregatedDataVariants&, std::vector >&, std::vector >, std::allocator > > >&, bool&) -DB::Aggregator::execute(std::shared_ptr const&, DB::AggregatedDataVariants&) -DB::AggregatingBlockInputStream::readImpl() -DB::IBlockInputStream::read() -DB::ExpressionBlockInputStream::readImpl() -DB::IBlockInputStream::read() -DB::ExpressionBlockInputStream::readImpl() -DB::IBlockInputStream::read() -DB::AsynchronousBlockInputStream::calculate() -std::_Function_handler::_M_invoke(std::_Any_data const&) -ThreadPoolImpl::worker(std::_List_iterator) -ThreadFromGlobalPool::ThreadFromGlobalPool::scheduleImpl(std::function, int, std::optional)::{lambda()#3}>(ThreadPoolImpl::scheduleImpl(std::function, int, std::optional)::{lambda()#3}&&)::{lambda()#1}::operator()() const -ThreadPoolImpl::worker(std::_List_iterator) -execute_native_thread_routine -start_thread -clone -``` diff --git a/docs/tr/sql-reference/functions/ip-address-functions.md b/docs/tr/sql-reference/functions/ip-address-functions.md deleted file mode 100644 index 248a87d8813..00000000000 --- a/docs/tr/sql-reference/functions/ip-address-functions.md +++ /dev/null @@ -1,248 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 55 -toc_title: "IP adresleriyle \xE7al\u0131\u015Fma" ---- - -# IP adresleriyle çalışmak için işlevler {#functions-for-working-with-ip-addresses} - -## Ipv4numtostring (num) {#ipv4numtostringnum} - -Bir Uınt32 numarası alır. Big endian'da bir IPv4 adresi olarak yorumlar. Karşılık gelen IPv4 adresini a. B. C. d biçiminde içeren bir dize döndürür (ondalık formda nokta ile ayrılmış sayılar). - -## Ipv4stringtonum (s) {#ipv4stringtonums} - -IPv4NumToString ters işlevi. IPv4 adresi geçersiz bir biçime sahipse, 0 döndürür. - -## Ipv4numtostringclassc (num) {#ipv4numtostringclasscnum} - -Ipv4numtostring'e benzer, ancak son sekizli yerine xxx kullanıyor. - -Örnek: - -``` sql -SELECT - IPv4NumToStringClassC(ClientIP) AS k, - count() AS c -FROM test.hits -GROUP BY k -ORDER BY c DESC -LIMIT 10 -``` - -``` text -┌─k──────────────┬─────c─┐ -│ 83.149.9.xxx │ 26238 │ -│ 217.118.81.xxx │ 26074 │ -│ 213.87.129.xxx │ 25481 │ -│ 83.149.8.xxx │ 24984 │ -│ 217.118.83.xxx │ 22797 │ -│ 78.25.120.xxx │ 22354 │ -│ 213.87.131.xxx │ 21285 │ -│ 78.25.121.xxx │ 20887 │ -│ 188.162.65.xxx │ 19694 │ -│ 83.149.48.xxx │ 17406 │ -└────────────────┴───────┘ -``` - -Kullanıl sincedığından beri ‘xxx’ son derece sıradışı, bu gelecekte değiştirilebilir. Bu parçanın tam biçimine güvenmemenizi öneririz. - -### Ipv6numtostring (x) {#ipv6numtostringx} - -IPv6 adresini ikili biçimde içeren bir FixedString(16) değerini kabul eder. Bu adresi metin biçiminde içeren bir dize döndürür. -IPv6 eşlemeli IPv4 adresleri ::ffff:111.222.33.44 biçiminde çıktıdır. Örnekler: - -``` sql -SELECT IPv6NumToString(toFixedString(unhex('2A0206B8000000000000000000000011'), 16)) AS addr -``` - -``` text -┌─addr─────────┐ -│ 2a02:6b8::11 │ -└──────────────┘ -``` - -``` sql -SELECT - IPv6NumToString(ClientIP6 AS k), - count() AS c -FROM hits_all -WHERE EventDate = today() AND substring(ClientIP6, 1, 12) != unhex('00000000000000000000FFFF') -GROUP BY k -ORDER BY c DESC -LIMIT 10 -``` - -``` text -┌─IPv6NumToString(ClientIP6)──────────────┬─────c─┐ -│ 2a02:2168:aaa:bbbb::2 │ 24695 │ -│ 2a02:2698:abcd:abcd:abcd:abcd:8888:5555 │ 22408 │ -│ 2a02:6b8:0:fff::ff │ 16389 │ -│ 2a01:4f8:111:6666::2 │ 16016 │ -│ 2a02:2168:888:222::1 │ 15896 │ -│ 2a01:7e00::ffff:ffff:ffff:222 │ 14774 │ -│ 2a02:8109:eee:ee:eeee:eeee:eeee:eeee │ 14443 │ -│ 2a02:810b:8888:888:8888:8888:8888:8888 │ 14345 │ -│ 2a02:6b8:0:444:4444:4444:4444:4444 │ 14279 │ -│ 2a01:7e00::ffff:ffff:ffff:ffff │ 13880 │ -└─────────────────────────────────────────┴───────┘ -``` - -``` sql -SELECT - IPv6NumToString(ClientIP6 AS k), - count() AS c -FROM hits_all -WHERE EventDate = today() -GROUP BY k -ORDER BY c DESC -LIMIT 10 -``` - -``` text -┌─IPv6NumToString(ClientIP6)─┬──────c─┐ -│ ::ffff:94.26.111.111 │ 747440 │ -│ ::ffff:37.143.222.4 │ 529483 │ -│ ::ffff:5.166.111.99 │ 317707 │ -│ ::ffff:46.38.11.77 │ 263086 │ -│ ::ffff:79.105.111.111 │ 186611 │ -│ ::ffff:93.92.111.88 │ 176773 │ -│ ::ffff:84.53.111.33 │ 158709 │ -│ ::ffff:217.118.11.22 │ 154004 │ -│ ::ffff:217.118.11.33 │ 148449 │ -│ ::ffff:217.118.11.44 │ 148243 │ -└────────────────────────────┴────────┘ -``` - -## Ipv6stringtonum (s) {#ipv6stringtonums} - -IPv6NumToString ters işlevi. IPv6 adresi geçersiz bir biçime sahipse, bir boş bayt dizesi döndürür. -HEX büyük veya küçük harf olabilir. - -## Ipv4toıpv6 (x) {#ipv4toipv6x} - -Alır bir `UInt32` numara. Bir IPv4 adresi olarak yorumlar [büyük endian](https://en.wikipedia.org/wiki/Endianness). Ret aur ANS a `FixedString(16)` IPv6 adresini ikili biçimde içeren değer. Örnekler: - -``` sql -SELECT IPv6NumToString(IPv4ToIPv6(IPv4StringToNum('192.168.0.1'))) AS addr -``` - -``` text -┌─addr───────────────┐ -│ ::ffff:192.168.0.1 │ -└────────────────────┘ -``` - -## cutİPv6 (x, bytesToCutForİPv6, bytesToCutForİPv4) {#cutipv6x-bytestocutforipv6-bytestocutforipv4} - -IPv6 adresini ikili biçimde içeren bir FixedString(16) değerini kabul eder. Metin biçiminde kaldırılan belirtilen bayt sayısının adresini içeren bir dize döndürür. Mesela: - -``` sql -WITH - IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D') AS ipv6, - IPv4ToIPv6(IPv4StringToNum('192.168.0.1')) AS ipv4 -SELECT - cutIPv6(ipv6, 2, 0), - cutIPv6(ipv4, 0, 2) -``` - -``` text -┌─cutIPv6(ipv6, 2, 0)─────────────────┬─cutIPv6(ipv4, 0, 2)─┐ -│ 2001:db8:ac10:fe01:feed:babe:cafe:0 │ ::ffff:192.168.0.0 │ -└─────────────────────────────────────┴─────────────────────┘ -``` - -## Ipv4cidrtorange(ıpv4, Cıdr), {#ipv4cidrtorangeipv4-cidr} - -İçeren bir IPv4 ve bir Uint8 değerini kabul eder [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing). Alt ağın alt aralığını ve daha yüksek aralığını içeren iki IPv4 içeren bir tuple döndürür. - -``` sql -SELECT IPv4CIDRToRange(toIPv4('192.168.5.2'), 16) -``` - -``` text -┌─IPv4CIDRToRange(toIPv4('192.168.5.2'), 16)─┐ -│ ('192.168.0.0','192.168.255.255') │ -└────────────────────────────────────────────┘ -``` - -## Ipv6cidrtorange(ıpv6, Cıdr), {#ipv6cidrtorangeipv6-cidr} - -CIDR'Yİ içeren bir IPv6 ve bir Uİnt8 değerini kabul eder. Alt ağın alt aralığını ve daha yüksek aralığını içeren iki IPv6 içeren bir tuple döndürür. - -``` sql -SELECT IPv6CIDRToRange(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32); -``` - -``` text -┌─IPv6CIDRToRange(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32)─┐ -│ ('2001:db8::','2001:db8:ffff:ffff:ffff:ffff:ffff:ffff') │ -└────────────────────────────────────────────────────────────────────────┘ -``` - -## toıpv4 (dize) {#toipv4string} - -İçin bir takma ad `IPv4StringToNum()` bu, IPv4 adresinin bir dize formunu alır ve değerini döndürür [Ipv44](../../sql-reference/data-types/domains/ipv4.md) tarafından döndürülen değere eşit ikili olan tür `IPv4StringToNum()`. - -``` sql -WITH - '171.225.130.45' as IPv4_string -SELECT - toTypeName(IPv4StringToNum(IPv4_string)), - toTypeName(toIPv4(IPv4_string)) -``` - -``` text -┌─toTypeName(IPv4StringToNum(IPv4_string))─┬─toTypeName(toIPv4(IPv4_string))─┐ -│ UInt32 │ IPv4 │ -└──────────────────────────────────────────┴─────────────────────────────────┘ -``` - -``` sql -WITH - '171.225.130.45' as IPv4_string -SELECT - hex(IPv4StringToNum(IPv4_string)), - hex(toIPv4(IPv4_string)) -``` - -``` text -┌─hex(IPv4StringToNum(IPv4_string))─┬─hex(toIPv4(IPv4_string))─┐ -│ ABE1822D │ ABE1822D │ -└───────────────────────────────────┴──────────────────────────┘ -``` - -## toıpv6 (dize) {#toipv6string} - -İçin bir takma ad `IPv6StringToNum()` bu, IPv6 adresinin bir dize formunu alır ve değerini döndürür [IPv6](../../sql-reference/data-types/domains/ipv6.md) tarafından döndürülen değere eşit ikili olan tür `IPv6StringToNum()`. - -``` sql -WITH - '2001:438:ffff::407d:1bc1' as IPv6_string -SELECT - toTypeName(IPv6StringToNum(IPv6_string)), - toTypeName(toIPv6(IPv6_string)) -``` - -``` text -┌─toTypeName(IPv6StringToNum(IPv6_string))─┬─toTypeName(toIPv6(IPv6_string))─┐ -│ FixedString(16) │ IPv6 │ -└──────────────────────────────────────────┴─────────────────────────────────┘ -``` - -``` sql -WITH - '2001:438:ffff::407d:1bc1' as IPv6_string -SELECT - hex(IPv6StringToNum(IPv6_string)), - hex(toIPv6(IPv6_string)) -``` - -``` text -┌─hex(IPv6StringToNum(IPv6_string))─┬─hex(toIPv6(IPv6_string))─────────┐ -│ 20010438FFFF000000000000407D1BC1 │ 20010438FFFF000000000000407D1BC1 │ -└───────────────────────────────────┴──────────────────────────────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/ip_address_functions/) diff --git a/docs/tr/sql-reference/functions/json-functions.md b/docs/tr/sql-reference/functions/json-functions.md deleted file mode 100644 index 36e741988e3..00000000000 --- a/docs/tr/sql-reference/functions/json-functions.md +++ /dev/null @@ -1,297 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 56 -toc_title: "Json ile \xE7al\u0131\u015Fma" ---- - -# Json ile çalışmak için fonksiyonlar {#functions-for-working-with-json} - -Üye Olarak.Metrica, JSON kullanıcılar tarafından oturum parametreleri olarak iletilir. Bu JSON ile çalışmak için bazı özel fonksiyonlar var. (Çoğu durumda, JSONs ek olarak önceden işlenir ve elde edilen değerler işlenmiş biçimlerinde ayrı sütunlara konur .) Tüm bu işlevler, JSON'UN ne olabileceğine dair güçlü varsayımlara dayanır, ancak işi yapmak için mümkün olduğunca az şey yapmaya çalışırlar. - -Aşağıdaki varsayımlar yapılır: - -1. Alan adı (işlev bağımsız değişkeni) sabit olmalıdır. -2. Alan adı bir şekilde json'da kanonik olarak kodlanmıştır. Mesela: `visitParamHas('{"abc":"def"}', 'abc') = 1`, ama `visitParamHas('{"\\u0061\\u0062\\u0063":"def"}', 'abc') = 0` -3. Alanlar, herhangi bir yuvalama düzeyinde, ayrım gözetmeksizin aranır. Birden çok eşleşen alan varsa, ilk olay kullanılır. -4. JSON, dize değişmezleri dışında boşluk karakterlerine sahip değildir. - -## visitParamHas (params, isim) {#visitparamhasparams-name} - -İle bir alan olup olmadığını denetler ‘name’ ad. - -## visitParamExtractUİnt (params, isim) {#visitparamextractuintparams-name} - -Uint64 adlı alanın değerinden ayrıştırır ‘name’. Bu bir dize alanı ise, dizenin başlangıcından itibaren bir sayıyı ayrıştırmaya çalışır. Alan yoksa veya varsa ancak bir sayı içermiyorsa, 0 döndürür. - -## visitParamExtractİnt (params, isim) {#visitparamextractintparams-name} - -Int64 için olduğu gibi. - -## visitParamExtractFloat (params, isim) {#visitparamextractfloatparams-name} - -Float64 için olduğu gibi. - -## visitParamExtractBool (params, isim) {#visitparamextractboolparams-name} - -True/false değerini ayrıştırır. Sonuç Uİnt8. - -## visitParamExtractRaw (params, isim) {#visitparamextractrawparams-name} - -Ayırıcılar da dahil olmak üzere bir alanın değerini döndürür. - -Örnekler: - -``` sql -visitParamExtractRaw('{"abc":"\\n\\u0000"}', 'abc') = '"\\n\\u0000"' -visitParamExtractRaw('{"abc":{"def":[1,2,3]}}', 'abc') = '{"def":[1,2,3]}' -``` - -## visitParamExtractString (params, isim) {#visitparamextractstringparams-name} - -Dizeyi çift tırnak içinde ayrıştırır. Değeri unescaped. Unescaping başarısız olursa, boş bir dize döndürür. - -Örnekler: - -``` sql -visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0' -visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺' -visitParamExtractString('{"abc":"\\u263"}', 'abc') = '' -visitParamExtractString('{"abc":"hello}', 'abc') = '' -``` - -Şu anda biçimdeki kod noktaları için destek yok `\uXXXX\uYYYY` bu temel çok dilli düzlemden değildir(UTF-8 yerine CESU-8'e dönüştürülürler). - -Aşağıdaki işlevler dayanmaktadır [simdjson](https://github.com/lemire/simdjson) daha karmaşık json ayrıştırma gereksinimleri için tasarlanmıştır. Yukarıda belirtilen varsayım 2 hala geçerlidir. - -## ısvalidjson(json) {#isvalidjsonjson} - -Dize geçirilen kontroller geçerli bir json'dur. - -Örnekler: - -``` sql -SELECT isValidJSON('{"a": "hello", "b": [-100, 200.0, 300]}') = 1 -SELECT isValidJSON('not a json') = 0 -``` - -## JSONHas(json\[, indices_or_keys\]…) {#jsonhasjson-indices-or-keys} - -Değer JSON belgesinde varsa, `1` iade edilecektir. - -Değer yoksa, `0` iade edilecektir. - -Örnekler: - -``` sql -SELECT JSONHas('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = 1 -SELECT JSONHas('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 4) = 0 -``` - -`indices_or_keys` sıfır veya daha fazla argüman listesi her biri dize veya tamsayı olabilir. - -- String = nesne üyesine anahtarla erişin. -- Pozitif tamsayı = n-inci üyesine / anahtarına baştan erişin. -- Negatif tamsayı = sondan n-inci üye/anahtara erişin. - -Elemanın minimum Endeksi 1'dir. Böylece 0 öğesi mevcut değildir. - -Hem json dizilerine hem de JSON nesnelerine erişmek için tamsayılar kullanabilirsiniz. - -Bu yüzden, örneğin : - -``` sql -SELECT JSONExtractKey('{"a": "hello", "b": [-100, 200.0, 300]}', 1) = 'a' -SELECT JSONExtractKey('{"a": "hello", "b": [-100, 200.0, 300]}', 2) = 'b' -SELECT JSONExtractKey('{"a": "hello", "b": [-100, 200.0, 300]}', -1) = 'b' -SELECT JSONExtractKey('{"a": "hello", "b": [-100, 200.0, 300]}', -2) = 'a' -SELECT JSONExtractString('{"a": "hello", "b": [-100, 200.0, 300]}', 1) = 'hello' -``` - -## JSONLength(json\[, indices_or_keys\]…) {#jsonlengthjson-indices-or-keys} - -Bir json dizisinin veya bir JSON nesnesinin uzunluğunu döndürür. - -Değer yoksa veya yanlış bir türe sahipse, `0` iade edilecektir. - -Örnekler: - -``` sql -SELECT JSONLength('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = 3 -SELECT JSONLength('{"a": "hello", "b": [-100, 200.0, 300]}') = 2 -``` - -## JSONType(json\[, indices_or_keys\]…) {#jsontypejson-indices-or-keys} - -Bir JSON değerinin türünü döndürür. - -Değer yoksa, `Null` iade edilecektir. - -Örnekler: - -``` sql -SELECT JSONType('{"a": "hello", "b": [-100, 200.0, 300]}') = 'Object' -SELECT JSONType('{"a": "hello", "b": [-100, 200.0, 300]}', 'a') = 'String' -SELECT JSONType('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = 'Array' -``` - -## JSONExtractUInt(json\[, indices_or_keys\]…) {#jsonextractuintjson-indices-or-keys} - -## JSONExtractInt(json\[, indices_or_keys\]…) {#jsonextractintjson-indices-or-keys} - -## JSONExtractFloat(json\[, indices_or_keys\]…) {#jsonextractfloatjson-indices-or-keys} - -## JSONExtractBool(json\[, indices_or_keys\]…) {#jsonextractbooljson-indices-or-keys} - -Bir JSON ayrıştırır ve bir değer ayıklayın. Bu işlevler benzer `visitParam` işlevler. - -Değer yoksa veya yanlış bir türe sahipse, `0` iade edilecektir. - -Örnekler: - -``` sql -SELECT JSONExtractInt('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 1) = -100 -SELECT JSONExtractFloat('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 2) = 200.0 -SELECT JSONExtractUInt('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', -1) = 300 -``` - -## JSONExtractString(json\[, indices_or_keys\]…) {#jsonextractstringjson-indices-or-keys} - -Bir json ayrıştırır ve bir dize ayıklayın. Bu işlev benzer `visitParamExtractString` işlevler. - -Değer yoksa veya yanlış bir tür varsa, boş bir dize döndürülür. - -Değeri unescaped. Unescaping başarısız olursa, boş bir dize döndürür. - -Örnekler: - -``` sql -SELECT JSONExtractString('{"a": "hello", "b": [-100, 200.0, 300]}', 'a') = 'hello' -SELECT JSONExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0' -SELECT JSONExtractString('{"abc":"\\u263a"}', 'abc') = '☺' -SELECT JSONExtractString('{"abc":"\\u263"}', 'abc') = '' -SELECT JSONExtractString('{"abc":"hello}', 'abc') = '' -``` - -## JSONExtract(json\[, indices_or_keys…\], Return_type) {#jsonextractjson-indices-or-keys-return-type} - -Bir Json ayrıştırır ve verilen ClickHouse veri türünün bir değerini çıkarır. - -Bu, önceki bir genellemedir `JSONExtract` işlevler. -Bu demektir -`JSONExtract(..., 'String')` tam olarak aynı döndürür `JSONExtractString()`, -`JSONExtract(..., 'Float64')` tam olarak aynı döndürür `JSONExtractFloat()`. - -Örnekler: - -``` sql -SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'Tuple(String, Array(Float64))') = ('hello',[-100,200,300]) -SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'Tuple(b Array(Float64), a String)') = ([-100,200,300],'hello') -SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 'Array(Nullable(Int8))') = [-100, NULL, NULL] -SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 4, 'Nullable(Int64)') = NULL -SELECT JSONExtract('{"passed": true}', 'passed', 'UInt8') = 1 -SELECT JSONExtract('{"day": "Thursday"}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \'Tuesday\' = 2, \'Wednesday\' = 3, \'Thursday\' = 4, \'Friday\' = 5, \'Saturday\' = 6)') = 'Thursday' -SELECT JSONExtract('{"day": 5}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \'Tuesday\' = 2, \'Wednesday\' = 3, \'Thursday\' = 4, \'Friday\' = 5, \'Saturday\' = 6)') = 'Friday' -``` - -## JSONExtractKeysAndValues(json\[, indices_or_keys…\], Value_type) {#jsonextractkeysandvaluesjson-indices-or-keys-value-type} - -Anahtar değer çiftlerini, değerlerin verilen ClickHouse veri türünde olduğu bir JSON'DAN ayrıştırır. - -Örnek: - -``` sql -SELECT JSONExtractKeysAndValues('{"x": {"a": 5, "b": 7, "c": 11}}', 'x', 'Int8') = [('a',5),('b',7),('c',11)] -``` - -## JSONExtractRaw(json\[, indices_or_keys\]…) {#jsonextractrawjson-indices-or-keys} - -Json'un bir bölümünü ayrıştırılmamış dize olarak döndürür. - -Bölüm yoksa veya yanlış bir türe sahipse, boş bir dize döndürülür. - -Örnek: - -``` sql -SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = '[-100, 200.0, 300]' -``` - -## JSONExtractArrayRaw(json\[, indices_or_keys…\]) {#jsonextractarrayrawjson-indices-or-keys} - -Her biri ayrıştırılmamış dize olarak temsil edilen json dizisinin öğeleriyle bir dizi döndürür. - -Bölüm yoksa veya dizi değilse, boş bir dizi döndürülür. - -Örnek: - -``` sql -SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"']' -``` - -## JSONExtractKeysAndValuesRaw {#json-extract-keys-and-values-raw} - -Bir json nesnesinden ham verileri ayıklar. - -**Sözdizimi** - -``` sql -JSONExtractKeysAndValuesRaw(json[, p, a, t, h]) -``` - -**Parametre** - -- `json` — [Dize](../data-types/string.md) geçerli JSON ile. -- `p, a, t, h` — Comma-separated indices or keys that specify the path to the inner field in a nested JSON object. Each argument can be either a [dize](../data-types/string.md) anahtar veya bir tarafından alan almak için [tamsayı](../data-types/int-uint.md) N-inci alanını almak için (1'den endeksli, negatif tamsayılar sondan sayılır). Ayarlanmazsa, tüm JSON üst düzey nesne olarak ayrıştırılır. İsteğe bağlı parametre. - -**Döndürülen değerler** - -- İle dizi `('key', 'value')` Demetler. Her iki tuple üyeleri dizeleri vardır. -- İstenen nesne yoksa veya giriş json geçersiz ise boş dizi. - -Tür: [Dizi](../data-types/array.md)([Demet](../data-types/tuple.md)([Dize](../data-types/string.md), [Dize](../data-types/string.md)). - -**Örnekler** - -Sorgu: - -``` sql -SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}') -``` - -Sonuç: - -``` text -┌─JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}')─┐ -│ [('a','[-100,200]'),('b','{"c":{"d":"hello","f":"world"}}')] │ -└──────────────────────────────────────────────────────────────────────────────────────────────┘ -``` - -Sorgu: - -``` sql -SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', 'b') -``` - -Sonuç: - -``` text -┌─JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', 'b')─┐ -│ [('c','{"d":"hello","f":"world"}')] │ -└───────────────────────────────────────────────────────────────────────────────────────────────────┘ -``` - -Sorgu: - -``` sql -SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', -1, 'c') -``` - -Sonuç: - -``` text -┌─JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', -1, 'c')─┐ -│ [('d','"hello"'),('f','"world"')] │ -└───────────────────────────────────────────────────────────────────────────────────────────────────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/json_functions/) diff --git a/docs/tr/sql-reference/functions/logical-functions.md b/docs/tr/sql-reference/functions/logical-functions.md deleted file mode 100644 index 526c5f4584e..00000000000 --- a/docs/tr/sql-reference/functions/logical-functions.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 37 -toc_title: "Mant\u0131kl\u0131" ---- - -# Mantıksal Fonksiyonlar {#logical-functions} - -Mantıksal işlevler herhangi bir sayısal türü kabul eder, ancak 0 veya 1'e eşit bir Uİnt8 numarası döndürür. - -Bir argüman olarak sıfır kabul edilir “false,” sıfır olmayan herhangi bir değer dikkate alınırken “true”. - -## ve, ve operatör {#and-and-operator} - -## or, or operat ,or {#or-or-operator} - -## değil, operatör değil {#not-not-operator} - -## xor {#xor} - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/logical_functions/) diff --git a/docs/tr/sql-reference/functions/machine-learning-functions.md b/docs/tr/sql-reference/functions/machine-learning-functions.md deleted file mode 100644 index 93aaaee1340..00000000000 --- a/docs/tr/sql-reference/functions/machine-learning-functions.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 64 -toc_title: "Makine \xD6\u011Frenme Fonksiyonlar\u0131" ---- - -# Makine Öğrenme Fonksiyonları {#machine-learning-functions} - -## evalMLMethod (tahmin) {#machine_learning_methods-evalmlmethod} - -Tak fittedılmış regresyon model usinglerini kullanarak tahmin `evalMLMethod` İşlev. Lin seeke bakınız `linearRegression`. - -### Stokastik Doğrusal Regresyon {#stochastic-linear-regression} - -Bu [stokastiklinearregression](../../sql-reference/aggregate-functions/reference.md#agg_functions-stochasticlinearregression) toplama fonksiyonu, doğrusal model ve MSE kayıp fonksiyonunu kullanarak stokastik Gradyan iniş yöntemini uygular. Kullanma `evalMLMethod` yeni veri üzerinde tahmin etmek için. - -### Stokastik Lojistik Regresyon {#stochastic-logistic-regression} - -Bu [stochasticLogisticRegression](../../sql-reference/aggregate-functions/reference.md#agg_functions-stochasticlogisticregression) toplama işlevi, ikili sınıflandırma problemi için stokastik Gradyan iniş yöntemini uygular. Kullanma `evalMLMethod` yeni veri üzerinde tahmin etmek için. diff --git a/docs/tr/sql-reference/functions/math-functions.md b/docs/tr/sql-reference/functions/math-functions.md deleted file mode 100644 index 7d84b8ac741..00000000000 --- a/docs/tr/sql-reference/functions/math-functions.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 44 -toc_title: Matematiksel ---- - -# Matematiksel Fonksiyonlar {#mathematical-functions} - -Tüm işlevler bir Float64 numarası döndürür. Sonucun doğruluğu mümkün olan en yüksek hassasiyete yakındır, ancak sonuç, ilgili gerçek sayıya en yakın makine temsil edilebilir numarası ile çakışmayabilir. - -## e() {#e} - -E numarasına yakın bir Float64 numarası döndürür. - -## pi sayısı() {#pi} - -Returns a Float64 number that is close to the number π. - -## exp(x) {#expx} - -Sayısal bir bağımsız değişken kabul eder ve bir Float64 sayı argümanın üs yakın döndürür. - -## log (x), L (n(x) {#logx-lnx} - -Sayısal bir bağımsız değişken kabul eder ve bağımsız değişken doğal logaritma yakın bir Float64 sayı döndürür. - -## exp2 (x) {#exp2x} - -Sayısal bir bağımsız değişkeni kabul eder ve X gücüne 2'ye yakın bir Float64 numarası döndürür. - -## log2 (x) {#log2x} - -Sayısal bir bağımsız değişken kabul eder ve değişken ikili logaritma yakın bir Float64 sayı döndürür. - -## exp10 (x) {#exp10x} - -Sayısal bir bağımsız değişkeni kabul eder ve 10'a yakın Float64 numarasını x gücüne döndürür. - -## log10(x) {#log10x} - -Sayısal bir bağımsız değişken kabul eder ve bir float64 sayı bağımsız değişken ondalık logaritması yakın döndürür. - -## sqrt(x) {#sqrtx} - -Sayısal bir bağımsız değişken kabul eder ve bağımsız değişken kareköküne yakın bir Float64 numarası döndürür. - -## TCMB (x) {#cbrtx} - -Sayısal bir bağımsız değişkeni kabul eder ve bağımsız değişken kübik köküne yakın bir Float64 numarası döndürür. - -## erf (x) {#erfx} - -Eğer ‘x’ negatif değil, o zaman `erf(x / σ√2)` standart sapma ile normal dağılıma sahip bir rasgele değişkenin olasılığı var mı ‘σ’ beklenen değerden daha fazla ayrılan değeri alır ‘x’. - -Örnek (üç sigma kuralı): - -``` sql -SELECT erf(3 / sqrt(2)) -``` - -``` text -┌─erf(divide(3, sqrt(2)))─┐ -│ 0.9973002039367398 │ -└─────────────────────────┘ -``` - -## erfc (x) {#erfcx} - -Sayısal bir bağımsız değişkeni kabul eder ve 1 - erf(x) yakın bir Float64 numarası döndürür, ancak büyük için hassasiyet kaybı olmadan ‘x’ değerler. - -## lgamma (x) {#lgammax} - -Gama fonksiyonunun logaritması. - -## tgamma (x) {#tgammax} - -Gama fonksiyonu. - -## günah(x) {#sinx} - -Sinüs. - -## C (os (x) {#cosx} - -Kosinüs. - -## tan (x) {#tanx} - -Teğet. - -## asin (x) {#asinx} - -Ark sinüsü. - -## acos (x) {#acosx} - -Ark kosinüsü. - -## atan (x) {#atanx} - -Ark teğet. - -## pow (x, y), güç (x, y)) {#powx-y-powerx-y} - -İki sayısal bağımsız değişken X ve y alır.X'e yakın bir Float64 numarasını y gücüne döndürür. - -## ıntexp2 {#intexp2} - -Sayısal bir bağımsız değişkeni kabul eder ve X'in gücüne 2'ye yakın bir uint64 numarası döndürür. - -## ıntexp10 {#intexp10} - -Sayısal bir bağımsız değişkeni kabul eder ve X gücüne 10'a yakın bir uint64 numarası döndürür. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/math_functions/) diff --git a/docs/tr/sql-reference/functions/other-functions.md b/docs/tr/sql-reference/functions/other-functions.md deleted file mode 100644 index bf055abfa45..00000000000 --- a/docs/tr/sql-reference/functions/other-functions.md +++ /dev/null @@ -1,1205 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 66 -toc_title: "Di\u011Fer" ---- - -# Diğer Fonksiyonlar {#other-functions} - -## hostnamename() {#hostname} - -Bu işlevin gerçekleştirildiği ana bilgisayarın adını içeren bir dize döndürür. Dağıtılmış işlem için, bu işlev uzak bir sunucuda gerçekleştirilirse, uzak sunucu ana bilgisayarının adıdır. - -## getMacro {#getmacro} - -Get as a nam AED value from the [makrolar](../../operations/server-configuration-parameters/settings.md#macros) sunucu yapılandırması bölümü. - -**Sözdizimi** - -``` sql -getMacro(name); -``` - -**Parametre** - -- `name` — Name to retrieve from the `macros` bölme. [Dize](../../sql-reference/data-types/string.md#string). - -**Döndürülen değer** - -- Belirtilen makro değeri. - -Tür: [Dize](../../sql-reference/data-types/string.md). - -**Örnek** - -Örnek `macros` sunucu yapılandırma dosyasındaki bölüm: - -``` xml - - Value - -``` - -Sorgu: - -``` sql -SELECT getMacro('test'); -``` - -Sonuç: - -``` text -┌─getMacro('test')─┐ -│ Value │ -└──────────────────┘ -``` - -Aynı değeri elde etmenin alternatif bir yolu: - -``` sql -SELECT * FROM system.macros -WHERE macro = 'test'; -``` - -``` text -┌─macro─┬─substitution─┐ -│ test │ Value │ -└───────┴──────────────┘ -``` - -## FQDN {#fqdn} - -Tam etki alanı adını döndürür. - -**Sözdizimi** - -``` sql -fqdn(); -``` - -Bu işlev büyük / küçük harf duyarsızdır. - -**Döndürülen değer** - -- Tam etki alanı adı ile dize. - -Tür: `String`. - -**Örnek** - -Sorgu: - -``` sql -SELECT FQDN(); -``` - -Sonuç: - -``` text -┌─FQDN()──────────────────────────┐ -│ clickhouse.ru-central1.internal │ -└─────────────────────────────────┘ -``` - -## basename {#basename} - -Son eğik çizgi veya ters eğik çizgiden sonra bir dizenin sondaki kısmını ayıklar. Bu işlev, genellikle bir yoldan dosya adını ayıklamak için kullanılır. - -``` sql -basename( expr ) -``` - -**Parametre** - -- `expr` — Expression resulting in a [Dize](../../sql-reference/data-types/string.md) type value. Tüm ters eğik çizgilerin ortaya çıkan değerden kaçması gerekir. - -**Döndürülen Değer** - -İçeren bir dize: - -- Son eğik çizgi veya ters eğik çizgiden sonra bir dizenin sondaki kısmı. - - If the input string contains a path ending with slash or backslash, for example, `/` or `c:\`, the function returns an empty string. - -- Eğik çizgi veya ters eğik çizgi yoksa orijinal dize. - -**Örnek** - -``` sql -SELECT 'some/long/path/to/file' AS a, basename(a) -``` - -``` text -┌─a──────────────────────┬─basename('some\\long\\path\\to\\file')─┐ -│ some\long\path\to\file │ file │ -└────────────────────────┴────────────────────────────────────────┘ -``` - -``` sql -SELECT 'some\\long\\path\\to\\file' AS a, basename(a) -``` - -``` text -┌─a──────────────────────┬─basename('some\\long\\path\\to\\file')─┐ -│ some\long\path\to\file │ file │ -└────────────────────────┴────────────────────────────────────────┘ -``` - -``` sql -SELECT 'some-file-name' AS a, basename(a) -``` - -``` text -┌─a──────────────┬─basename('some-file-name')─┐ -│ some-file-name │ some-file-name │ -└────────────────┴────────────────────────────┘ -``` - -## visibleWidth(x) {#visiblewidthx} - -Değerleri konsola metin biçiminde (sekmeyle ayrılmış) çıkarırken yaklaşık genişliği hesaplar. -Bu işlev, sistem tarafından güzel formatların uygulanması için kullanılır. - -`NULL` karşılık gelen bir dize olarak temsil edilir `NULL` içinde `Pretty` biçimliler. - -``` sql -SELECT visibleWidth(NULL) -``` - -``` text -┌─visibleWidth(NULL)─┐ -│ 4 │ -└────────────────────┘ -``` - -## toTypeName (x) {#totypenamex} - -Geçirilen bağımsız değişken türü adını içeren bir dize döndürür. - -Eğer `NULL` fonksiyona girdi olarak geçirilir, daha sonra `Nullable(Nothing)` bir iç karşılık gelen türü `NULL` Clickhouse'da temsil. - -## blockSize() {#function-blocksize} - -Bloğun boyutunu alır. -Clickhouse'da, sorgular her zaman bloklarda (sütun parçaları kümeleri) çalıştırılır. Bu işlev, aradığınız bloğun boyutunu almanızı sağlar. - -## materialize (x) {#materializex} - -Bir sabiti yalnızca bir değer içeren tam bir sütuna dönüştürür. -Clickhouse'da, tam sütunlar ve sabitler bellekte farklı şekilde temsil edilir. İşlevler, sabit argümanlar ve normal argümanlar için farklı şekilde çalışır (farklı kod yürütülür), ancak sonuç hemen hemen her zaman aynıdır. Bu işlev, bu davranış hata ayıklama içindir. - -## ignore(…) {#ignore} - -Dahil olmak üzere herhangi bir argümanı kabul eder `NULL`. Her zaman 0 döndürür. -Ancak, argüman hala değerlendirilir. Bu kriterler için kullanılabilir. - -## uyku (saniye) {#sleepseconds} - -Uykular ‘seconds’ her veri bloğunda saniye. Bir tamsayı veya kayan noktalı sayı belirtebilirsiniz. - -## sleepEachRow (saniye) {#sleepeachrowseconds} - -Uykular ‘seconds’ her satırda saniye. Bir tamsayı veya kayan noktalı sayı belirtebilirsiniz. - -## currentDatabase() {#currentdatabase} - -Geçerli veritabanının adını döndürür. -Bu işlevi, veritabanını belirtmeniz gereken bir tablo oluştur sorgusunda tablo altyapısı parametrelerinde kullanabilirsiniz. - -## currentUser() {#other-function-currentuser} - -Geçerli kullanıcının oturum açma döndürür. Kullanıcı girişi, bu başlatılan sorgu, durumda distibuted sorguda iade edilecektir. - -``` sql -SELECT currentUser(); -``` - -Takma ad: `user()`, `USER()`. - -**Döndürülen değerler** - -- Geçerli kullanıcının girişi. -- Disributed sorgu durumunda sorgu başlatılan kullanıcının giriş. - -Tür: `String`. - -**Örnek** - -Sorgu: - -``` sql -SELECT currentUser(); -``` - -Sonuç: - -``` text -┌─currentUser()─┐ -│ default │ -└───────────────┘ -``` - -## ısconstant {#is-constant} - -Bağımsız değişken sabit bir ifade olup olmadığını denetler. - -A constant expression means an expression whose resulting value is known at the query analysis (i.e. before execution). For example, expressions over [harfler](../syntax.md#literals) sabit ifadelerdir. - -Fonksiyon geliştirme, hata ayıklama ve gösteri için tasarlanmıştır. - -**Sözdizimi** - -``` sql -isConstant(x) -``` - -**Parametre** - -- `x` — Expression to check. - -**Döndürülen değerler** - -- `1` — `x` sabit istir. -- `0` — `x` sabit olmayan. - -Tür: [Uİnt8](../data-types/int-uint.md). - -**Örnekler** - -Sorgu: - -``` sql -SELECT isConstant(x + 1) FROM (SELECT 43 AS x) -``` - -Sonuç: - -``` text -┌─isConstant(plus(x, 1))─┐ -│ 1 │ -└────────────────────────┘ -``` - -Sorgu: - -``` sql -WITH 3.14 AS pi SELECT isConstant(cos(pi)) -``` - -Sonuç: - -``` text -┌─isConstant(cos(pi))─┐ -│ 1 │ -└─────────────────────┘ -``` - -Sorgu: - -``` sql -SELECT isConstant(number) FROM numbers(1) -``` - -Sonuç: - -``` text -┌─isConstant(number)─┐ -│ 0 │ -└────────────────────┘ -``` - -## isFinite (x) {#isfinitex} - -Float32 ve Float64 kabul eder ve bağımsız değişken sonsuz değilse ve bir NaN değilse, Uint8'i 1'e eşit olarak döndürür, aksi halde 0. - -## isİnfinite (x) {#isinfinitex} - -Float32 ve Float64 kabul eder ve bağımsız değişken sonsuz ise 1'e eşit Uİnt8 döndürür, aksi takdirde 0. Bir NaN için 0 döndürüldüğünü unutmayın. - -## ifNotFinite {#ifnotfinite} - -Kayan nokta değerinin sonlu olup olmadığını kontrol eder. - -**Sözdizimi** - - ifNotFinite(x,y) - -**Parametre** - -- `x` — Value to be checked for infinity. Type: [Yüzdürmek\*](../../sql-reference/data-types/float.md). -- `y` — Fallback value. Type: [Yüzdürmek\*](../../sql-reference/data-types/float.md). - -**Döndürülen değer** - -- `x` eğer `x` son isludur. -- `y` eğer `x` sonlu değildir. - -**Örnek** - -Sorgu: - - SELECT 1/0 as infimum, ifNotFinite(infimum,42) - -Sonuç: - - ┌─infimum─┬─ifNotFinite(divide(1, 0), 42)─┐ - │ inf │ 42 │ - └─────────┴───────────────────────────────┘ - -Kullanarak benzer sonuç alabilirsiniz [üçlü operatör](conditional-functions.md#ternary-operator): `isFinite(x) ? x : y`. - -## ısnan (x) {#isnanx} - -Float32 ve Float64 kabul eder ve bağımsız değişken bir NaN, aksi takdirde 0 ise 1'e eşit uint8 döndürür. - -## hasColumnİnTable(\[‘hostname’\[, ‘username’\[, ‘password’\]\],\] ‘database’, ‘table’, ‘column’) {#hascolumnintablehostname-username-password-database-table-column} - -Sabit dizeleri kabul eder: veritabanı adı, tablo adı ve sütun adı. Bir sütun varsa 1'e eşit bir uint8 sabit ifadesi döndürür, aksi halde 0. Hostname parametresi ayarlanmışsa, sınama uzak bir sunucuda çalışır. -Tablo yoksa, işlev bir özel durum atar. -İç içe veri yapısındaki öğeler için işlev, bir sütunun varlığını denetler. İç içe veri yapısının kendisi için işlev 0 döndürür. - -## bar {#function-bar} - -Unicode-art diyagramı oluşturmaya izin verir. - -`bar(x, min, max, width)` genişliği orantılı olan bir bant çizer `(x - min)` ve eşit `width` karakterler ne zaman `x = max`. - -Parametre: - -- `x` — Size to display. -- `min, max` — Integer constants. The value must fit in `Int64`. -- `width` — Constant, positive integer, can be fractional. - -Bant, bir sembolün sekizde birine doğrulukla çizilir. - -Örnek: - -``` sql -SELECT - toHour(EventTime) AS h, - count() AS c, - bar(c, 0, 600000, 20) AS bar -FROM test.hits -GROUP BY h -ORDER BY h ASC -``` - -``` text -┌──h─┬──────c─┬─bar────────────────┐ -│ 0 │ 292907 │ █████████▋ │ -│ 1 │ 180563 │ ██████ │ -│ 2 │ 114861 │ ███▋ │ -│ 3 │ 85069 │ ██▋ │ -│ 4 │ 68543 │ ██▎ │ -│ 5 │ 78116 │ ██▌ │ -│ 6 │ 113474 │ ███▋ │ -│ 7 │ 170678 │ █████▋ │ -│ 8 │ 278380 │ █████████▎ │ -│ 9 │ 391053 │ █████████████ │ -│ 10 │ 457681 │ ███████████████▎ │ -│ 11 │ 493667 │ ████████████████▍ │ -│ 12 │ 509641 │ ████████████████▊ │ -│ 13 │ 522947 │ █████████████████▍ │ -│ 14 │ 539954 │ █████████████████▊ │ -│ 15 │ 528460 │ █████████████████▌ │ -│ 16 │ 539201 │ █████████████████▊ │ -│ 17 │ 523539 │ █████████████████▍ │ -│ 18 │ 506467 │ ████████████████▊ │ -│ 19 │ 520915 │ █████████████████▎ │ -│ 20 │ 521665 │ █████████████████▍ │ -│ 21 │ 542078 │ ██████████████████ │ -│ 22 │ 493642 │ ████████████████▍ │ -│ 23 │ 400397 │ █████████████▎ │ -└────┴────────┴────────────────────┘ -``` - -## dönüştürmek {#transform} - -Bir değeri, bazı öğelerin açıkça tanımlanmış eşlemesine göre diğer öğelere dönüştürür. -Bu fonksiyonun iki varyasyonu vardır: - -### transform (x, array_from, array_to, varsayılan) {#transformx-array-from-array-to-default} - -`x` – What to transform. - -`array_from` – Constant array of values for converting. - -`array_to` – Constant array of values to convert the values in ‘from’ -e doğru. - -`default` – Which value to use if ‘x’ değer anylerden hiçbir equaline eşit değildir. ‘from’. - -`array_from` ve `array_to` – Arrays of the same size. - -Türler: - -`transform(T, Array(T), Array(U), U) -> U` - -`T` ve `U` sayısal, dize veya tarih veya DateTime türleri olabilir. -Aynı harfin belirtildiği (t veya U), sayısal türler için bunlar eşleşen türler değil, ortak bir türe sahip türler olabilir. -Örneğin, ilk bağımsız değişken Int64 türüne sahip olabilir, ikincisi ise Array(Uİnt16) türüne sahiptir. - -Eğer... ‘x’ değer, içindeki öğelerden birine eşittir. ‘array_from’ array, varolan öğeyi döndürür (aynı numaralandırılır) ‘array_to’ dizi. Aksi takdirde, döner ‘default’. İçinde birden fazla eşleşen öğe varsa ‘array_from’, maçlardan birini döndürür. - -Örnek: - -``` sql -SELECT - transform(SearchEngineID, [2, 3], ['Yandex', 'Google'], 'Other') AS title, - count() AS c -FROM test.hits -WHERE SearchEngineID != 0 -GROUP BY title -ORDER BY c DESC -``` - -``` text -┌─title─────┬──────c─┐ -│ Yandex │ 498635 │ -│ Google │ 229872 │ -│ Other │ 104472 │ -└───────────┴────────┘ -``` - -### transform (x, array_from, array_to) {#transformx-array-from-array-to} - -İlk vary thatasyon differsdan farklıdır. ‘default’ argüman atlandı. -Eğer... ‘x’ değer, içindeki öğelerden birine eşittir. ‘array_from’ array, eşleşen öğeyi (aynı numaralandırılmış) döndürür ‘array_to’ dizi. Aksi takdirde, döner ‘x’. - -Türler: - -`transform(T, Array(T), Array(T)) -> T` - -Örnek: - -``` sql -SELECT - transform(domain(Referer), ['yandex.ru', 'google.ru', 'vk.com'], ['www.yandex', 'example.com']) AS s, - count() AS c -FROM test.hits -GROUP BY domain(Referer) -ORDER BY count() DESC -LIMIT 10 -``` - -``` text -┌─s──────────────┬───────c─┐ -│ │ 2906259 │ -│ www.yandex │ 867767 │ -│ ███████.ru │ 313599 │ -│ mail.yandex.ru │ 107147 │ -│ ██████.ru │ 100355 │ -│ █████████.ru │ 65040 │ -│ news.yandex.ru │ 64515 │ -│ ██████.net │ 59141 │ -│ example.com │ 57316 │ -└────────────────┴─────────┘ -``` - -## formatReadableSize (x) {#formatreadablesizex} - -Boyutu (bayt sayısı) kabul eder. Bir sonek (KiB, MıB, vb.) ile yuvarlak bir boyut döndürür.) bir dize olarak. - -Örnek: - -``` sql -SELECT - arrayJoin([1, 1024, 1024*1024, 192851925]) AS filesize_bytes, - formatReadableSize(filesize_bytes) AS filesize -``` - -``` text -┌─filesize_bytes─┬─filesize───┐ -│ 1 │ 1.00 B │ -│ 1024 │ 1.00 KiB │ -│ 1048576 │ 1.00 MiB │ -│ 192851925 │ 183.92 MiB │ -└────────────────┴────────────┘ -``` - -## en az (a, b) {#leasta-b} - -A ve B'den en küçük değeri döndürür. - -## en büyük (a, b) {#greatesta-b} - -A ve B'nin en büyük değerini döndürür. - -## çalışma süresi() {#uptime} - -Sunucunun çalışma süresini saniyeler içinde döndürür. - -## sürüm() {#version} - -Sunucu sürümünü bir dize olarak döndürür. - -## saat dilimi() {#timezone} - -Sunucunun saat dilimini döndürür. - -## blockNumber {#blocknumber} - -Satırın bulunduğu veri bloğunun sıra numarasını döndürür. - -## rowNumberİnBlock {#function-rownumberinblock} - -Veri bloğundaki satırın sıra numarasını döndürür. Farklı veri blokları her zaman yeniden hesaplanır. - -## rownumberınallblocks() {#rownumberinallblocks} - -Veri bloğundaki satırın sıra numarasını döndürür. Bu işlev yalnızca etkilenen veri bloklarını dikkate alır. - -## komşuluk {#neighbor} - -Belirli bir sütunun geçerli satırından önce veya sonra gelen belirli bir ofsette bir satıra erişim sağlayan pencere işlevi. - -**Sözdizimi** - -``` sql -neighbor(column, offset[, default_value]) -``` - -İşlevin sonucu, etkilenen veri bloklarına ve bloktaki veri sırasına bağlıdır. -ORDER BY ile bir alt sorgu yaparsanız ve alt sorgunun dışından işlevi çağırırsanız, beklenen sonucu alabilirsiniz. - -**Parametre** - -- `column` — A column name or scalar expression. -- `offset` — The number of rows forwards or backwards from the current row of `column`. [Int64](../../sql-reference/data-types/int-uint.md). -- `default_value` — Optional. The value to be returned if offset goes beyond the scope of the block. Type of data blocks affected. - -**Döndürülen değerler** - -- İçin değer `column` içinde `offset` eğer geçerli satırdan uzaklık `offset` değer blok sınırları dışında değil. -- İçin varsayılan değer `column` eğer `offset` değer, blok sınırlarının dışındadır. Eğer `default_value` verilir, daha sonra kullanılacaktır. - -Tür: etkilenen veri bloklarının türü veya varsayılan değer türü. - -**Örnek** - -Sorgu: - -``` sql -SELECT number, neighbor(number, 2) FROM system.numbers LIMIT 10; -``` - -Sonuç: - -``` text -┌─number─┬─neighbor(number, 2)─┐ -│ 0 │ 2 │ -│ 1 │ 3 │ -│ 2 │ 4 │ -│ 3 │ 5 │ -│ 4 │ 6 │ -│ 5 │ 7 │ -│ 6 │ 8 │ -│ 7 │ 9 │ -│ 8 │ 0 │ -│ 9 │ 0 │ -└────────┴─────────────────────┘ -``` - -Sorgu: - -``` sql -SELECT number, neighbor(number, 2, 999) FROM system.numbers LIMIT 10; -``` - -Sonuç: - -``` text -┌─number─┬─neighbor(number, 2, 999)─┐ -│ 0 │ 2 │ -│ 1 │ 3 │ -│ 2 │ 4 │ -│ 3 │ 5 │ -│ 4 │ 6 │ -│ 5 │ 7 │ -│ 6 │ 8 │ -│ 7 │ 9 │ -│ 8 │ 999 │ -│ 9 │ 999 │ -└────────┴──────────────────────────┘ -``` - -Bu işlev, yıldan yıla metrik değeri hesaplamak için kullanılabilir: - -Sorgu: - -``` sql -WITH toDate('2018-01-01') AS start_date -SELECT - toStartOfMonth(start_date + (number * 32)) AS month, - toInt32(month) % 100 AS money, - neighbor(money, -12) AS prev_year, - round(prev_year / money, 2) AS year_over_year -FROM numbers(16) -``` - -Sonuç: - -``` text -┌──────month─┬─money─┬─prev_year─┬─year_over_year─┐ -│ 2018-01-01 │ 32 │ 0 │ 0 │ -│ 2018-02-01 │ 63 │ 0 │ 0 │ -│ 2018-03-01 │ 91 │ 0 │ 0 │ -│ 2018-04-01 │ 22 │ 0 │ 0 │ -│ 2018-05-01 │ 52 │ 0 │ 0 │ -│ 2018-06-01 │ 83 │ 0 │ 0 │ -│ 2018-07-01 │ 13 │ 0 │ 0 │ -│ 2018-08-01 │ 44 │ 0 │ 0 │ -│ 2018-09-01 │ 75 │ 0 │ 0 │ -│ 2018-10-01 │ 5 │ 0 │ 0 │ -│ 2018-11-01 │ 36 │ 0 │ 0 │ -│ 2018-12-01 │ 66 │ 0 │ 0 │ -│ 2019-01-01 │ 97 │ 32 │ 0.33 │ -│ 2019-02-01 │ 28 │ 63 │ 2.25 │ -│ 2019-03-01 │ 56 │ 91 │ 1.62 │ -│ 2019-04-01 │ 87 │ 22 │ 0.25 │ -└────────────┴───────┴───────────┴────────────────┘ -``` - -## runningDifference (x) {#other_functions-runningdifference} - -Calculates the difference between successive row values ​​in the data block. -İlk satır için 0 ve sonraki her satır için önceki satırdan farkı döndürür. - -İşlevin sonucu, etkilenen veri bloklarına ve bloktaki veri sırasına bağlıdır. -ORDER BY ile bir alt sorgu yaparsanız ve alt sorgunun dışından işlevi çağırırsanız, beklenen sonucu alabilirsiniz. - -Örnek: - -``` sql -SELECT - EventID, - EventTime, - runningDifference(EventTime) AS delta -FROM -( - SELECT - EventID, - EventTime - FROM events - WHERE EventDate = '2016-11-24' - ORDER BY EventTime ASC - LIMIT 5 -) -``` - -``` text -┌─EventID─┬───────────EventTime─┬─delta─┐ -│ 1106 │ 2016-11-24 00:00:04 │ 0 │ -│ 1107 │ 2016-11-24 00:00:05 │ 1 │ -│ 1108 │ 2016-11-24 00:00:05 │ 0 │ -│ 1109 │ 2016-11-24 00:00:09 │ 4 │ -│ 1110 │ 2016-11-24 00:00:10 │ 1 │ -└─────────┴─────────────────────┴───────┘ -``` - -Lütfen dikkat - blok boyutu sonucu etkiler. Her yeni blok ile, `runningDifference` durum sıfırlandı. - -``` sql -SELECT - number, - runningDifference(number + 1) AS diff -FROM numbers(100000) -WHERE diff != 1 -``` - -``` text -┌─number─┬─diff─┐ -│ 0 │ 0 │ -└────────┴──────┘ -┌─number─┬─diff─┐ -│ 65536 │ 0 │ -└────────┴──────┘ -``` - -``` sql -set max_block_size=100000 -- default value is 65536! - -SELECT - number, - runningDifference(number + 1) AS diff -FROM numbers(100000) -WHERE diff != 1 -``` - -``` text -┌─number─┬─diff─┐ -│ 0 │ 0 │ -└────────┴──────┘ -``` - -## runningDifferenceStartingWithFirstvalue {#runningdifferencestartingwithfirstvalue} - -İçin aynı [runningDifference](./other-functions.md#other_functions-runningdifference), fark ilk satırın değeridir, ilk satırın değerini döndürdü ve sonraki her satır önceki satırdan farkı döndürür. - -## MACNumToString (num) {#macnumtostringnum} - -Bir uınt64 numarasını kabul eder. Big endian'da bir MAC adresi olarak yorumlar. AA:BB:CC:DD:EE:FF biçiminde karşılık gelen MAC adresini içeren bir dize döndürür (onaltılık formda iki nokta üst üste ayrılmış sayılar). - -## MACStringToNum (s) {#macstringtonums} - -MACNumToString ters işlevi. MAC adresi geçersiz bir biçime sahipse, 0 döndürür. - -## MACStringToOUİ (s) {#macstringtoouis} - -AA:BB:CC:DD:EE:FF (onaltılık formda iki nokta üst üste ayrılmış sayılar) biçiminde bir MAC adresi kabul eder. İlk üç sekizli uint64 numarası olarak döndürür. MAC adresi geçersiz bir biçime sahipse, 0 döndürür. - -## getSizeOfEnumType {#getsizeofenumtype} - -Alan sayısını döndürür [Enum](../../sql-reference/data-types/enum.md). - -``` sql -getSizeOfEnumType(value) -``` - -**Parametre:** - -- `value` — Value of type `Enum`. - -**Döndürülen değerler** - -- İle alan sayısı `Enum` giriş değerleri. -- Tür değilse bir istisna atılır `Enum`. - -**Örnek** - -``` sql -SELECT getSizeOfEnumType( CAST('a' AS Enum8('a' = 1, 'b' = 2) ) ) AS x -``` - -``` text -┌─x─┐ -│ 2 │ -└───┘ -``` - -## blockSerializedSize {#blockserializedsize} - -Diskteki boyutu döndürür (sıkıştırmayı hesaba katmadan). - -``` sql -blockSerializedSize(value[, value[, ...]]) -``` - -**Parametre:** - -- `value` — Any value. - -**Döndürülen değerler** - -- (Sıkıştırma olmadan) değerler bloğu için diske yazılacak bayt sayısı. - -**Örnek** - -``` sql -SELECT blockSerializedSize(maxState(1)) as x -``` - -``` text -┌─x─┐ -│ 2 │ -└───┘ -``` - -## toColumnTypeName {#tocolumntypename} - -RAM'DEKİ sütunun veri türünü temsil eden sınıfın adını döndürür. - -``` sql -toColumnTypeName(value) -``` - -**Parametre:** - -- `value` — Any type of value. - -**Döndürülen değerler** - -- Temsil etmek için kullanılan sınıfın adını içeren bir dize `value` RAM veri türü. - -**Arasındaki fark örneği`toTypeName ' and ' toColumnTypeName`** - -``` sql -SELECT toTypeName(CAST('2018-01-01 01:02:03' AS DateTime)) -``` - -``` text -┌─toTypeName(CAST('2018-01-01 01:02:03', 'DateTime'))─┐ -│ DateTime │ -└─────────────────────────────────────────────────────┘ -``` - -``` sql -SELECT toColumnTypeName(CAST('2018-01-01 01:02:03' AS DateTime)) -``` - -``` text -┌─toColumnTypeName(CAST('2018-01-01 01:02:03', 'DateTime'))─┐ -│ Const(UInt32) │ -└───────────────────────────────────────────────────────────┘ -``` - -Örnek gösteriyor ki `DateTime` veri türü olarak bellekte saklanır `Const(UInt32)`. - -## dumpColumnStructure {#dumpcolumnstructure} - -Ram'deki veri yapılarının ayrıntılı bir açıklamasını verir - -``` sql -dumpColumnStructure(value) -``` - -**Parametre:** - -- `value` — Any type of value. - -**Döndürülen değerler** - -- Temsil etmek için kullanılan yapıyı açıklayan bir dize `value` RAM veri türü. - -**Örnek** - -``` sql -SELECT dumpColumnStructure(CAST('2018-01-01 01:02:03', 'DateTime')) -``` - -``` text -┌─dumpColumnStructure(CAST('2018-01-01 01:02:03', 'DateTime'))─┐ -│ DateTime, Const(size = 1, UInt32(size = 1)) │ -└──────────────────────────────────────────────────────────────┘ -``` - -## defaultValueOfArgumentType {#defaultvalueofargumenttype} - -Veri türü için varsayılan değeri verir. - -Kullanıcı tarafından ayarlanan özel sütunlar için varsayılan değerleri içermez. - -``` sql -defaultValueOfArgumentType(expression) -``` - -**Parametre:** - -- `expression` — Arbitrary type of value or an expression that results in a value of an arbitrary type. - -**Döndürülen değerler** - -- `0` sayılar için. -- Dizeler için boş dize. -- `ᴺᵁᴸᴸ` için [Nullable](../../sql-reference/data-types/nullable.md). - -**Örnek** - -``` sql -SELECT defaultValueOfArgumentType( CAST(1 AS Int8) ) -``` - -``` text -┌─defaultValueOfArgumentType(CAST(1, 'Int8'))─┐ -│ 0 │ -└─────────────────────────────────────────────┘ -``` - -``` sql -SELECT defaultValueOfArgumentType( CAST(1 AS Nullable(Int8) ) ) -``` - -``` text -┌─defaultValueOfArgumentType(CAST(1, 'Nullable(Int8)'))─┐ -│ ᴺᵁᴸᴸ │ -└───────────────────────────────────────────────────────┘ -``` - -## çoğaltmak {#other-functions-replicate} - -Tek bir değere sahip bir dizi oluşturur. - -İç uygulama için kullanılan [arrayJoin](array-join.md#functions_arrayjoin). - -``` sql -SELECT replicate(x, arr); -``` - -**Parametre:** - -- `arr` — Original array. ClickHouse creates a new array of the same length as the original and fills it with the value `x`. -- `x` — The value that the resulting array will be filled with. - -**Döndürülen değer** - -Değerle dolu bir dizi `x`. - -Tür: `Array`. - -**Örnek** - -Sorgu: - -``` sql -SELECT replicate(1, ['a', 'b', 'c']) -``` - -Sonuç: - -``` text -┌─replicate(1, ['a', 'b', 'c'])─┐ -│ [1,1,1] │ -└───────────────────────────────┘ -``` - -## filesystemAvailable {#filesystemavailable} - -Veritabanlarının dosyalarının bulunduğu dosya sisteminde kalan alan miktarını döndürür. Her zaman toplam boş alandan daha küçüktür ([filesystemFree](#filesystemfree)) çünkü OS için biraz alan ayrılmıştır. - -**Sözdizimi** - -``` sql -filesystemAvailable() -``` - -**Döndürülen değer** - -- Bayt olarak kullanılabilir kalan alan miktarı. - -Tür: [Uİnt64](../../sql-reference/data-types/int-uint.md). - -**Örnek** - -Sorgu: - -``` sql -SELECT formatReadableSize(filesystemAvailable()) AS "Available space", toTypeName(filesystemAvailable()) AS "Type"; -``` - -Sonuç: - -``` text -┌─Available space─┬─Type───┐ -│ 30.75 GiB │ UInt64 │ -└─────────────────┴────────┘ -``` - -## filesystemFree {#filesystemfree} - -Veritabanlarının dosyalarının bulunduğu dosya sistemindeki boş alanın toplam miktarını döndürür. Ayrıca bakınız `filesystemAvailable` - -**Sözdizimi** - -``` sql -filesystemFree() -``` - -**Döndürülen değer** - -- Bayt cinsinden boş alan miktarı. - -Tür: [Uİnt64](../../sql-reference/data-types/int-uint.md). - -**Örnek** - -Sorgu: - -``` sql -SELECT formatReadableSize(filesystemFree()) AS "Free space", toTypeName(filesystemFree()) AS "Type"; -``` - -Sonuç: - -``` text -┌─Free space─┬─Type───┐ -│ 32.39 GiB │ UInt64 │ -└────────────┴────────┘ -``` - -## filesystemCapacity {#filesystemcapacity} - -Dosya sisteminin kapasitesini bayt cinsinden döndürür. Değerlendirme için, [yol](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-path) veri dizinine yapılandırılmalıdır. - -**Sözdizimi** - -``` sql -filesystemCapacity() -``` - -**Döndürülen değer** - -- Dosya sisteminin bayt cinsinden kapasite bilgisi. - -Tür: [Uİnt64](../../sql-reference/data-types/int-uint.md). - -**Örnek** - -Sorgu: - -``` sql -SELECT formatReadableSize(filesystemCapacity()) AS "Capacity", toTypeName(filesystemCapacity()) AS "Type" -``` - -Sonuç: - -``` text -┌─Capacity──┬─Type───┐ -│ 39.32 GiB │ UInt64 │ -└───────────┴────────┘ -``` - -## finalizeAggregation {#function-finalizeaggregation} - -Toplama işlevinin durumunu alır. Toplama sonucunu döndürür (kesinleşmiş durum). - -## runningAccumulate {#function-runningaccumulate} - -Toplama işlevinin durumlarını alır ve değerleri olan bir sütun döndürür, bu durumların bir dizi blok satırı için ilk satırdan geçerli satıra birikmesinin sonucudur. -Örneğin, toplama işlevinin durumunu alır (örnek runningAccumulate (uniqState (Userıd))) ve her blok satırı için, önceki tüm Satırların ve geçerli satırın durumlarının birleştirilmesinde toplama işlevinin sonucunu döndürür. -Bu nedenle, işlevin sonucu, verilerin bloklara bölünmesine ve blok içindeki verilerin sırasına bağlıdır. - -## joinGet {#joinget} - -İşlev, tablodan verileri bir tablodan aynı şekilde ayıklamanızı sağlar [sözlük](../../sql-reference/dictionaries/index.md). - -Veri alır [Katmak](../../engines/table-engines/special/join.md#creating-a-table) belirtilen birleştirme anahtarını kullanarak tablolar. - -Sadece ile oluşturulan tabloları destekler `ENGINE = Join(ANY, LEFT, )` deyim. - -**Sözdizimi** - -``` sql -joinGet(join_storage_table_name, `value_column`, join_keys) -``` - -**Parametre** - -- `join_storage_table_name` — an [tanıtıcı](../syntax.md#syntax-identifiers) aramanın nerede yapıldığını gösterir. Tanımlayıcı varsayılan veritabanında aranır (bkz. parametre `default_database` config dosyası). Varsayılan veritabanını geçersiz kılmak için `USE db_name` veya ayırıcı aracılığıyla veritabanını ve tabloyu belirtin `db_name.db_table` örnek bakın. -- `value_column` — name of the column of the table that contains required data. -- `join_keys` — list of keys. - -**Döndürülen değer** - -Anahtarların listesine karşılık gelen değerlerin listesini döndürür. - -Kaynak tabloda kesin yoksa o zaman `0` veya `null` esas alınarak iade edilecektir [join_use_nulls](../../operations/settings/settings.md#join_use_nulls) ayar. - -Hakkında daha fazla bilgi `join_use_nulls` içinde [Birleştirme işlemi](../../engines/table-engines/special/join.md). - -**Örnek** - -Giriş tablosu: - -``` sql -CREATE DATABASE db_test -CREATE TABLE db_test.id_val(`id` UInt32, `val` UInt32) ENGINE = Join(ANY, LEFT, id) SETTINGS join_use_nulls = 1 -INSERT INTO db_test.id_val VALUES (1,11)(2,12)(4,13) -``` - -``` text -┌─id─┬─val─┐ -│ 4 │ 13 │ -│ 2 │ 12 │ -│ 1 │ 11 │ -└────┴─────┘ -``` - -Sorgu: - -``` sql -SELECT joinGet(db_test.id_val,'val',toUInt32(number)) from numbers(4) SETTINGS join_use_nulls = 1 -``` - -Sonuç: - -``` text -┌─joinGet(db_test.id_val, 'val', toUInt32(number))─┐ -│ 0 │ -│ 11 │ -│ 12 │ -│ 0 │ -└──────────────────────────────────────────────────┘ -``` - -## modelEvaluate(model_name, …) {#function-modelevaluate} - -Dış modeli değerlendirin. -Bir model adı ve model bağımsız değişkenleri kabul eder. Float64 Döndürür. - -## throwİf(x \[, custom_message\]) {#throwifx-custom-message} - -Argüman sıfır değilse bir istisna atın. -custom_message-isteğe bağlı bir parametredir: sabit bir dize, bir hata mesajı sağlar - -``` sql -SELECT throwIf(number = 3, 'Too many') FROM numbers(10); -``` - -``` text -↙ Progress: 0.00 rows, 0.00 B (0.00 rows/s., 0.00 B/s.) Received exception from server (version 19.14.1): -Code: 395. DB::Exception: Received from localhost:9000. DB::Exception: Too many. -``` - -## kimlik {#identity} - -Bağımsız değişkeni olarak kullanılan aynı değeri döndürür. Hata ayıklama ve test için kullanılan, dizin kullanarak iptal ve tam bir tarama sorgu performansını almak için izin verir. Olası dizin kullanımı için sorgu analiz edildiğinde, analizör içeriye bakmaz `identity` işlevler. - -**Sözdizimi** - -``` sql -identity(x) -``` - -**Örnek** - -Sorgu: - -``` sql -SELECT identity(42) -``` - -Sonuç: - -``` text -┌─identity(42)─┐ -│ 42 │ -└──────────────┘ -``` - -## randomPrintableASCİİ {#randomascii} - -Rastgele bir dizi ile bir dize oluşturur [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters) yazdırılabilir karakterler. - -**Sözdizimi** - -``` sql -randomPrintableASCII(length) -``` - -**Parametre** - -- `length` — Resulting string length. Positive integer. - - If you pass `length < 0`, behavior of the function is undefined. - -**Döndürülen değer** - -- Rastgele bir dizi dize [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters) yazdırılabilir karakterler. - -Tür: [Dize](../../sql-reference/data-types/string.md) - -**Örnek** - -``` sql -SELECT number, randomPrintableASCII(30) as str, length(str) FROM system.numbers LIMIT 3 -``` - -``` text -┌─number─┬─str────────────────────────────┬─length(randomPrintableASCII(30))─┐ -│ 0 │ SuiCOSTvC0csfABSw=UcSzp2.`rv8x │ 30 │ -│ 1 │ 1Ag NlJ &RCN:*>HVPG;PE-nO"SUFD │ 30 │ -│ 2 │ /"+<"wUTh:=LjJ Vm!c&hI*m#XTfzz │ 30 │ -└────────┴────────────────────────────────┴──────────────────────────────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/other_functions/) diff --git a/docs/tr/sql-reference/functions/random-functions.md b/docs/tr/sql-reference/functions/random-functions.md deleted file mode 100644 index 8ec986cbb18..00000000000 --- a/docs/tr/sql-reference/functions/random-functions.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 51 -toc_title: "S\xF6zde Rasgele Say\u0131lar Olu\u015Fturma" ---- - -# Sözde rasgele sayılar üretmek için fonksiyonlar {#functions-for-generating-pseudo-random-numbers} - -Sözde rasgele sayıların kriptografik olmayan jeneratörleri kullanılır. - -Tüm işlevler sıfır bağımsız değişkeni veya bir bağımsız değişkeni kabul eder. -Bir argüman geçirilirse, herhangi bir tür olabilir ve değeri hiçbir şey için kullanılmaz. -Bu argümanın tek amacı, aynı işlevin iki farklı örneğinin farklı rasgele sayılarla farklı sütunlar döndürmesi için ortak alt ifade eliminasyonunu önlemektir. - -## Güney Afrika parası {#rand} - -Tüm uint32 tipi sayılar arasında eşit olarak dağıtılan bir sözde rasgele uint32 numarası döndürür. -Doğrusal bir uyumlu jeneratör kullanır. - -## rand64 {#rand64} - -Tüm uint64 tipi sayılar arasında eşit olarak dağıtılan sözde rasgele bir uint64 numarası döndürür. -Doğrusal bir uyumlu jeneratör kullanır. - -## randConstant {#randconstant} - -Rasgele bir değere sahip sabit bir sütun üretir. - -**Sözdizimi** - -``` sql -randConstant([x]) -``` - -**Parametre** - -- `x` — [İfade](../syntax.md#syntax-expressions) sonuç olarak herhangi bir [desteklenen veri türleri](../data-types/index.md#data_types). Elde edilen değer atılır, ancak baypas için kullanıldığında ifadenin kendisi [ortak subexpression eliminasyonu](index.md#common-subexpression-elimination) işlev bir sorguda birden çok kez çağrılırsa. İsteğe bağlı parametre. - -**Döndürülen değer** - -- Sözde rasgele sayı. - -Tür: [Uİnt32](../data-types/int-uint.md). - -**Örnek** - -Sorgu: - -``` sql -SELECT rand(), rand(1), rand(number), randConstant(), randConstant(1), randConstant(number) -FROM numbers(3) -``` - -Sonuç: - -``` text -┌─────rand()─┬────rand(1)─┬─rand(number)─┬─randConstant()─┬─randConstant(1)─┬─randConstant(number)─┐ -│ 3047369878 │ 4132449925 │ 4044508545 │ 2740811946 │ 4229401477 │ 1924032898 │ -│ 2938880146 │ 1267722397 │ 4154983056 │ 2740811946 │ 4229401477 │ 1924032898 │ -│ 956619638 │ 4238287282 │ 1104342490 │ 2740811946 │ 4229401477 │ 1924032898 │ -└────────────┴────────────┴──────────────┴────────────────┴─────────────────┴──────────────────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/random_functions/) diff --git a/docs/tr/sql-reference/functions/rounding-functions.md b/docs/tr/sql-reference/functions/rounding-functions.md deleted file mode 100644 index 5dc185c6128..00000000000 --- a/docs/tr/sql-reference/functions/rounding-functions.md +++ /dev/null @@ -1,190 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 45 -toc_title: "D\xF6nm\xFC\u015F" ---- - -# Yuvarlama Fonksiyonları {#rounding-functions} - -## kat(x \[, N\]) {#floorx-n} - -Küçük veya eşit olan en büyük yuvarlak sayıyı döndürür `x`. Yuvarlak bir sayı, 1/10N'NİN katları veya 1 / 10N tam değilse, uygun veri türünün en yakın sayısıdır. -‘N’ bir tamsayı sabiti, isteğe bağlı parametredir. Varsayılan olarak sıfırdır, bu da bir tam sayıya yuvarlamak anlamına gelir. -‘N’ negatif olabilir. - -Örnekler: `floor(123.45, 1) = 123.4, floor(123.45, -1) = 120.` - -`x` herhangi bir sayısal türüdür. Sonuç aynı türden bir sayıdır. -Tamsayı argümanları için, bir negatif ile yuvarlamak mantıklıdır `N` değer (negatif olmayan için `N`, işlev hiçbir şey yapmaz). -Yuvarlama taşmasına neden olursa (örneğin, floor (-128, -1)), uygulamaya özgü bir sonuç döndürülür. - -## tavan(x \[, N\]), tavan (x \[, N\]) {#ceilx-n-ceilingx-n} - -Büyük veya eşit olan en küçük yuvarlak sayıyı döndürür `x`. Diğer her şekilde, aynı `floor` (yukarıda) işlevi. - -## trunc(x \[, N\]), truncate(x \[, N\]) {#truncx-n-truncatex-n} - -Mutlak değeri küçük veya eşit olan en büyük mutlak değere sahip yuvarlak sayıyı döndürür `x`‘s. In every other way, it is the same as the ’floor’ (yukarıda) işlevi. - -## Yuvarlak(x \[, N\]) {#rounding_functions-round} - -Belirtilen sayıda ondalık basamak için bir değer yuvarlar. - -İşlev, belirtilen siparişin en yakın numarasını döndürür. Verilen sayı çevreleyen sayılara eşit mesafeye sahip olduğunda, işlev, float sayı türleri için bankacının yuvarlamasını kullanır ve diğer sayı türleri için sıfırdan uzaklaşır. - -``` sql -round(expression [, decimal_places]) -``` - -**Parametre:** - -- `expression` — A number to be rounded. Can be any [ifade](../syntax.md#syntax-expressions) sayısal dönen [veri türü](../../sql-reference/data-types/index.md#data_types). -- `decimal-places` — An integer value. - - Eğer `decimal-places > 0` sonra işlev değeri ondalık noktanın sağına yuvarlar. - - Eğer `decimal-places < 0` ardından işlev değeri ondalık noktanın soluna yuvarlar. - - Eğer `decimal-places = 0` sonra işlev değeri tamsayı olarak yuvarlar. Bu durumda argüman ihmal edilebilir. - -**Döndürülen değer:** - -Giriş numarası ile aynı türden yuvarlatılmış sayı. - -### Örnekler {#examples} - -**Kullanım örneği** - -``` sql -SELECT number / 2 AS x, round(x) FROM system.numbers LIMIT 3 -``` - -``` text -┌───x─┬─round(divide(number, 2))─┐ -│ 0 │ 0 │ -│ 0.5 │ 0 │ -│ 1 │ 1 │ -└─────┴──────────────────────────┘ -``` - -**Yuvarlama örnekleri** - -En yakın numaraya yuvarlama. - -``` text -round(3.2, 0) = 3 -round(4.1267, 2) = 4.13 -round(22,-1) = 20 -round(467,-2) = 500 -round(-467,-2) = -500 -``` - -Bankacı yuvarlanıyor. - -``` text -round(3.5) = 4 -round(4.5) = 4 -round(3.55, 1) = 3.6 -round(3.65, 1) = 3.6 -``` - -**Ayrıca Bakınız** - -- [roundBankers](#roundbankers) - -## roundBankers {#roundbankers} - -Bir sayıyı belirtilen ondalık konuma yuvarlar. - -- Yuvarlama sayısı iki sayı arasında yarıya ise, işlev banker yuvarlama kullanır. - - Banker's rounding is a method of rounding fractional numbers. When the rounding number is halfway between two numbers, it's rounded to the nearest even digit at the specified decimal position. For example: 3.5 rounds up to 4, 2.5 rounds down to 2. - - It's the default rounding method for floating point numbers defined in [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754#Roundings_to_nearest). The [round](#rounding_functions-round) function performs the same rounding for floating point numbers. The `roundBankers` function also rounds integers the same way, for example, `roundBankers(45, -1) = 40`. - -- Diğer durumlarda, işlev sayıları en yakın tam sayıya yuvarlar. - -Banker yuvarlama kullanarak, yuvarlama numaraları toplama veya bu sayıları çıkarma sonuçları üzerindeki etkisini azaltabilir. - -Örneğin, farklı yuvarlama ile 1.5, 2.5, 3.5, 4.5 sayılarını topla: - -- Yuvarlama yok: 1.5 + 2.5 + 3.5 + 4.5 = 12. -- Bankacı yuvarlama: 2 + 2 + 4 + 4 = 12. -- En yakın tam sayıya yuvarlama: 2 + 3 + 4 + 5 = 14. - -**Sözdizimi** - -``` sql -roundBankers(expression [, decimal_places]) -``` - -**Parametre** - -- `expression` — A number to be rounded. Can be any [ifade](../syntax.md#syntax-expressions) sayısal dönen [veri türü](../../sql-reference/data-types/index.md#data_types). -- `decimal-places` — Decimal places. An integer number. - - `decimal-places > 0` — The function rounds the number to the given position right of the decimal point. Example: `roundBankers(3.55, 1) = 3.6`. - - `decimal-places < 0` — The function rounds the number to the given position left of the decimal point. Example: `roundBankers(24.55, -1) = 20`. - - `decimal-places = 0` — The function rounds the number to an integer. In this case the argument can be omitted. Example: `roundBankers(2.5) = 2`. - -**Döndürülen değer** - -Banker yuvarlama yöntemi tarafından yuvarlanan bir değer. - -### Örnekler {#examples-1} - -**Kullanım örneği** - -Sorgu: - -``` sql - SELECT number / 2 AS x, roundBankers(x, 0) AS b fROM system.numbers limit 10 -``` - -Sonuç: - -``` text -┌───x─┬─b─┐ -│ 0 │ 0 │ -│ 0.5 │ 0 │ -│ 1 │ 1 │ -│ 1.5 │ 2 │ -│ 2 │ 2 │ -│ 2.5 │ 2 │ -│ 3 │ 3 │ -│ 3.5 │ 4 │ -│ 4 │ 4 │ -│ 4.5 │ 4 │ -└─────┴───┘ -``` - -**Bankacı yuvarlama örnekleri** - -``` text -roundBankers(0.4) = 0 -roundBankers(-3.5) = -4 -roundBankers(4.5) = 4 -roundBankers(3.55, 1) = 3.6 -roundBankers(3.65, 1) = 3.6 -roundBankers(10.35, 1) = 10.4 -roundBankers(10.755, 2) = 11,76 -``` - -**Ayrıca Bakınız** - -- [turlu](#rounding_functions-round) - -## roundToExp2 (num) {#roundtoexp2num} - -Bir sayı kabul eder. Sayı birden az ise, 0 döndürür. Aksi takdirde, sayıyı en yakın (negatif olmayan) iki dereceye yuvarlar. - -## roundDuration (num) {#rounddurationnum} - -Bir sayı kabul eder. Sayı birden az ise, 0 döndürür. Aksi takdirde, sayıyı kümeden sayılara yuvarlar: 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000. Bu fonksiyon (kayıt olmak için özeldir.Metrica ve oturum uzunluğu raporu uygulamak için kullanılır. - -## roundAge (num) {#roundagenum} - -Bir sayı kabul eder. Sayı 18'den küçükse, 0 döndürür. Aksi takdirde, sayıyı kümeden bir sayıya yuvarlar: 18, 25, 35, 45, 55. Bu fonksiyon (kayıt olmak için özeldir.Metrica ve kullanıcı yaş raporu uygulamak için kullanılır. - -## roundDown (num, arr) {#rounddownnum-arr} - -Bir sayıyı kabul eder ve belirtilen Dizideki bir öğeye yuvarlar. Değer en düşük sınırdan küçükse, en düşük sınır döndürülür. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/rounding_functions/) diff --git a/docs/tr/sql-reference/functions/splitting-merging-functions.md b/docs/tr/sql-reference/functions/splitting-merging-functions.md deleted file mode 100644 index b8c89489d33..00000000000 --- a/docs/tr/sql-reference/functions/splitting-merging-functions.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 47 -toc_title: "Dizeleri ve dizileri b\xF6lme ve birle\u015Ftirme" ---- - -# Dizeleri ve dizileri bölme ve birleştirme işlevleri {#functions-for-splitting-and-merging-strings-and-arrays} - -## splitByChar (ayırıcı, s) {#splitbycharseparator-s} - -Bir dizeyi belirtilen bir karakterle ayrılmış alt dizelere böler. Sabit bir dize kullanır `separator` tam olarak bir karakterden oluşan. -Seçili alt dizelerin bir dizisini döndürür. Ayırıcı dizenin başında veya sonunda oluşursa veya ardışık birden çok ayırıcı varsa, boş alt dizeler seçilebilir. - -**Sözdizimi** - -``` sql -splitByChar(, ) -``` - -**Parametre** - -- `separator` — The separator which should contain exactly one character. [Dize](../../sql-reference/data-types/string.md). -- `s` — The string to split. [Dize](../../sql-reference/data-types/string.md). - -**Döndürülen değer (ler)** - -Seçili alt dizelerin bir dizisini döndürür. Boş alt dizeler şu durumlarda seçilebilir: - -- Dizenin başında veya sonunda bir ayırıcı oluşur; -- Birden fazla ardışık ayırıcı vardır; -- Orijinal dize `s` boş. - -Tür: [Dizi](../../sql-reference/data-types/array.md) -den [Dize](../../sql-reference/data-types/string.md). - -**Örnek** - -``` sql -SELECT splitByChar(',', '1,2,3,abcde') -``` - -``` text -┌─splitByChar(',', '1,2,3,abcde')─┐ -│ ['1','2','3','abcde'] │ -└─────────────────────────────────┘ -``` - -## splitByString (ayırıcı, s) {#splitbystringseparator-s} - -Bir dizeyi bir dizeyle ayrılmış alt dizelere böler. Sabit bir dize kullanır `separator` ayırıcı olarak birden fazla karakter. Eğer dize `separator` boş olduğunu, bu bölünmüş dize `s` tek karakter dizisine. - -**Sözdizimi** - -``` sql -splitByString(, ) -``` - -**Parametre** - -- `separator` — The separator. [Dize](../../sql-reference/data-types/string.md). -- `s` — The string to split. [Dize](../../sql-reference/data-types/string.md). - -**Döndürülen değer (ler)** - -Seçili alt dizelerin bir dizisini döndürür. Boş alt dizeler şu durumlarda seçilebilir: - -Tür: [Dizi](../../sql-reference/data-types/array.md) -den [Dize](../../sql-reference/data-types/string.md). - -- Boş olmayan bir ayırıcı dizenin başında veya sonunda oluşur; -- Birden fazla ardışık boş olmayan ayırıcı vardır; -- Orijinal dize `s` ayırıcı boş değilken boş. - -**Örnek** - -``` sql -SELECT splitByString(', ', '1, 2 3, 4,5, abcde') -``` - -``` text -┌─splitByString(', ', '1, 2 3, 4,5, abcde')─┐ -│ ['1','2 3','4,5','abcde'] │ -└───────────────────────────────────────────┘ -``` - -``` sql -SELECT splitByString('', 'abcde') -``` - -``` text -┌─splitByString('', 'abcde')─┐ -│ ['a','b','c','d','e'] │ -└────────────────────────────┘ -``` - -## arrayStringConcat(arr \[, ayırıcı\]) {#arraystringconcatarr-separator} - -Dizide listelenen dizeleri ayırıcı ile birleştirir.'separator' isteğe bağlı bir parametredir: varsayılan olarak boş bir dizeye ayarlanmış sabit bir dize. -Dizeyi döndürür. - -## alphaTokens (s) {#alphatokenss} - -A-z ve A-Z aralıklarından ardışık baytların alt dizelerini seçer. - -**Örnek** - -``` sql -SELECT alphaTokens('abca1abc') -``` - -``` text -┌─alphaTokens('abca1abc')─┐ -│ ['abca','abc'] │ -└─────────────────────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/splitting_merging_functions/) diff --git a/docs/tr/sql-reference/functions/string-functions.md b/docs/tr/sql-reference/functions/string-functions.md deleted file mode 100644 index b2a3a58f1f2..00000000000 --- a/docs/tr/sql-reference/functions/string-functions.md +++ /dev/null @@ -1,489 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 40 -toc_title: "Dizeleri ile \xE7al\u0131\u015Fma" ---- - -# Dizelerle çalışmak için işlevler {#functions-for-working-with-strings} - -## boş {#empty} - -Boş bir dize için 1 veya boş olmayan bir dize için 0 döndürür. -Sonuç türü Uint8'dir. -Bir boşluk veya boş bayt olsa bile, en az bir bayt içeriyorsa, bir dize boş olarak kabul edilir. -İşlev ayrıca diziler için de çalışır. - -## notEmpty {#notempty} - -Boş bir dize için 0 veya boş olmayan bir dize için 1 döndürür. -Sonuç türü Uint8'dir. -İşlev ayrıca diziler için de çalışır. - -## uzunluk {#length} - -Bir dizenin uzunluğunu bayt cinsinden döndürür (karakterlerde değil, kod noktalarında değil). -Sonuç türü Uint64'tür. -İşlev ayrıca diziler için de çalışır. - -## lengthUTF8 {#lengthutf8} - -Dizenin UTF-8 kodlanmış metni oluşturan bir bayt kümesi içerdiğini varsayarak, Unicode kod noktalarında (karakterlerde değil) bir dizenin uzunluğunu döndürür. Bu varsayım karşılanmazsa, bir sonuç döndürür (bir istisna atmaz). -Sonuç türü Uint64'tür. - -## char_length, CHAR_LENGTH {#char-length} - -Dizenin UTF-8 kodlanmış metni oluşturan bir bayt kümesi içerdiğini varsayarak, Unicode kod noktalarında (karakterlerde değil) bir dizenin uzunluğunu döndürür. Bu varsayım karşılanmazsa, bir sonuç döndürür (bir istisna atmaz). -Sonuç türü Uint64'tür. - -## character_length, CHARACTER_LENGTH {#character-length} - -Dizenin UTF-8 kodlanmış metni oluşturan bir bayt kümesi içerdiğini varsayarak, Unicode kod noktalarında (karakterlerde değil) bir dizenin uzunluğunu döndürür. Bu varsayım karşılanmazsa, bir sonuç döndürür (bir istisna atmaz). -Sonuç türü Uint64'tür. - -## alt, lcase {#lower} - -Bir dizedeki ASCII Latin sembollerini küçük harfe dönüştürür. - -## üst, ucase {#upper} - -Bir dizedeki ASCII Latin sembollerini büyük harfe dönüştürür. - -## lowerUTF8 {#lowerutf8} - -Dizenin UTF-8 kodlu bir metni oluşturan bir bayt kümesi içerdiğini varsayarak bir dizeyi küçük harfe dönüştürür. -Dili algılamaz. Yani Türkçe için sonuç tam olarak doğru olmayabilir. -UTF-8 bayt dizisinin uzunluğu bir kod noktasının büyük ve küçük harf için farklıysa, sonuç bu kod noktası için yanlış olabilir. -Dize, UTF-8 olmayan bir bayt kümesi içeriyorsa, davranış tanımsızdır. - -## upperUTF8 {#upperutf8} - -Dize, UTF-8 kodlanmış bir metni oluşturan bir bayt kümesi içerdiğini varsayarak bir dizeyi büyük harfe dönüştürür. -Dili algılamaz. Yani Türkçe için sonuç tam olarak doğru olmayabilir. -UTF-8 bayt dizisinin uzunluğu bir kod noktasının büyük ve küçük harf için farklıysa, sonuç bu kod noktası için yanlış olabilir. -Dize, UTF-8 olmayan bir bayt kümesi içeriyorsa, davranış tanımsızdır. - -## ısvalidutf8 {#isvalidutf8} - -Bayt kümesi geçerli UTF-8 kodlanmış, aksi takdirde 0 ise, 1 döndürür. - -## toValidUTF8 {#tovalidutf8} - -Geçersiz UTF-8 karakterlerini değiştirir `�` (U+FFFD) karakteri. Bir satırda çalışan tüm geçersiz karakterler bir yedek karaktere daraltılır. - -``` sql -toValidUTF8( input_string ) -``` - -Parametre: - -- input_string — Any set of bytes represented as the [Dize](../../sql-reference/data-types/string.md) veri türü nesnesi. - -Döndürülen değer: geçerli UTF-8 dizesi. - -**Örnek** - -``` sql -SELECT toValidUTF8('\x61\xF0\x80\x80\x80b') -``` - -``` text -┌─toValidUTF8('a����b')─┐ -│ a�b │ -└───────────────────────┘ -``` - -## tekrarlama {#repeat} - -Bir dizeyi belirtilen kadar çok tekrarlar ve çoğaltılmış değerleri tek bir dize olarak birleştirir. - -**Sözdizimi** - -``` sql -repeat(s, n) -``` - -**Parametre** - -- `s` — The string to repeat. [Dize](../../sql-reference/data-types/string.md). -- `n` — The number of times to repeat the string. [Uİnt](../../sql-reference/data-types/int-uint.md). - -**Döndürülen değer** - -Dize içeren tek dize `s` tekrarlanan `n` zamanlar. Eğer `n` \< 1, işlev boş dize döndürür. - -Tür: `String`. - -**Örnek** - -Sorgu: - -``` sql -SELECT repeat('abc', 10) -``` - -Sonuç: - -``` text -┌─repeat('abc', 10)──────────────┐ -│ abcabcabcabcabcabcabcabcabcabc │ -└────────────────────────────────┘ -``` - -## tersi {#reverse} - -Dizeyi tersine çevirir (bayt dizisi olarak). - -## reverseUTF8 {#reverseutf8} - -Dizenin UTF-8 metnini temsil eden bir bayt kümesi içerdiğini varsayarak bir Unicode kod noktası dizisini tersine çevirir. Aksi takdirde, başka bir şey yapar(bir istisna atmaz). - -## format(pattern, s0, s1, …) {#format} - -Bağımsız değişkenlerde listelenen dize ile sabit desen biçimlendirme. `pattern` basitleştirilmiş bir Python biçimi desenidir. Biçim dizesi içerir “replacement fields” kıvırcık parantez ile çevrili `{}`. Parantez içinde bulunmayan herhangi bir şey, çıktıya değişmeden kopyalanan hazır metin olarak kabul edilir. Literal metne bir ayraç karakteri eklemeniz gerekiyorsa, iki katına çıkararak kaçabilir: `{{ '{{' }}` ve `{{ '}}' }}`. Alan adları sayılar (sıfırdan başlayarak) veya boş olabilir (daha sonra sonuç numaraları olarak kabul edilir). - -``` sql -SELECT format('{1} {0} {1}', 'World', 'Hello') -``` - -``` text -┌─format('{1} {0} {1}', 'World', 'Hello')─┐ -│ Hello World Hello │ -└─────────────────────────────────────────┘ -``` - -``` sql -SELECT format('{} {}', 'Hello', 'World') -``` - -``` text -┌─format('{} {}', 'Hello', 'World')─┐ -│ Hello World │ -└───────────────────────────────────┘ -``` - -## concat {#concat} - -Bağımsız değişkenlerde listelenen dizeleri ayırıcı olmadan birleştirir. - -**Sözdizimi** - -``` sql -concat(s1, s2, ...) -``` - -**Parametre** - -String veya FixedString türünün değerleri. - -**Döndürülen değerler** - -Bağımsız değişkenlerin birleştirilmesinden kaynaklanan dizeyi döndürür. - -Argüman değerlerinden herhangi biri ise `NULL`, `concat` dönüşler `NULL`. - -**Örnek** - -Sorgu: - -``` sql -SELECT concat('Hello, ', 'World!') -``` - -Sonuç: - -``` text -┌─concat('Hello, ', 'World!')─┐ -│ Hello, World! │ -└─────────────────────────────┘ -``` - -## concatassumeınjective {#concatassumeinjective} - -Aynı olarak [concat](#concat) emin olun bu ihtiyaç fark var `concat(s1, s2, ...) → sn` enjekte edilir, grup tarafından optimizasyonu için kullanılacaktır. - -İşlev adlı “injective” bağımsız değişkenlerin farklı değerleri için her zaman farklı sonuç döndürürse. Başka bir deyişle: farklı argümanlar asla aynı sonucu vermez. - -**Sözdizimi** - -``` sql -concatAssumeInjective(s1, s2, ...) -``` - -**Parametre** - -String veya FixedString türünün değerleri. - -**Döndürülen değerler** - -Bağımsız değişkenlerin birleştirilmesinden kaynaklanan dizeyi döndürür. - -Argüman değerlerinden herhangi biri ise `NULL`, `concatAssumeInjective` dönüşler `NULL`. - -**Örnek** - -Giriş tablosu: - -``` sql -CREATE TABLE key_val(`key1` String, `key2` String, `value` UInt32) ENGINE = TinyLog; -INSERT INTO key_val VALUES ('Hello, ','World',1), ('Hello, ','World',2), ('Hello, ','World!',3), ('Hello',', World!',2); -SELECT * from key_val; -``` - -``` text -┌─key1────┬─key2─────┬─value─┐ -│ Hello, │ World │ 1 │ -│ Hello, │ World │ 2 │ -│ Hello, │ World! │ 3 │ -│ Hello │ , World! │ 2 │ -└─────────┴──────────┴───────┘ -``` - -Sorgu: - -``` sql -SELECT concat(key1, key2), sum(value) FROM key_val GROUP BY concatAssumeInjective(key1, key2) -``` - -Sonuç: - -``` text -┌─concat(key1, key2)─┬─sum(value)─┐ -│ Hello, World! │ 3 │ -│ Hello, World! │ 2 │ -│ Hello, World │ 3 │ -└────────────────────┴────────────┘ -``` - -## alt dize (s, ofset, uzunluk), orta (s, ofset, uzunluk), substr (s, ofset, uzunluk) {#substring} - -Bayttan başlayarak bir alt dize döndürür ‘offset’ ind thatex yani ‘length’ uzun bayt. Karakter indeksleme birinden başlar (standart SQL'DE olduğu gibi). Bu ‘offset’ ve ‘length’ bağımsız değişkenler sabit olmalıdır. - -## substringUTF8(s, ofset, uzunluk) {#substringutf8} - -Olarak aynı ‘substring’, ancak Unicode kod noktaları için. Dizenin UTF-8 kodlanmış bir metni temsil eden bir bayt kümesi içerdiği varsayımı altında çalışır. Bu varsayım karşılanmazsa, bir sonuç döndürür (bir istisna atmaz). - -## appendTrailingCharİfAbsent (s, c) {#appendtrailingcharifabsent} - -Eğer... ‘s’ dize boş değildir ve ‘c’ sonunda karakter, ekler ‘c’ sonuna kadar karakter. - -## convertCharset (s, from, to) {#convertcharset} - -Dize döndürür ‘s’ bu kodlamadan dönüştürüldü ‘from’ kod encodinglamaya ‘to’. - -## base64Encode (s) {#base64encode} - -Kodluyor ‘s’ Base64 içine dize - -## base64Decode (s) {#base64decode} - -Base64 kodlu dizeyi çözme ‘s’ orijinal dizeye. Başarısızlık durumunda bir istisna yükseltir. - -## tryBase64Decode (s) {#trybase64decode} - -Base64decode'a benzer, ancak hata durumunda boş bir dize döndürülür. - -## endsWith (s, sonek) {#endswith} - -Belirtilen sonek ile bitip bitmeyeceğini döndürür. Dize belirtilen sonek ile biterse 1 değerini döndürür, aksi takdirde 0 değerini döndürür. - -## startsWith (str, önek) {#startswith} - -Dize belirtilen önek ile başlayıp başlamadığını 1 döndürür, aksi halde 0 döndürür. - -``` sql -SELECT startsWith('Spider-Man', 'Spi'); -``` - -**Döndürülen değerler** - -- 1, dize belirtilen önek ile başlarsa. -- 0, dize belirtilen önek ile başlamazsa. - -**Örnek** - -Sorgu: - -``` sql -SELECT startsWith('Hello, world!', 'He'); -``` - -Sonuç: - -``` text -┌─startsWith('Hello, world!', 'He')─┐ -│ 1 │ -└───────────────────────────────────┘ -``` - -## kırpmak {#trim} - -Belirtilen tüm karakterleri bir dizenin başlangıcından veya sonundan kaldırır. -Varsayılan olarak, bir dizenin her iki ucundan ortak boşlukların (ASCII karakteri 32) tüm ardışık tekrarlarını kaldırır. - -**Sözdizimi** - -``` sql -trim([[LEADING|TRAILING|BOTH] trim_character FROM] input_string) -``` - -**Parametre** - -- `trim_character` — specified characters for trim. [Dize](../../sql-reference/data-types/string.md). -- `input_string` — string for trim. [Dize](../../sql-reference/data-types/string.md). - -**Döndürülen değer** - -Önde gelen ve (veya) belirtilen karakterleri izleyen bir dize. - -Tür: `String`. - -**Örnek** - -Sorgu: - -``` sql -SELECT trim(BOTH ' ()' FROM '( Hello, world! )') -``` - -Sonuç: - -``` text -┌─trim(BOTH ' ()' FROM '( Hello, world! )')─┐ -│ Hello, world! │ -└───────────────────────────────────────────────┘ -``` - -## trimLeft {#trimleft} - -Bir dizenin başlangıcından ortak boşluk (ASCII karakteri 32) tüm ardışık tekrarlarını kaldırır. Diğer boşluk karakterlerini (sekme, boşluksuz boşluk, vb.) kaldırmaz.). - -**Sözdizimi** - -``` sql -trimLeft(input_string) -``` - -Takma ad: `ltrim(input_string)`. - -**Parametre** - -- `input_string` — string to trim. [Dize](../../sql-reference/data-types/string.md). - -**Döndürülen değer** - -Bir dize olmadan lider ortak whitespaces. - -Tür: `String`. - -**Örnek** - -Sorgu: - -``` sql -SELECT trimLeft(' Hello, world! ') -``` - -Sonuç: - -``` text -┌─trimLeft(' Hello, world! ')─┐ -│ Hello, world! │ -└─────────────────────────────────────┘ -``` - -## trimRight {#trimright} - -Bir dizenin sonundan ortak boşluk (ASCII karakteri 32) tüm ardışık tekrarlarını kaldırır. Diğer boşluk karakterlerini (sekme, boşluksuz boşluk, vb.) kaldırmaz.). - -**Sözdizimi** - -``` sql -trimRight(input_string) -``` - -Takma ad: `rtrim(input_string)`. - -**Parametre** - -- `input_string` — string to trim. [Dize](../../sql-reference/data-types/string.md). - -**Döndürülen değer** - -Ortak whitespaces firar olmadan bir dize. - -Tür: `String`. - -**Örnek** - -Sorgu: - -``` sql -SELECT trimRight(' Hello, world! ') -``` - -Sonuç: - -``` text -┌─trimRight(' Hello, world! ')─┐ -│ Hello, world! │ -└──────────────────────────────────────┘ -``` - -## trimBoth {#trimboth} - -Bir dizenin her iki ucundan ortak boşluk (ASCII karakteri 32) tüm ardışık tekrarlarını kaldırır. Diğer boşluk karakterlerini (sekme, boşluksuz boşluk, vb.) kaldırmaz.). - -**Sözdizimi** - -``` sql -trimBoth(input_string) -``` - -Takma ad: `trim(input_string)`. - -**Parametre** - -- `input_string` — string to trim. [Dize](../../sql-reference/data-types/string.md). - -**Döndürülen değer** - -Bir dize olmadan lider ve sondaki ortak whitespaces. - -Tür: `String`. - -**Örnek** - -Sorgu: - -``` sql -SELECT trimBoth(' Hello, world! ') -``` - -Sonuç: - -``` text -┌─trimBoth(' Hello, world! ')─┐ -│ Hello, world! │ -└─────────────────────────────────────┘ -``` - -## CRC32 (s) {#crc32} - -CRC-32-IEEE 802.3 polinom ve başlangıç değerini kullanarak bir dizenin CRC32 sağlama toplamını döndürür `0xffffffff` (zlib uygulaması). - -Sonuç türü Uint32'dir. - -## Crc32ieee (s) {#crc32ieee} - -CRC-32-IEEE 802.3 polinomunu kullanarak bir dizenin CRC32 sağlama toplamını döndürür. - -Sonuç türü Uint32'dir. - -## CRC64 (s) {#crc64} - -CRC-64-ECMA polinomunu kullanarak bir dizenin CRC64 sağlama toplamını döndürür. - -Sonuç türü Uint64'tür. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/string_functions/) diff --git a/docs/tr/sql-reference/functions/string-replace-functions.md b/docs/tr/sql-reference/functions/string-replace-functions.md deleted file mode 100644 index 4e52c8ebfe5..00000000000 --- a/docs/tr/sql-reference/functions/string-replace-functions.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 42 -toc_title: "Dizelerde de\u011Fi\u015Ftirilmesi i\xE7in" ---- - -# Dizelerde arama ve değiştirme işlevleri {#functions-for-searching-and-replacing-in-strings} - -## replaceOne(Samanlık, desen, değiştirme) {#replaceonehaystack-pattern-replacement} - -Varsa, ilk oluş replacesumun yerini ‘pattern’ substring içinde ‘haystack’ ile... ‘replacement’ dize. -Ahiret, ‘pattern’ ve ‘replacement’ sabitleri olması gerekir. - -## replaceAll (Samanlık, desen, değiştirme), değiştirin (Samanlık, desen, değiştirme) {#replaceallhaystack-pattern-replacement-replacehaystack-pattern-replacement} - -Tüm oluşumları değiştirir ‘pattern’ substring içinde ‘haystack’ ile... ‘replacement’ dize. - -## replaceRegexpOne(Samanlık, desen, değiştirme) {#replaceregexponehaystack-pattern-replacement} - -Kullanarak değiştirme ‘pattern’ düzenli ifade. Re2 düzenli ifade. -Varsa, yalnızca ilk oluşumu değiştirir. -Bir desen olarak belirtilebilir ‘replacement’. Bu desen değiştirmeleri içerebilir `\0-\9`. -İkame `\0` tüm düzenli ifadeyi içerir. İkameler `\1-\9` alt desene karşılık gelir numbers.To use the `\` bir şablondaki karakter, kullanarak kaçış `\`. -Ayrıca, bir dize literalinin ekstra bir kaçış gerektirdiğini unutmayın. - -Örnek 1. Tarihi Amerikan format convertingına dönüştürme: - -``` sql -SELECT DISTINCT - EventDate, - replaceRegexpOne(toString(EventDate), '(\\d{4})-(\\d{2})-(\\d{2})', '\\2/\\3/\\1') AS res -FROM test.hits -LIMIT 7 -FORMAT TabSeparated -``` - -``` text -2014-03-17 03/17/2014 -2014-03-18 03/18/2014 -2014-03-19 03/19/2014 -2014-03-20 03/20/2014 -2014-03-21 03/21/2014 -2014-03-22 03/22/2014 -2014-03-23 03/23/2014 -``` - -Örnek 2. Bir dize on kez kopyalama: - -``` sql -SELECT replaceRegexpOne('Hello, World!', '.*', '\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0') AS res -``` - -``` text -┌─res────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ -│ Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World! │ -└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ -``` - -## replaceRegexpAll(Samanlık, desen, değiştirme) {#replaceregexpallhaystack-pattern-replacement} - -Bu aynı şeyi yapar, ancak tüm oluşumların yerini alır. Örnek: - -``` sql -SELECT replaceRegexpAll('Hello, World!', '.', '\\0\\0') AS res -``` - -``` text -┌─res────────────────────────┐ -│ HHeelllloo,, WWoorrlldd!! │ -└────────────────────────────┘ -``` - -Normal bir ifade boş bir alt dize üzerinde çalıştıysa, bir istisna olarak, değiştirme birden çok kez yapılmaz. -Örnek: - -``` sql -SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res -``` - -``` text -┌─res─────────────────┐ -│ here: Hello, World! │ -└─────────────────────┘ -``` - -## regexpQuoteMeta (s) {#regexpquotemetas} - -İşlev, dizedeki bazı önceden tanımlanmış karakterlerden önce bir ters eğik çizgi ekler. -Önceden tanımlanmış karakterler: ‘0’, ‘\\’, ‘\|’, ‘(’, ‘)’, ‘^’, ‘$’, ‘.’, ‘\[’, '\]', ‘?’, '\*‘,’+‘,’{‘,’:‘,’-'. -Bu uygulama biraz re2::RE2::QuoteMeta farklıdır. Sıfır bayttan 00 yerine \\0 olarak çıkar ve yalnızca gerekli karakterlerden kaçar. -Daha fazla bilgi için bağlantıya bakın: [RE2](https://github.com/google/re2/blob/master/re2/re2.cc#L473) - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/string_replace_functions/) diff --git a/docs/tr/sql-reference/functions/string-search-functions.md b/docs/tr/sql-reference/functions/string-search-functions.md deleted file mode 100644 index b80df910972..00000000000 --- a/docs/tr/sql-reference/functions/string-search-functions.md +++ /dev/null @@ -1,383 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 41 -toc_title: "Arama Dizeleri \u0130\xE7in" ---- - -# Dizeleri aramak için işlevler {#functions-for-searching-strings} - -Arama, tüm bu işlevlerde varsayılan olarak büyük / küçük harf duyarlıdır. Büyük / küçük harf duyarlı arama için ayrı Varyantlar vardır. - -## pozisyon (Samanlık, iğne), bulun (Samanlık, iğne) {#position} - -1'den başlayarak dizedeki bulunan alt dizenin konumunu (bayt cinsinden) döndürür. - -Dize, tek baytlık kodlanmış bir metni temsil eden bir bayt kümesi içerdiği varsayımı altında çalışır. Bu varsayım karşılanmazsa ve bir karakter tek bir bayt kullanılarak temsil edilemezse, işlev bir istisna atmaz ve beklenmeyen bir sonuç döndürür. Karakter iki bayt kullanılarak temsil edilebilirse, iki bayt vb. kullanır. - -Büyük / küçük harf duyarsız arama için işlevi kullanın [positionCaseİnsensitive](#positioncaseinsensitive). - -**Sözdizimi** - -``` sql -position(haystack, needle[, start_pos]) -``` - -Takma ad: `locate(haystack, needle[, start_pos])`. - -**Parametre** - -- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal). -- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal). -- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md) - -**Döndürülen değerler** - -- Alt dize bulunursa, bayt cinsinden başlangıç pozisyonu (1'den sayma). -- 0, alt dize bulunamadı. - -Tür: `Integer`. - -**Örnekler** - -İfade “Hello, world!” tek baytla kodlanmış bir metni temsil eden bir bayt kümesi içerir. İşlev beklenen bazı sonuçları döndürür: - -Sorgu: - -``` sql -SELECT position('Hello, world!', '!') -``` - -Sonuç: - -``` text -┌─position('Hello, world!', '!')─┐ -│ 13 │ -└────────────────────────────────┘ -``` - -Rusça'daki aynı ifade, tek bir bayt kullanılarak temsil edilemeyen karakterler içerir. İşlev beklenmedik bir sonuç verir (kullanım [positionUTF8](#positionutf8) çok bayt kodlu metin için işlev): - -Sorgu: - -``` sql -SELECT position('Привет, мир!', '!') -``` - -Sonuç: - -``` text -┌─position('Привет, мир!', '!')─┐ -│ 21 │ -└───────────────────────────────┘ -``` - -## positionCaseİnsensitive {#positioncaseinsensitive} - -Olarak aynı [konum](#position) 1'den başlayarak dizedeki bulunan alt dizenin konumunu (bayt cinsinden) döndürür. Büyük / küçük harf duyarlı bir arama için işlevi kullanın. - -Dize, tek baytlık kodlanmış bir metni temsil eden bir bayt kümesi içerdiği varsayımı altında çalışır. Bu varsayım karşılanmazsa ve bir karakter tek bir bayt kullanılarak temsil edilemezse, işlev bir istisna atmaz ve beklenmeyen bir sonuç döndürür. Karakter iki bayt kullanılarak temsil edilebilirse, iki bayt vb. kullanır. - -**Sözdizimi** - -``` sql -positionCaseInsensitive(haystack, needle[, start_pos]) -``` - -**Parametre** - -- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal). -- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal). -- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md) - -**Döndürülen değerler** - -- Alt dize bulunursa, bayt cinsinden başlangıç pozisyonu (1'den sayma). -- 0, alt dize bulunamadı. - -Tür: `Integer`. - -**Örnek** - -Sorgu: - -``` sql -SELECT positionCaseInsensitive('Hello, world!', 'hello') -``` - -Sonuç: - -``` text -┌─positionCaseInsensitive('Hello, world!', 'hello')─┐ -│ 1 │ -└───────────────────────────────────────────────────┘ -``` - -## positionUTF8 {#positionutf8} - -1'den başlayarak dizedeki bulunan alt dizenin konumunu (Unicode noktalarında) döndürür. - -Dizenin UTF-8 kodlanmış bir metni temsil eden bir bayt kümesi içerdiği varsayımı altında çalışır. Bu varsayım karşılanmazsa, işlev bir istisna atmaz ve beklenmeyen bir sonuç döndürür. Karakter iki Unicode noktası kullanılarak temsil edilebilirse, iki vb. kullanır. - -Büyük / küçük harf duyarsız arama için işlevi kullanın [positionCaseİnsensitiveUTF8](#positioncaseinsensitiveutf8). - -**Sözdizimi** - -``` sql -positionUTF8(haystack, needle[, start_pos]) -``` - -**Parametre** - -- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal). -- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal). -- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md) - -**Döndürülen değerler** - -- Unicode noktalarında başlangıç pozisyonu (1'den sayma), eğer alt dize bulundu. -- 0, alt dize bulunamadı. - -Tür: `Integer`. - -**Örnekler** - -İfade “Hello, world!” rusça'da, tek noktalı kodlanmış bir metni temsil eden bir dizi Unicode noktası bulunur. İşlev beklenen bazı sonuçları döndürür: - -Sorgu: - -``` sql -SELECT positionUTF8('Привет, мир!', '!') -``` - -Sonuç: - -``` text -┌─positionUTF8('Привет, мир!', '!')─┐ -│ 12 │ -└───────────────────────────────────┘ -``` - -İfade “Salut, étudiante!” karakter nerede `é` bir nokta kullanılarak temsil edilebilir (`U+00E9`) veya iki puan (`U+0065U+0301`) fonksiyon bazı beklenmedik sonuç iade edilebilir: - -Mektup için sorgu `é` bir Unicode noktasını temsil eden `U+00E9`: - -``` sql -SELECT positionUTF8('Salut, étudiante!', '!') -``` - -Sonuç: - -``` text -┌─positionUTF8('Salut, étudiante!', '!')─┐ -│ 17 │ -└────────────────────────────────────────┘ -``` - -Mektup için sorgu `é`, iki Unicode noktası temsil edilen `U+0065U+0301`: - -``` sql -SELECT positionUTF8('Salut, étudiante!', '!') -``` - -Sonuç: - -``` text -┌─positionUTF8('Salut, étudiante!', '!')─┐ -│ 18 │ -└────────────────────────────────────────┘ -``` - -## positionCaseİnsensitiveUTF8 {#positioncaseinsensitiveutf8} - -Olarak aynı [positionUTF8](#positionutf8) ama büyük küçük harf duyarlı. 1'den başlayarak dizedeki bulunan alt dizenin konumunu (Unicode noktalarında) döndürür. - -Dizenin UTF-8 kodlanmış bir metni temsil eden bir bayt kümesi içerdiği varsayımı altında çalışır. Bu varsayım karşılanmazsa, işlev bir istisna atmaz ve beklenmeyen bir sonuç döndürür. Karakter iki Unicode noktası kullanılarak temsil edilebilirse, iki vb. kullanır. - -**Sözdizimi** - -``` sql -positionCaseInsensitiveUTF8(haystack, needle[, start_pos]) -``` - -**Parametre** - -- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal). -- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal). -- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md) - -**Döndürülen değer** - -- Unicode noktalarında başlangıç pozisyonu (1'den sayma), eğer alt dize bulundu. -- 0, alt dize bulunamadı. - -Tür: `Integer`. - -**Örnek** - -Sorgu: - -``` sql -SELECT positionCaseInsensitiveUTF8('Привет, мир!', 'Мир') -``` - -Sonuç: - -``` text -┌─positionCaseInsensitiveUTF8('Привет, мир!', 'Мир')─┐ -│ 9 │ -└────────────────────────────────────────────────────┘ -``` - -## multiSearchAllPositions {#multisearchallpositions} - -Olarak aynı [konum](string-search-functions.md#position) ama döner `Array` dizede bulunan karşılık gelen alt dizelerin konumlarının (bayt cinsinden). Pozisyonlar 1'den başlayarak endekslenir. - -Arama, dize kodlaması ve harmanlama ile ilgili olmayan bayt dizileri üzerinde gerçekleştirilir. - -- Büyük / küçük harf duyarlı ASCII arama için işlevi kullanın `multiSearchAllPositionsCaseInsensitive`. -- UTF-8'de arama yapmak için işlevi kullanın [multiSearchAllPositionsUTF8](#multiSearchAllPositionsUTF8). -- Büyük / küçük harf duyarlı UTF-8 arama için multisearchallpositionscaseınsensitiveutf8 işlevini kullanın. - -**Sözdizimi** - -``` sql -multiSearchAllPositions(haystack, [needle1, needle2, ..., needlen]) -``` - -**Parametre** - -- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal). -- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal). - -**Döndürülen değerler** - -- Bayt cinsinden başlangıç pozisyonları dizisi (1'den sayma), karşılık gelen alt dize bulunursa ve 0 bulunmazsa. - -**Örnek** - -Sorgu: - -``` sql -SELECT multiSearchAllPositions('Hello, World!', ['hello', '!', 'world']) -``` - -Sonuç: - -``` text -┌─multiSearchAllPositions('Hello, World!', ['hello', '!', 'world'])─┐ -│ [0,13,0] │ -└───────────────────────────────────────────────────────────────────┘ -``` - -## multiSearchAllPositionsUTF8 {#multiSearchAllPositionsUTF8} - -Görmek `multiSearchAllPositions`. - -## multiSearchFirstPosition (Samanlık, \[iğne1, iğne2, …, needleve\]) {#multisearchfirstposition} - -Olarak aynı `position` ancak dizenin en soldaki ofsetini döndürür `haystack` bu bazı iğnelerle eşleşti. - -Büyük/küçük harfe duyarsız arama veya / VE UTF-8 biçiminde kullanım işlevleri için `multiSearchFirstPositionCaseInsensitive, multiSearchFirstPositionUTF8, multiSearchFirstPositionCaseInsensitiveUTF8`. - -## multiSearchFirstİndex (Samanlık, \[iğne1, iğne2, …, needleve\]) {#multisearchfirstindexhaystack-needle1-needle2-needlen} - -Dizini döndürür `i` en soldaki bulunan iğnenin (1'den başlayarak)ben diz inede `haystack` ve 0 aksi takdirde. - -Büyük/küçük harfe duyarsız arama veya / VE UTF-8 biçiminde kullanım işlevleri için `multiSearchFirstIndexCaseInsensitive, multiSearchFirstIndexUTF8, multiSearchFirstIndexCaseInsensitiveUTF8`. - -## multiSearchAny (Samanlık, \[iğne1, iğne2, …, needleve\]) {#function-multisearchany} - -Döner 1, Eğer en az bir dize iğneben dize ile eşleşir `haystack` ve 0 aksi takdirde. - -Büyük/küçük harfe duyarsız arama veya / VE UTF-8 biçiminde kullanım işlevleri için `multiSearchAnyCaseInsensitive, multiSearchAnyUTF8, multiSearchAnyCaseInsensitiveUTF8`. - -!!! note "Not" - Tamamı `multiSearch*` fonksiyonlar iğne sayısı 2'den az olmalıdır8 uygulama şartname nedeniyle. - -## maç (Samanlık, desen) {#matchhaystack-pattern} - -Dize eşleşip eşleşmediğini denetler `pattern` düzenli ifade. Bir `re2` düzenli ifade. Bu [sözdizimi](https://github.com/google/re2/wiki/Syntax) of the `re2` düzenli ifadeler, Perl düzenli ifadelerin sözdiziminden daha sınırlıdır. - -Eşleşmezse 0 veya eşleşirse 1 değerini döndürür. - -Ters eğik çizgi sembolünün (`\`) normal ifadede kaçmak için kullanılır. Aynı sembol, dize değişmezlerinde kaçmak için kullanılır. Bu nedenle, normal bir ifadede sembolden kaçmak için, bir dize literalinde iki ters eğik çizgi (\\) yazmanız gerekir. - -Normal ifade, bir bayt kümesiymiş gibi dizeyle çalışır. Normal ifade boş bayt içeremez. -Bir dizedeki alt dizeleri aramak için desenler için, LİKE veya ‘position’, çok daha hızlı çalıştıkları için. - -## multiMatchAny (Samanlık, \[desen1, desen2, …, patternve\]) {#multimatchanyhaystack-pattern1-pattern2-patternn} - -Olarak aynı `match`, ancak normal ifadelerin hiçbiri eşleşmezse 0 ve desenlerden herhangi biri eşleşirse 1 değerini döndürür. Kullanır [hyperscan](https://github.com/intel/hyperscan) kitaplık. Bir dizede alt dizeleri aramak için desenler için, kullanmak daha iyidir `multiSearchAny` çok daha hızlı çalıştığı için. - -!!! note "Not" - Herhangi birinin uzunluğu `haystack` dize 2'den az olmalıdır32 bayt aksi takdirde özel durum atılır. Bu kısıtlama, hyperscan API nedeniyle gerçekleşir. - -## multimatchanyındex (haystack, \[desen1, desen2, …, patternve\]) {#multimatchanyindexhaystack-pattern1-pattern2-patternn} - -Olarak aynı `multiMatchAny`, ancak Samanlık eşleşen herhangi bir dizin döndürür. - -## multiMatchAllİndices (haystack, \[desen1, desen2, …, patternve\]) {#multimatchallindiceshaystack-pattern1-pattern2-patternn} - -Olarak aynı `multiMatchAny`, ancak herhangi bir sırada Samanlık eşleşen tüm indicies dizisini döndürür. - -## multiFuzzyMatchAny (Samanlık, mesafe, \[desen1, desen2, …, patternve\]) {#multifuzzymatchanyhaystack-distance-pattern1-pattern2-patternn} - -Olarak aynı `multiMatchAny`, ancak herhangi bir desen samanlıkta bir sabitle eşleşirse 1 döndürür [mesafeyi Düzenle](https://en.wikipedia.org/wiki/Edit_distance). Bu fonksiyon aynı zamanda deneysel bir moddadır ve son derece yavaş olabilir. Daha fazla bilgi için bkz. [hyperscan belgeleri](https://intel.github.io/hyperscan/dev-reference/compilation.html#approximate-matching). - -## multifuzzymatchanyındex (Samanlık, mesafe, \[desen1, desen2, …, patternve\]) {#multifuzzymatchanyindexhaystack-distance-pattern1-pattern2-patternn} - -Olarak aynı `multiFuzzyMatchAny`, ancak sabit bir düzenleme mesafesi içinde Samanlık eşleşen herhangi bir dizin döndürür. - -## multiFuzzyMatchAllİndices (Samanlık, mesafe, \[desen1, desen2, …, patternve\]) {#multifuzzymatchallindiceshaystack-distance-pattern1-pattern2-patternn} - -Olarak aynı `multiFuzzyMatchAny`, ancak sabit bir düzenleme mesafesi içinde saman yığını ile eşleşen herhangi bir sırada tüm dizinlerin dizisini döndürür. - -!!! note "Not" - `multiFuzzyMatch*` işlevler UTF-8 normal ifadeleri desteklemez ve bu tür ifadeler hyperscan kısıtlaması nedeniyle bayt olarak kabul edilir. - -!!! note "Not" - Hyperscan kullanan tüm işlevleri kapatmak için, ayarı kullanın `SET allow_hyperscan = 0;`. - -## özü (Samanlık, desen) {#extracthaystack-pattern} - -Normal ifade kullanarak bir dize parçasını ayıklar. Eğer ‘haystack’ eşleşmiyor ‘pattern’ regex, boş bir dize döndürülür. Regex alt desenler içermiyorsa, tüm regex ile eşleşen parçayı alır. Aksi takdirde, ilk alt desenle eşleşen parçayı alır. - -## extractAll(Samanlık, desen) {#extractallhaystack-pattern} - -Normal bir ifade kullanarak bir dizenin tüm parçalarını ayıklar. Eğer ‘haystack’ eşleşmiyor ‘pattern’ regex, boş bir dize döndürülür. Regex için tüm eşleşmelerden oluşan bir dizi dizeyi döndürür. Genel olarak, davranış ile aynıdır ‘extract’ işlev (bir alt desen yoksa ilk alt deseni veya tüm ifadeyi alır). - -## gibi (Samanlık, desen), Samanlık gibi desen operatörü {#function-like} - -Bir dizenin basit bir normal ifadeyle eşleşip eşleşmediğini denetler. -Normal ifade metasymbols içerebilir `%` ve `_`. - -`%` herhangi bir bayt miktarını (sıfır karakter dahil) gösterir. - -`_` herhangi bir bayt gösterir. - -Ters eğik çizgi kullanın (`\`) metasimbollerden kaçmak için. Açıklamasında kaçan nota bakın ‘match’ İşlev. - -Gibi düzenli ifadeler için `%needle%`, kod daha optimal ve hızlı olarak çalışır `position` İşlev. -Diğer normal ifadeler için kod, ‘match’ İşlev. - -## notLike (Samanlık, desen), Samanlık desen operatörü gibi değil {#function-notlike} - -Aynı şey ‘like’ ama negatif. - -## ngramDistance(Samanlık, iğne) {#ngramdistancehaystack-needle} - -Arasındaki 4 gram distancelık mesaf theeyi hesaplar `haystack` ve `needle`: counts the symmetric difference between two multisets of 4-grams and normalizes it by the sum of their cardinalities. Returns float number from 0 to 1 – the closer to zero, the more strings are similar to each other. If the constant `needle` veya `haystack` 32kb'den fazla, bir istisna atar. Sabit olmayan bazı `haystack` veya `needle` dizeler 32kb'den daha fazladır, mesafe her zaman birdir. - -Büyük/küçük harf duyarsız arama veya / VE UTF-8 formatında kullanım işlevleri için `ngramDistanceCaseInsensitive, ngramDistanceUTF8, ngramDistanceCaseInsensitiveUTF8`. - -## ngramsearch(Samanlık, iğne) {#ngramsearchhaystack-needle} - -Aynı olarak `ngramDistance` ama arasındaki simetrik olmayan farkı hesaplar `needle` ve `haystack` – the number of n-grams from needle minus the common number of n-grams normalized by the number of `needle` n-büyükanne. Daha yakın, daha `needle` is in the `haystack`. Bulanık dize arama için yararlı olabilir. - -Büyük/küçük harf duyarsız arama veya / VE UTF-8 formatında kullanım işlevleri için `ngramSearchCaseInsensitive, ngramSearchUTF8, ngramSearchCaseInsensitiveUTF8`. - -!!! note "Not" - For UTF-8 case we use 3-gram distance. All these are not perfectly fair n-gram distances. We use 2-byte hashes to hash n-grams and then calculate the (non-)symmetric difference between these hash tables – collisions may occur. With UTF-8 case-insensitive format we do not use fair `tolower` function – we zero the 5-th bit (starting from zero) of each codepoint byte and first bit of zeroth byte if bytes more than one – this works for Latin and mostly for all Cyrillic letters. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/string_search_functions/) diff --git a/docs/tr/sql-reference/functions/type-conversion-functions.md b/docs/tr/sql-reference/functions/type-conversion-functions.md deleted file mode 100644 index c767d0e8063..00000000000 --- a/docs/tr/sql-reference/functions/type-conversion-functions.md +++ /dev/null @@ -1,534 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 38 -toc_title: "Tip D\xF6n\xFC\u015Ft\xFCrme" ---- - -# Tip Dönüştürme Fonksiyonları {#type-conversion-functions} - -## Sayısal dönüşümlerin ortak sorunları {#numeric-conversion-issues} - -Bir değeri birinden başka bir veri türüne dönüştürdüğünüzde, ortak durumda, veri kaybına neden olabilecek güvenli olmayan bir işlem olduğunu unutmamalısınız. Değeri daha büyük bir veri türünden daha küçük bir veri türüne sığdırmaya çalışırsanız veya değerleri farklı veri türleri arasında dönüştürürseniz, veri kaybı oluşabilir. - -ClickHouse vardır [C++ programları ile aynı davranış](https://en.cppreference.com/w/cpp/language/implicit_conversion). - -## toİnt(8/16/32/64) {#toint8163264} - -Bir giriş değeri dönüştürür [Tamsayı](../../sql-reference/data-types/int-uint.md) veri türü. Bu işlev ailesi şunları içerir: - -- `toInt8(expr)` — Results in the `Int8` veri türü. -- `toInt16(expr)` — Results in the `Int16` veri türü. -- `toInt32(expr)` — Results in the `Int32` veri türü. -- `toInt64(expr)` — Results in the `Int64` veri türü. - -**Parametre** - -- `expr` — [İfade](../syntax.md#syntax-expressions) bir sayının ondalık gösterimiyle bir sayı veya dize döndürülmesi. Sayıların ikili, sekizli ve onaltılık gösterimleri desteklenmez. Önde gelen sıfırlar soyulur. - -**Döndürülen değer** - -Tamsayı değeri `Int8`, `Int16`, `Int32`, veya `Int64` veri türü. - -Fonksiyonlar kullanımı [sıfıra doğru yuvarlama](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), yani sayıların kesirli rakamlarını keserler. - -Fonksiyon behaviorların davranışı [N andan ve In andf](../../sql-reference/data-types/float.md#data_type-float-nan-inf) argümanlar tanımsızdır. Hakkında hatırla [sayısal convertions sorunları](#numeric-conversion-issues), fonksiyonları kullanırken. - -**Örnek** - -``` sql -SELECT toInt64(nan), toInt32(32), toInt16('16'), toInt8(8.8) -``` - -``` text -┌─────────toInt64(nan)─┬─toInt32(32)─┬─toInt16('16')─┬─toInt8(8.8)─┐ -│ -9223372036854775808 │ 32 │ 16 │ 8 │ -└──────────────────────┴─────────────┴───────────────┴─────────────┘ -``` - -## toİnt (8/16/32/64)OrZero {#toint8163264orzero} - -String türünde bir argüman alır ve İnt içine ayrıştırmaya çalışır(8 \| 16 \| 32 \| 64). Başarısız olursa, 0 döndürür. - -**Örnek** - -``` sql -select toInt64OrZero('123123'), toInt8OrZero('123qwe123') -``` - -``` text -┌─toInt64OrZero('123123')─┬─toInt8OrZero('123qwe123')─┐ -│ 123123 │ 0 │ -└─────────────────────────┴───────────────────────────┘ -``` - -## toİnt(8/16/32/64) OrNull {#toint8163264ornull} - -String türünde bir argüman alır ve İnt içine ayrıştırmaya çalışır(8 \| 16 \| 32 \| 64). Başarısız olursa, NULL döndürür. - -**Örnek** - -``` sql -select toInt64OrNull('123123'), toInt8OrNull('123qwe123') -``` - -``` text -┌─toInt64OrNull('123123')─┬─toInt8OrNull('123qwe123')─┐ -│ 123123 │ ᴺᵁᴸᴸ │ -└─────────────────────────┴───────────────────────────┘ -``` - -## toUİnt(8/16/32/64) {#touint8163264} - -Bir giriş değeri dönüştürür [Uİnt](../../sql-reference/data-types/int-uint.md) veri türü. Bu işlev ailesi şunları içerir: - -- `toUInt8(expr)` — Results in the `UInt8` veri türü. -- `toUInt16(expr)` — Results in the `UInt16` veri türü. -- `toUInt32(expr)` — Results in the `UInt32` veri türü. -- `toUInt64(expr)` — Results in the `UInt64` veri türü. - -**Parametre** - -- `expr` — [İfade](../syntax.md#syntax-expressions) bir sayının ondalık gösterimiyle bir sayı veya dize döndürülmesi. Sayıların ikili, sekizli ve onaltılık gösterimleri desteklenmez. Önde gelen sıfırlar soyulur. - -**Döndürülen değer** - -Tamsayı değeri `UInt8`, `UInt16`, `UInt32`, veya `UInt64` veri türü. - -Fonksiyonlar kullanımı [sıfıra doğru yuvarlama](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), yani sayıların kesirli rakamlarını keserler. - -Olumsuz agruments için işlevlerin davranışı ve [N andan ve In andf](../../sql-reference/data-types/float.md#data_type-float-nan-inf) argümanlar tanımsızdır. Örneğin, negatif bir sayı ile bir dize geçirirseniz `'-32'`, ClickHouse bir özel durum yükseltir. Hakkında hatırla [sayısal convertions sorunları](#numeric-conversion-issues), fonksiyonları kullanırken. - -**Örnek** - -``` sql -SELECT toUInt64(nan), toUInt32(-32), toUInt16('16'), toUInt8(8.8) -``` - -``` text -┌───────toUInt64(nan)─┬─toUInt32(-32)─┬─toUInt16('16')─┬─toUInt8(8.8)─┐ -│ 9223372036854775808 │ 4294967264 │ 16 │ 8 │ -└─────────────────────┴───────────────┴────────────────┴──────────────┘ -``` - -## toUİnt (8/16/32/64)OrZero {#touint8163264orzero} - -## toUİnt(8/16/32/64) OrNull {#touint8163264ornull} - -## toFloat(32/64) {#tofloat3264} - -## toFloat (32/64)OrZero {#tofloat3264orzero} - -## toFloat(32/64) OrNull {#tofloat3264ornull} - -## toDate {#todate} - -## toDateOrZero {#todateorzero} - -## toDateOrNull {#todateornull} - -## toDateTime {#todatetime} - -## toDateTimeOrZero {#todatetimeorzero} - -## toDateTimeOrNull {#todatetimeornull} - -## toDecimal(32/64/128) {#todecimal3264128} - -Dönüşüyo `value` to the [Ondalık](../../sql-reference/data-types/decimal.md) hassas veri türü `S`. Bu `value` bir sayı veya bir dize olabilir. Bu `S` (scale) parametresi ondalık basamak sayısını belirtir. - -- `toDecimal32(value, S)` -- `toDecimal64(value, S)` -- `toDecimal128(value, S)` - -## toDecimal(32/64/128) OrNull {#todecimal3264128ornull} - -Bir giriş dizesini bir [Nullable (Ondalık (P, S))](../../sql-reference/data-types/decimal.md) veri türü değeri. Bu işlev ailesi şunları içerir: - -- `toDecimal32OrNull(expr, S)` — Results in `Nullable(Decimal32(S))` veri türü. -- `toDecimal64OrNull(expr, S)` — Results in `Nullable(Decimal64(S))` veri türü. -- `toDecimal128OrNull(expr, S)` — Results in `Nullable(Decimal128(S))` veri türü. - -Bu işlevler yerine kullanılmalıdır `toDecimal*()` fonksiyonlar, eğer bir almak için tercih `NULL` bir giriş değeri ayrıştırma hatası durumunda bir özel durum yerine değer. - -**Parametre** - -- `expr` — [İfade](../syntax.md#syntax-expressions) bir değeri döndürür [Dize](../../sql-reference/data-types/string.md) veri türü. ClickHouse ondalık sayının metinsel temsilini bekler. Mesela, `'1.111'`. -- `S` — Scale, the number of decimal places in the resulting value. - -**Döndürülen değer** - -İçinde bir değer `Nullable(Decimal(P,S))` veri türü. Değeri içerir: - -- İle sayı `S` ondalık basamaklar, ClickHouse giriş dizesi bir sayı olarak yorumlar. -- `NULL`, ClickHouse giriş dizesini bir sayı olarak yorumlayamazsa veya giriş numarası birden fazla içeriyorsa `S` ondalık basamaklar. - -**Örnekler** - -``` sql -SELECT toDecimal32OrNull(toString(-1.111), 5) AS val, toTypeName(val) -``` - -``` text -┌──────val─┬─toTypeName(toDecimal32OrNull(toString(-1.111), 5))─┐ -│ -1.11100 │ Nullable(Decimal(9, 5)) │ -└──────────┴────────────────────────────────────────────────────┘ -``` - -``` sql -SELECT toDecimal32OrNull(toString(-1.111), 2) AS val, toTypeName(val) -``` - -``` text -┌──val─┬─toTypeName(toDecimal32OrNull(toString(-1.111), 2))─┐ -│ ᴺᵁᴸᴸ │ Nullable(Decimal(9, 2)) │ -└──────┴────────────────────────────────────────────────────┘ -``` - -## toDecimal (32/64/128)OrZero {#todecimal3264128orzero} - -Bir giriş değeri dönüştürür [Ondalık(P, S)](../../sql-reference/data-types/decimal.md) veri türü. Bu işlev ailesi şunları içerir: - -- `toDecimal32OrZero( expr, S)` — Results in `Decimal32(S)` veri türü. -- `toDecimal64OrZero( expr, S)` — Results in `Decimal64(S)` veri türü. -- `toDecimal128OrZero( expr, S)` — Results in `Decimal128(S)` veri türü. - -Bu işlevler yerine kullanılmalıdır `toDecimal*()` fonksiyonlar, eğer bir almak için tercih `0` bir giriş değeri ayrıştırma hatası durumunda bir özel durum yerine değer. - -**Parametre** - -- `expr` — [İfade](../syntax.md#syntax-expressions) bir değeri döndürür [Dize](../../sql-reference/data-types/string.md) veri türü. ClickHouse ondalık sayının metinsel temsilini bekler. Mesela, `'1.111'`. -- `S` — Scale, the number of decimal places in the resulting value. - -**Döndürülen değer** - -İçinde bir değer `Nullable(Decimal(P,S))` veri türü. Değeri içerir: - -- İle sayı `S` ondalık basamaklar, ClickHouse giriş dizesi bir sayı olarak yorumlar. -- 0 ile `S` ondalık basamaklar, ClickHouse giriş dizesini bir sayı olarak yorumlayamazsa veya giriş numarası birden fazla içeriyorsa `S` ondalık basamaklar. - -**Örnek** - -``` sql -SELECT toDecimal32OrZero(toString(-1.111), 5) AS val, toTypeName(val) -``` - -``` text -┌──────val─┬─toTypeName(toDecimal32OrZero(toString(-1.111), 5))─┐ -│ -1.11100 │ Decimal(9, 5) │ -└──────────┴────────────────────────────────────────────────────┘ -``` - -``` sql -SELECT toDecimal32OrZero(toString(-1.111), 2) AS val, toTypeName(val) -``` - -``` text -┌──val─┬─toTypeName(toDecimal32OrZero(toString(-1.111), 2))─┐ -│ 0.00 │ Decimal(9, 2) │ -└──────┴────────────────────────────────────────────────────┘ -``` - -## toString {#tostring} - -Sayılar, dizeler (ancak sabit olmayan dizeler), tarihler ve tarihlerle saatler arasında dönüştürme işlevleri. -Tüm bu işlevler bir argümanı kabul eder. - -Bir dizeye veya dizeye dönüştürürken, değer, sekmeyle aynı kuralları kullanarak biçimlendirilir veya ayrıştırılır. ayrı biçim (ve hemen hemen tüm diğer metin biçimleri). Dize ayrıştırılamazsa, bir istisna atılır ve istek iptal edilir. - -Tarihleri sayılara dönüştürürken veya tam tersi, Tarih Unix döneminin başlangıcından bu yana geçen gün sayısına karşılık gelir. -Tarihleri zamanlarla sayılara dönüştürürken veya tam tersi olduğunda, zaman ile tarih, Unix döneminin başlangıcından bu yana geçen saniye sayısına karşılık gelir. - -ToDate / toDateTime işlevleri için tarih ve saatli tarih biçimleri aşağıdaki gibi tanımlanır: - -``` text -YYYY-MM-DD -YYYY-MM-DD hh:mm:ss -``` - -Özel durum olarak, uınt32, Int32, Uınt64 veya Int64 sayısal türlerinden bugüne dönüştürme ve sayı 65536'dan büyük veya eşitse, sayı Unıx zaman damgası (ve gün sayısı olarak değil) olarak yorumlanır ve tarihe yuvarlanır. Bu, yaygın yazı oluşumu için destek sağlar ‘toDate(unix_timestamp)’, aksi takdirde bir hata olur ve daha hantal yazmayı gerektirir ‘toDate(toDateTime(unix_timestamp))’. - -Bir tarih ve tarih ile saat arasında dönüştürme doğal bir şekilde gerçekleştirilir: boş bir zaman ekleyerek veya saati bırakarak. - -Sayısal türler arasındaki dönüştürme, C++ ' daki farklı sayısal türler arasındaki atamalarla aynı kuralları kullanır. - -Ayrıca, Tostring işlevi DateTime bağımsız değişkeni, saat dilimi adını içeren ikinci bir dize bağımsız değişkeni alabilir. Örnek: `Asia/Yekaterinburg` Bu durumda, saat belirtilen saat dilimine göre biçimlendirilir. - -``` sql -SELECT - now() AS now_local, - toString(now(), 'Asia/Yekaterinburg') AS now_yekat -``` - -``` text -┌───────────now_local─┬─now_yekat───────────┐ -│ 2016-06-15 00:11:21 │ 2016-06-15 02:11:21 │ -└─────────────────────┴─────────────────────┘ -``` - -Ayrıca bakınız `toUnixTimestamp` İşlev. - -## toFixedString(s, N) {#tofixedstrings-n} - -Bir dize türü bağımsız değişkeni dönüştürür bir FixedString(N) türü (sabit uzunlukta bir dize N). N sabit olmalıdır. -Dize n'den daha az bayt varsa, sağa boş bayt ile doldurulur. Dize n'den daha fazla bayt varsa, bir özel durum atılır. - -## tostringcuttozero (s) {#tostringcuttozeros} - -Bir dize veya fixedstring bağımsız değişkeni kabul eder. Bulunan ilk sıfır baytta kesilmiş içeriği olan dizeyi döndürür. - -Örnek: - -``` sql -SELECT toFixedString('foo', 8) AS s, toStringCutToZero(s) AS s_cut -``` - -``` text -┌─s─────────────┬─s_cut─┐ -│ foo\0\0\0\0\0 │ foo │ -└───────────────┴───────┘ -``` - -``` sql -SELECT toFixedString('foo\0bar', 8) AS s, toStringCutToZero(s) AS s_cut -``` - -``` text -┌─s──────────┬─s_cut─┐ -│ foo\0bar\0 │ foo │ -└────────────┴───────┘ -``` - -## reinterpretAsUİnt(8/16/32/64) {#reinterpretasuint8163264} - -## reinterpretAsİnt(8/16/32/64) {#reinterpretasint8163264} - -## reinterpretAsFloat (32/64) {#reinterpretasfloat3264} - -## reinterpretAsDate {#reinterpretasdate} - -## reinterpretAsDateTime {#reinterpretasdatetime} - -Bu işlevler bir dizeyi kabul eder ve dizenin başına yerleştirilen baytları ana bilgisayar düzeninde (little endian) bir sayı olarak yorumlar. Dize yeterince uzun değilse, işlevler dize gerekli sayıda boş baytla doldurulmuş gibi çalışır. Dize gerekenden daha uzunsa, ek bayt yoksayılır. Bir tarih, Unix döneminin başlangıcından bu yana geçen gün sayısı olarak yorumlanır ve zamana sahip bir tarih, Unix döneminin başlangıcından bu yana geçen saniye sayısı olarak yorumlanır. - -## reinterpretAsString {#type_conversion_functions-reinterpretAsString} - -Bu işlev, bir sayı veya tarih veya tarih saat ile kabul eder ve ana bilgisayar düzeninde (little endian) karşılık gelen değeri temsil eden bayt içeren bir dize döndürür. Boş bayt sondan bırakılır. Örneğin, 255 uint32 türü değeri bir bayt uzunluğunda bir dizedir. - -## reinterpretAsFixedString {#reinterpretasfixedstring} - -Bu işlev, bir sayı veya tarih veya tarih saat ile kabul eder ve karşılık gelen değeri ana bilgisayar sırasına (little endian) temsil eden bayt içeren bir FixedString döndürür. Boş bayt sondan bırakılır. Örneğin, 255 uint32 türü değeri bir bayt uzunluğunda bir FixedString. - -## CAS (t(x, T) {#type_conversion_function-cast} - -Dönüşüyo ‘x’ to the ‘t’ veri türü. Sözdizimi CAST (x AS t) da desteklenmektedir. - -Örnek: - -``` sql -SELECT - '2016-06-15 23:00:00' AS timestamp, - CAST(timestamp AS DateTime) AS datetime, - CAST(timestamp AS Date) AS date, - CAST(timestamp, 'String') AS string, - CAST(timestamp, 'FixedString(22)') AS fixed_string -``` - -``` text -┌─timestamp───────────┬────────────datetime─┬───────date─┬─string──────────────┬─fixed_string──────────────┐ -│ 2016-06-15 23:00:00 │ 2016-06-15 23:00:00 │ 2016-06-15 │ 2016-06-15 23:00:00 │ 2016-06-15 23:00:00\0\0\0 │ -└─────────────────────┴─────────────────────┴────────────┴─────────────────────┴───────────────────────────┘ -``` - -Fixedstring(N) ' ye dönüştürme yalnızca String veya FixedString(N) türünde argümanlar için çalışır. - -Type con conversionvers conversionion to [Nullable](../../sql-reference/data-types/nullable.md) ve geri desteklenmektedir. Örnek: - -``` sql -SELECT toTypeName(x) FROM t_null -``` - -``` text -┌─toTypeName(x)─┐ -│ Int8 │ -│ Int8 │ -└───────────────┘ -``` - -``` sql -SELECT toTypeName(CAST(x, 'Nullable(UInt16)')) FROM t_null -``` - -``` text -┌─toTypeName(CAST(x, 'Nullable(UInt16)'))─┐ -│ Nullable(UInt16) │ -│ Nullable(UInt16) │ -└─────────────────────────────────────────┘ -``` - -## toİnterval(yıl\|Çeyrek\|Ay\|hafta\|Gün\|Saat\|Dakika / Saniye) {#function-tointerval} - -Bir sayı türü argümanını bir [Aralıklı](../../sql-reference/data-types/special-data-types/interval.md) veri türü. - -**Sözdizimi** - -``` sql -toIntervalSecond(number) -toIntervalMinute(number) -toIntervalHour(number) -toIntervalDay(number) -toIntervalWeek(number) -toIntervalMonth(number) -toIntervalQuarter(number) -toIntervalYear(number) -``` - -**Parametre** - -- `number` — Duration of interval. Positive integer number. - -**Döndürülen değerler** - -- Değeri `Interval` veri türü. - -**Örnek** - -``` sql -WITH - toDate('2019-01-01') AS date, - INTERVAL 1 WEEK AS interval_week, - toIntervalWeek(1) AS interval_to_week -SELECT - date + interval_week, - date + interval_to_week -``` - -``` text -┌─plus(date, interval_week)─┬─plus(date, interval_to_week)─┐ -│ 2019-01-08 │ 2019-01-08 │ -└───────────────────────────┴──────────────────────────────┘ -``` - -## parseDateTimeBestEffort {#parsedatetimebesteffort} - -Bir tarih ve saati dönüştürür [Dize](../../sql-reference/data-types/string.md) temsil etmek [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) veri türü. - -İşlev ayrıştırır [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601), [RFC 1123-5.2.14 RFC-822 Tarih ve Saat özellikleri](https://tools.ietf.org/html/rfc1123#page-55), ClickHouse ve diğer bazı tarih ve saat biçimleri. - -**Sözdizimi** - -``` sql -parseDateTimeBestEffort(time_string [, time_zone]); -``` - -**Parametre** - -- `time_string` — String containing a date and time to convert. [Dize](../../sql-reference/data-types/string.md). -- `time_zone` — Time zone. The function parses `time_string` saat dilimine göre. [Dize](../../sql-reference/data-types/string.md). - -**Desteklenen standart dışı formatlar** - -- 9 içeren bir dize..10 haneli [unix zaman damgası](https://en.wikipedia.org/wiki/Unix_time). -- Tarih ve saat bileşeni olan bir dize: `YYYYMMDDhhmmss`, `DD/MM/YYYY hh:mm:ss`, `DD-MM-YY hh:mm`, `YYYY-MM-DD hh:mm:ss` vb. -- Bir tarih, ancak hiçbir zaman bileşeni ile bir dize: `YYYY`, `YYYYMM`, `YYYY*MM`, `DD/MM/YYYY`, `DD-MM-YY` vb. -- Bir gün ve Saat ile bir dize: `DD`, `DD hh`, `DD hh:mm`. Bu durumda `YYYY-MM` olarak ikame edilir `2000-01`. -- Tarih ve Saat Saat Dilimi uzaklık bilgileri ile birlikte içeren bir dize: `YYYY-MM-DD hh:mm:ss ±h:mm` vb. Mesela, `2020-12-12 17:36:00 -5:00`. - -Ayırıcılı tüm formatlar için işlev, tam adlarıyla veya bir ay adının ilk üç harfiyle ifade edilen ay adlarını ayrıştırır. Örnekler: `24/DEC/18`, `24-Dec-18`, `01-September-2018`. - -**Döndürülen değer** - -- `time_string` dönüştürül thedü `DateTime` veri türü. - -**Örnekler** - -Sorgu: - -``` sql -SELECT parseDateTimeBestEffort('12/12/2020 12:12:57') -AS parseDateTimeBestEffort; -``` - -Sonuç: - -``` text -┌─parseDateTimeBestEffort─┐ -│ 2020-12-12 12:12:57 │ -└─────────────────────────┘ -``` - -Sorgu: - -``` sql -SELECT parseDateTimeBestEffort('Sat, 18 Aug 2018 07:22:16 GMT', 'Europe/Moscow') -AS parseDateTimeBestEffort -``` - -Sonuç: - -``` text -┌─parseDateTimeBestEffort─┐ -│ 2018-08-18 10:22:16 │ -└─────────────────────────┘ -``` - -Sorgu: - -``` sql -SELECT parseDateTimeBestEffort('1284101485') -AS parseDateTimeBestEffort -``` - -Sonuç: - -``` text -┌─parseDateTimeBestEffort─┐ -│ 2015-07-07 12:04:41 │ -└─────────────────────────┘ -``` - -Sorgu: - -``` sql -SELECT parseDateTimeBestEffort('2018-12-12 10:12:12') -AS parseDateTimeBestEffort -``` - -Sonuç: - -``` text -┌─parseDateTimeBestEffort─┐ -│ 2018-12-12 10:12:12 │ -└─────────────────────────┘ -``` - -Sorgu: - -``` sql -SELECT parseDateTimeBestEffort('10 20:19') -``` - -Sonuç: - -``` text -┌─parseDateTimeBestEffort('10 20:19')─┐ -│ 2000-01-10 20:19:00 │ -└─────────────────────────────────────┘ -``` - -**Ayrıca Bakınız** - -- \[ISO 8601 duyuru @xkcd\](https://xkcd.com/1179/) -- [RFC 1123](https://tools.ietf.org/html/rfc1123) -- [toDate](#todate) -- [toDateTime](#todatetime) - -## parseDateTimeBestEffortOrNull {#parsedatetimebesteffortornull} - -İçin aynı [parseDateTimeBestEffort](#parsedatetimebesteffort) işlenemeyen bir tarih biçimiyle karşılaştığında null döndürmesi dışında. - -## parseDateTimeBestEffortOrZero {#parsedatetimebesteffortorzero} - -İçin aynı [parseDateTimeBestEffort](#parsedatetimebesteffort) bunun dışında, işlenemeyen bir tarih biçimiyle karşılaştığında sıfır tarih veya sıfır tarih saati döndürür. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/type_conversion_functions/) diff --git a/docs/tr/sql-reference/functions/url-functions.md b/docs/tr/sql-reference/functions/url-functions.md deleted file mode 100644 index ac4f9c1430d..00000000000 --- a/docs/tr/sql-reference/functions/url-functions.md +++ /dev/null @@ -1,209 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 54 -toc_title: "URL'ler ile \xE7al\u0131\u015Fma" ---- - -# URL'ler ile çalışmak için işlevler {#functions-for-working-with-urls} - -Tüm bu işlevler RFC'Yİ takip etmez. Geliştirilmiş performans için maksimum derecede basitleştirilmişlerdir. - -## Bir URL'nin bölümlerini Ayıklayan işlevler {#functions-that-extract-parts-of-a-url} - -İlgili bölüm bir URL'de yoksa, boş bir dize döndürülür. - -### protokol {#protocol} - -Protokolü bir URL'den ayıklar. - -Examples of typical returned values: http, https, ftp, mailto, tel, magnet… - -### etki {#domain} - -Ana bilgisayar adını bir URL'den ayıklar. - -``` sql -domain(url) -``` - -**Parametre** - -- `url` — URL. Type: [Dize](../../sql-reference/data-types/string.md). - -URL, bir şema ile veya şema olmadan belirtilebilir. Örnekler: - -``` text -svn+ssh://some.svn-hosting.com:80/repo/trunk -some.svn-hosting.com:80/repo/trunk -https://yandex.com/time/ -``` - -Bu örnekler için, `domain` işlev aşağıdaki sonuçları döndürür: - -``` text -some.svn-hosting.com -some.svn-hosting.com -yandex.com -``` - -**Döndürülen değerler** - -- Adı ana. ClickHouse giriş dizesini bir URL olarak ayrıştırırsa. -- Boş dize. ClickHouse giriş dizesini bir URL olarak ayrıştıramazsa. - -Tür: `String`. - -**Örnek** - -``` sql -SELECT domain('svn+ssh://some.svn-hosting.com:80/repo/trunk') -``` - -``` text -┌─domain('svn+ssh://some.svn-hosting.com:80/repo/trunk')─┐ -│ some.svn-hosting.com │ -└────────────────────────────────────────────────────────┘ -``` - -### domainWithoutWWW {#domainwithoutwww} - -Etki alanını döndürür ve birden fazla kaldırır ‘www.’ başlangıcına, eğer var dan. - -### topLevelDomain {#topleveldomain} - -Üst düzey etki alanını bir URL'den ayıklar. - -``` sql -topLevelDomain(url) -``` - -**Parametre** - -- `url` — URL. Type: [Dize](../../sql-reference/data-types/string.md). - -URL, bir şema ile veya şema olmadan belirtilebilir. Örnekler: - -``` text -svn+ssh://some.svn-hosting.com:80/repo/trunk -some.svn-hosting.com:80/repo/trunk -https://yandex.com/time/ -``` - -**Döndürülen değerler** - -- Etki alanı adı. ClickHouse giriş dizesini bir URL olarak ayrıştırırsa. -- Boş dize. ClickHouse giriş dizesini bir URL olarak ayrıştıramazsa. - -Tür: `String`. - -**Örnek** - -``` sql -SELECT topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk') -``` - -``` text -┌─topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk')─┐ -│ com │ -└────────────────────────────────────────────────────────────────────┘ -``` - -### firstSignificantSubdomain {#firstsignificantsubdomain} - -Ret theur thens the “first significant subdomain”. Bu, Yandex'e özgü standart olmayan bir kavramdır.Metrica. İlk önemli alt etki alanı ise ikinci düzey bir etki alanıdır ‘com’, ‘net’, ‘org’, veya ‘co’. Aksi takdirde, üçüncü düzey bir alandır. Mesela, `firstSignificantSubdomain (‘https://news.yandex.ru/’) = ‘yandex’, firstSignificantSubdomain (‘https://news.yandex.com.tr/’) = ‘yandex’`. Listesi “insignificant” ikinci düzey etki alanları ve diğer uygulama ayrıntıları gelecekte değişebilir. - -### cutToFirstSignificantSubdomain {#cuttofirstsignificantsubdomain} - -En üst düzey alt etki alanlarını içeren etki alanının bir bölümünü döndürür. “first significant subdomain” (yukarıdaki açıklamaya bakınız). - -Mesela, `cutToFirstSignificantSubdomain('https://news.yandex.com.tr/') = 'yandex.com.tr'`. - -### yol {#path} - -Yolu döndürür. Örnek: `/top/news.html` Yol sorgu dizesini içermez. - -### pathFull {#pathfull} - -Yukarıdaki ile aynı, ancak sorgu dizesi ve parça dahil. Örnek: / top / haberler.html?Sayfa = 2 # yorumlar - -### queryString {#querystring} - -Sorgu dizesini döndürür. Örnek: Sayfa = 1 & lr = 213. sorgu dizesi, ilk soru işaretinin yanı sıra # ve # sonrası her şeyi içermez. - -### parça {#fragment} - -Parça tanımlayıcısını döndürür. fragment ilk karma sembolü içermez. - -### queryStringAndFragment {#querystringandfragment} - -Sorgu dizesini ve parça tanımlayıcısını döndürür. Örnek: Sayfa = 1#29390. - -### extractURLParameter (URL, isim) {#extracturlparameterurl-name} - -Değerini döndürür ‘name’ varsa, URL'DEKİ parametre. Aksi takdirde, boş bir dize. Bu ada sahip birçok parametre varsa, ilk oluşumu döndürür. Bu işlev, parametre adının URL'de geçirilen bağımsız değişkenle aynı şekilde kodlandığı varsayımı altında çalışır. - -### extractURLParameters (URL) {#extracturlparametersurl} - -Bir dizi döndürür name = URL parametrelerine karşılık gelen değer dizeleri. Değerler hiçbir şekilde deşifre edilmez. - -### extractURLParameterNames(URL) {#extracturlparameternamesurl} - -URL parametrelerinin adlarına karşılık gelen bir dizi ad dizesi döndürür. Değerler hiçbir şekilde deşifre edilmez. - -### URLHierarchy(URL) {#urlhierarchyurl} - -Sonunda/,? simgeleriyle kesilen URL'yi içeren bir dizi döndürür yol ve sorgu dizesinde. Ardışık ayırıcı karakterler bir olarak sayılır. Kesim, tüm ardışık ayırıcı karakterlerden sonra pozisyonda yapılır. - -### URLPathHierarchy(URL) {#urlpathhierarchyurl} - -Yukarıdaki ile aynı, ancak sonuçta protokol ve ana bilgisayar olmadan. / Eleman (kök) dahil değildir. Örnek: işlev, yandex'te URL'yi ağaç raporları uygulamak için kullanılır. Ölçü. - -``` text -URLPathHierarchy('https://example.com/browse/CONV-6788') = -[ - '/browse/', - '/browse/CONV-6788' -] -``` - -### decodeURLComponent (URL) {#decodeurlcomponenturl} - -Çözülmüş URL'yi döndürür. -Örnek: - -``` sql -SELECT decodeURLComponent('http://127.0.0.1:8123/?query=SELECT%201%3B') AS DecodedURL; -``` - -``` text -┌─DecodedURL─────────────────────────────┐ -│ http://127.0.0.1:8123/?query=SELECT 1; │ -└────────────────────────────────────────┘ -``` - -## URL'nin bir bölümünü kaldıran işlevler {#functions-that-remove-part-of-a-url} - -URL'de benzer bir şey yoksa, URL değişmeden kalır. - -### cutWWW {#cutwww} - -Birden fazla kaldırır ‘www.’ varsa, URL'nin etki alanının başından itibaren. - -### cutQueryString {#cutquerystring} - -Sorgu dizesini kaldırır. Soru işareti de kaldırılır. - -### cutFragment {#cutfragment} - -Parça tanımlayıcısını kaldırır. Sayı işareti de kaldırılır. - -### cutQueryStringAndFragment {#cutquerystringandfragment} - -Sorgu dizesini ve parça tanımlayıcısını kaldırır. Soru işareti ve sayı işareti de kaldırılır. - -### cutURLParameter (URL, isim) {#cuturlparameterurl-name} - -Kaldırır ‘name’ Varsa URL parametresi. Bu işlev, parametre adının URL'de geçirilen bağımsız değişkenle aynı şekilde kodlandığı varsayımı altında çalışır. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/url_functions/) diff --git a/docs/tr/sql-reference/functions/uuid-functions.md b/docs/tr/sql-reference/functions/uuid-functions.md deleted file mode 100644 index 97692e98b9f..00000000000 --- a/docs/tr/sql-reference/functions/uuid-functions.md +++ /dev/null @@ -1,122 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 53 -toc_title: "UUID ile \xE7al\u0131\u015Fma" ---- - -# UUID ile çalışmak için fonksiyonlar {#functions-for-working-with-uuid} - -UUID ile çalışmak için işlevler aşağıda listelenmiştir. - -## generateuuıdv4 {#uuid-function-generate} - -Üretir [UUID](../../sql-reference/data-types/uuid.md) -den [sürüm 4](https://tools.ietf.org/html/rfc4122#section-4.4). - -``` sql -generateUUIDv4() -``` - -**Döndürülen değer** - -UUID türü değeri. - -**Kullanım örneği** - -Bu örnek, UUID türü sütunuyla bir tablo oluşturma ve tabloya bir değer ekleme gösterir. - -``` sql -CREATE TABLE t_uuid (x UUID) ENGINE=TinyLog - -INSERT INTO t_uuid SELECT generateUUIDv4() - -SELECT * FROM t_uuid -``` - -``` text -┌────────────────────────────────────x─┐ -│ f4bf890f-f9dc-4332-ad5c-0c18e73f28e9 │ -└──────────────────────────────────────┘ -``` - -## toUUİD (x) {#touuid-x} - -Dize türü değerini UUID türüne dönüştürür. - -``` sql -toUUID(String) -``` - -**Döndürülen değer** - -UUID türü değeri. - -**Kullanım örneği** - -``` sql -SELECT toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0') AS uuid -``` - -``` text -┌─────────────────────────────────uuid─┐ -│ 61f0c404-5cb3-11e7-907b-a6006ad3dba0 │ -└──────────────────────────────────────┘ -``` - -## UUİDStringToNum {#uuidstringtonum} - -Biçiminde 36 karakter içeren bir dize kabul eder `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` ve bir bayt kümesi olarak döndürür [FixedString (16)](../../sql-reference/data-types/fixedstring.md). - -``` sql -UUIDStringToNum(String) -``` - -**Döndürülen değer** - -FixedString (16) - -**Kullanım örnekleri** - -``` sql -SELECT - '612f3c40-5d3b-217e-707b-6a546a3d7b29' AS uuid, - UUIDStringToNum(uuid) AS bytes -``` - -``` text -┌─uuid─────────────────────────────────┬─bytes────────────┐ -│ 612f3c40-5d3b-217e-707b-6a546a3d7b29 │ a/<@];!~p{jTj={) │ -└──────────────────────────────────────┴──────────────────┘ -``` - -## UUİDNumToString {#uuidnumtostring} - -Kabul eder bir [FixedString (16)](../../sql-reference/data-types/fixedstring.md) değer ve metin biçiminde 36 karakter içeren bir dize döndürür. - -``` sql -UUIDNumToString(FixedString(16)) -``` - -**Döndürülen değer** - -Dize. - -**Kullanım örneği** - -``` sql -SELECT - 'a/<@];!~p{jTj={)' AS bytes, - UUIDNumToString(toFixedString(bytes, 16)) AS uuid -``` - -``` text -┌─bytes────────────┬─uuid─────────────────────────────────┐ -│ a/<@];!~p{jTj={) │ 612f3c40-5d3b-217e-707b-6a546a3d7b29 │ -└──────────────────┴──────────────────────────────────────┘ -``` - -## Ayrıca Bakınız {#see-also} - -- [dictGetUUİD](ext-dict-functions.md#ext_dict_functions-other) - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/uuid_function/) diff --git a/docs/tr/sql-reference/functions/ym-dict-functions.md b/docs/tr/sql-reference/functions/ym-dict-functions.md deleted file mode 100644 index f7bdefb10eb..00000000000 --- a/docs/tr/sql-reference/functions/ym-dict-functions.md +++ /dev/null @@ -1,155 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 59 -toc_title: "Yandex ile \xE7al\u0131\u015Fmak.Metrica S\xF6zl\xFCkleri" ---- - -# Yandex ile çalışmak için fonksiyonlar.Metrica Sözlükleri {#functions-for-working-with-yandex-metrica-dictionaries} - -Aşağıdaki işlevlerin çalışması için, sunucu yapılandırmasının tüm Yandex'i almak için yolları ve adresleri belirtmesi gerekir.Metrica sözlükler. Sözlükler, bu işlevlerden herhangi birinin ilk çağrısında yüklenir. Başvuru listeleri yüklenemiyorsa, bir özel durum atılır. - -Başvuru listeleri oluşturma hakkında daha fazla bilgi için bölüme bakın “Dictionaries”. - -## Çoklu Geobazlar {#multiple-geobases} - -ClickHouse, belirli bölgelerin hangi ülkelere ait olduğu konusunda çeşitli perspektifleri desteklemek için aynı anda birden fazla alternatif jeobaz (bölgesel hiyerarşiler) ile çalışmayı destekler. - -Bu ‘clickhouse-server’ config, dosyayı bölgesel hiyerarşi ile belirtir::`/opt/geo/regions_hierarchy.txt` - -Bu dosyanın yanı sıra, yakındaki _ sembolüne ve isme eklenen herhangi bir sonek (dosya uzantısından önce) olan dosyaları da arar. -Örneğin, dosyayı da bulacaktır `/opt/geo/regions_hierarchy_ua.txt` varsa. - -`ua` sözlük anahtarı denir. Soneksiz bir sözlük için anahtar boş bir dizedir. - -Tüm sözlükler çalışma zamanında yeniden yüklenir (buıltın_dıctıonarıes_reload_ınterval yapılandırma parametresinde tanımlandığı gibi belirli sayıda saniyede bir kez veya varsayılan olarak saatte bir kez). Ancak, sunucu başladığında kullanılabilir sözlüklerin listesi bir kez tanımlanır. - -All functions for working with regions have an optional argument at the end – the dictionary key. It is referred to as the geobase. -Örnek: - -``` sql -regionToCountry(RegionID) – Uses the default dictionary: /opt/geo/regions_hierarchy.txt -regionToCountry(RegionID, '') – Uses the default dictionary: /opt/geo/regions_hierarchy.txt -regionToCountry(RegionID, 'ua') – Uses the dictionary for the 'ua' key: /opt/geo/regions_hierarchy_ua.txt -``` - -### regionToCity (id \[, geobase\]) {#regiontocityid-geobase} - -Accepts a UInt32 number – the region ID from the Yandex geobase. If this region is a city or part of a city, it returns the region ID for the appropriate city. Otherwise, returns 0. - -### regionToArea (id \[, geobase\]) {#regiontoareaid-geobase} - -Bir bölgeyi bir alana dönüştürür (geobase içinde 5 yazın). Diğer her şekilde, bu işlev aynıdır ‘regionToCity’. - -``` sql -SELECT DISTINCT regionToName(regionToArea(toUInt32(number), 'ua')) -FROM system.numbers -LIMIT 15 -``` - -``` text -┌─regionToName(regionToArea(toUInt32(number), \'ua\'))─┐ -│ │ -│ Moscow and Moscow region │ -│ St. Petersburg and Leningrad region │ -│ Belgorod region │ -│ Ivanovsk region │ -│ Kaluga region │ -│ Kostroma region │ -│ Kursk region │ -│ Lipetsk region │ -│ Orlov region │ -│ Ryazan region │ -│ Smolensk region │ -│ Tambov region │ -│ Tver region │ -│ Tula region │ -└──────────────────────────────────────────────────────┘ -``` - -### regionToDistrict (id \[, geobase\]) {#regiontodistrictid-geobase} - -Bir bölgeyi federal bir bölgeye dönüştürür (geobase içinde tip 4). Diğer her şekilde, bu işlev aynıdır ‘regionToCity’. - -``` sql -SELECT DISTINCT regionToName(regionToDistrict(toUInt32(number), 'ua')) -FROM system.numbers -LIMIT 15 -``` - -``` text -┌─regionToName(regionToDistrict(toUInt32(number), \'ua\'))─┐ -│ │ -│ Central federal district │ -│ Northwest federal district │ -│ South federal district │ -│ North Caucases federal district │ -│ Privolga federal district │ -│ Ural federal district │ -│ Siberian federal district │ -│ Far East federal district │ -│ Scotland │ -│ Faroe Islands │ -│ Flemish region │ -│ Brussels capital region │ -│ Wallonia │ -│ Federation of Bosnia and Herzegovina │ -└──────────────────────────────────────────────────────────┘ -``` - -### regionToCountry (ıd \[, geobase\]) {#regiontocountryid-geobase} - -Bir bölgeyi bir ülkeye dönüştürür. Diğer her şekilde, bu işlev aynıdır ‘regionToCity’. -Örnek: `regionToCountry(toUInt32(213)) = 225` Moskova'yı (213) Rusya'ya (225) dönüştürür. - -### regionToContinent (id \[, geobase\]) {#regiontocontinentid-geobase} - -Bir bölgeyi bir kıtaya dönüştürür. Diğer her şekilde, bu işlev aynıdır ‘regionToCity’. -Örnek: `regionToContinent(toUInt32(213)) = 10001` Moskova'yı (213) Avrasya'ya (10001) dönüştürür. - -### regionToTopContinent (#regiontotopcontinent) {#regiontotopcontinent-regiontotopcontinent} - -Bölgenin hiyerarşisinde en yüksek kıtayı bulur. - -**Sözdizimi** - -``` sql -regionToTopContinent(id[, geobase]); -``` - -**Parametre** - -- `id` — Region ID from the Yandex geobase. [Uİnt32](../../sql-reference/data-types/int-uint.md). -- `geobase` — Dictionary key. See [Çoklu Geobazlar](#multiple-geobases). [Dize](../../sql-reference/data-types/string.md). İsteğe bağlı. - -**Döndürülen değer** - -- Üst düzey kıtanın tanımlayıcısı (bölgeler hiyerarşisine tırmandığınızda ikincisi). -- 0, yoksa. - -Tür: `UInt32`. - -### regionToPopulation (id \[, geobase\]) {#regiontopopulationid-geobase} - -Bir bölge için nüfusu alır. -Nüfus geobase ile dosyalarda kaydedilebilir. Bölümüne bakınız “External dictionaries”. -Bölge için nüfus kaydedilmezse, 0 döndürür. -Yandex geobase'de, nüfus alt bölgeler için kaydedilebilir, ancak üst bölgeler için kaydedilemez. - -### regionİn (lhs, rhs \[, geobase\]) {#regioninlhs-rhs-geobase} - -Olup olmadığını denetler bir ‘lhs’ bölge bir ‘rhs’ bölge. Aitse 1'e eşit bir Uİnt8 numarası veya ait değilse 0 döndürür. -The relationship is reflexive – any region also belongs to itself. - -### regionHierarchy (id \[, geobase\]) {#regionhierarchyid-geobase} - -Accepts a UInt32 number – the region ID from the Yandex geobase. Returns an array of region IDs consisting of the passed region and all parents along the chain. -Örnek: `regionHierarchy(toUInt32(213)) = [213,1,3,225,10001,10000]`. - -### regionToName (id \[, lang\]) {#regiontonameid-lang} - -Accepts a UInt32 number – the region ID from the Yandex geobase. A string with the name of the language can be passed as a second argument. Supported languages are: ru, en, ua, uk, by, kz, tr. If the second argument is omitted, the language ‘ru’ is used. If the language is not supported, an exception is thrown. Returns a string – the name of the region in the corresponding language. If the region with the specified ID doesn't exist, an empty string is returned. - -`ua` ve `uk` hem Ukrayna demek. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/ym_dict_functions/) diff --git a/docs/tr/sql-reference/index.md b/docs/tr/sql-reference/index.md deleted file mode 100644 index 9cdf88083e9..00000000000 --- a/docs/tr/sql-reference/index.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "SQL ba\u015Fvurusu" -toc_hidden: true -toc_priority: 28 -toc_title: "gizlenmi\u015F" ---- - -# SQL başvurusu {#sql-reference} - -ClickHouse aşağıdaki sorgu türlerini destekler: - -- [SELECT](statements/select/index.md) -- [INSERT INTO](statements/insert-into.md) -- [CREATE](statements/create.md) -- [ALTER](statements/alter.md#query_language_queries_alter) -- [Diğer sorgu türleri](statements/misc.md) - -[Orijinal makale](https://clickhouse.tech/docs/en/sql-reference/) diff --git a/docs/tr/sql-reference/operators/in.md b/docs/tr/sql-reference/operators/in.md deleted file mode 120000 index 3a2feda2f61..00000000000 --- a/docs/tr/sql-reference/operators/in.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/sql-reference/operators/in.md \ No newline at end of file diff --git a/docs/tr/sql-reference/operators/index.md b/docs/tr/sql-reference/operators/index.md deleted file mode 100644 index 20a0b625d2f..00000000000 --- a/docs/tr/sql-reference/operators/index.md +++ /dev/null @@ -1,277 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 37 -toc_title: "Operat\xF6rler" ---- - -# Operatörler {#operators} - -ClickHouse onların öncelik, öncelik ve ilişkilendirme göre sorgu ayrıştırma aşamasında karşılık gelen işlevlere işleçleri dönüştürür. - -## Erişim Operatörleri {#access-operators} - -`a[N]` – Access to an element of an array. The `arrayElement(a, N)` İşlev. - -`a.N` – Access to a tuple element. The `tupleElement(a, N)` İşlev. - -## Sayısal Olumsuzlama Operatörü {#numeric-negation-operator} - -`-a` – The `negate (a)` İşlev. - -## Çarpma ve bölme operatörleri {#multiplication-and-division-operators} - -`a * b` – The `multiply (a, b)` İşlev. - -`a / b` – The `divide(a, b)` İşlev. - -`a % b` – The `modulo(a, b)` İşlev. - -## Toplama ve çıkarma operatörleri {#addition-and-subtraction-operators} - -`a + b` – The `plus(a, b)` İşlev. - -`a - b` – The `minus(a, b)` İşlev. - -## Karşılaştırma Operatörleri {#comparison-operators} - -`a = b` – The `equals(a, b)` İşlev. - -`a == b` – The `equals(a, b)` İşlev. - -`a != b` – The `notEquals(a, b)` İşlev. - -`a <> b` – The `notEquals(a, b)` İşlev. - -`a <= b` – The `lessOrEquals(a, b)` İşlev. - -`a >= b` – The `greaterOrEquals(a, b)` İşlev. - -`a < b` – The `less(a, b)` İşlev. - -`a > b` – The `greater(a, b)` İşlev. - -`a LIKE s` – The `like(a, b)` İşlev. - -`a NOT LIKE s` – The `notLike(a, b)` İşlev. - -`a BETWEEN b AND c` – The same as `a >= b AND a <= c`. - -`a NOT BETWEEN b AND c` – The same as `a < b OR a > c`. - -## Veri kümeleriyle çalışmak için operatörler {#operators-for-working-with-data-sets} - -*Görmek [Operatör İNLERDE](in.md).* - -`a IN ...` – The `in(a, b)` İşlev. - -`a NOT IN ...` – The `notIn(a, b)` İşlev. - -`a GLOBAL IN ...` – The `globalIn(a, b)` İşlev. - -`a GLOBAL NOT IN ...` – The `globalNotIn(a, b)` İşlev. - -## Tarih ve Saatlerle çalışmak için operatörler {#operators-datetime} - -### EXTRACT {#operator-extract} - -``` sql -EXTRACT(part FROM date); -``` - -Belirli bir tarihten parçaları ayıklayın. Örneğin, belirli bir tarihten bir ay veya bir zamandan bir saniye alabilirsiniz. - -Bu `part` parametre almak için tarihin hangi bölümünü belirtir. Aşağıdaki değerler kullanılabilir: - -- `DAY` — The day of the month. Possible values: 1–31. -- `MONTH` — The number of a month. Possible values: 1–12. -- `YEAR` — The year. -- `SECOND` — The second. Possible values: 0–59. -- `MINUTE` — The minute. Possible values: 0–59. -- `HOUR` — The hour. Possible values: 0–23. - -Bu `part` parametre büyük / küçük harf duyarsızdır. - -Bu `date` parametre, işlenecek tarihi veya saati belirtir. Ya [Tarihli](../../sql-reference/data-types/date.md) veya [DateTime](../../sql-reference/data-types/datetime.md) türü desteklenir. - -Örnekler: - -``` sql -SELECT EXTRACT(DAY FROM toDate('2017-06-15')); -SELECT EXTRACT(MONTH FROM toDate('2017-06-15')); -SELECT EXTRACT(YEAR FROM toDate('2017-06-15')); -``` - -Aşağıdaki örnekte bir tablo oluşturuyoruz ve içine bir değer ekliyoruz `DateTime` tür. - -``` sql -CREATE TABLE test.Orders -( - OrderId UInt64, - OrderName String, - OrderDate DateTime -) -ENGINE = Log; -``` - -``` sql -INSERT INTO test.Orders VALUES (1, 'Jarlsberg Cheese', toDateTime('2008-10-11 13:23:44')); -``` - -``` sql -SELECT - toYear(OrderDate) AS OrderYear, - toMonth(OrderDate) AS OrderMonth, - toDayOfMonth(OrderDate) AS OrderDay, - toHour(OrderDate) AS OrderHour, - toMinute(OrderDate) AS OrderMinute, - toSecond(OrderDate) AS OrderSecond -FROM test.Orders; -``` - -``` text -┌─OrderYear─┬─OrderMonth─┬─OrderDay─┬─OrderHour─┬─OrderMinute─┬─OrderSecond─┐ -│ 2008 │ 10 │ 11 │ 13 │ 23 │ 44 │ -└───────────┴────────────┴──────────┴───────────┴─────────────┴─────────────┘ -``` - -Daha fazla örnek görebilirsiniz [testler](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00619_extract.sql). - -### INTERVAL {#operator-interval} - -Oluşturur bir [Aralıklı](../../sql-reference/data-types/special-data-types/interval.md)- aritmetik işlemlerde kullanılması gereken tip değeri [Tarihli](../../sql-reference/data-types/date.md) ve [DateTime](../../sql-reference/data-types/datetime.md)- tip değerleri. - -Aralık türleri: -- `SECOND` -- `MINUTE` -- `HOUR` -- `DAY` -- `WEEK` -- `MONTH` -- `QUARTER` -- `YEAR` - -!!! warning "Uyarıcı" - Farklı tiplere sahip aralıklar birleştirilemez. Gibi ifadeler kullanamazsınız `INTERVAL 4 DAY 1 HOUR`. Aralıkların, örneğin aralığın en küçük birimine eşit veya daha küçük olan birimlerdeki aralıkları belirtin, `INTERVAL 25 HOUR`. Aşağıdaki örnekte olduğu gibi ardışık işlemleri kullanabilirsiniz. - -Örnek: - -``` sql -SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL 3 HOUR -``` - -``` text -┌───current_date_time─┬─plus(plus(now(), toIntervalDay(4)), toIntervalHour(3))─┐ -│ 2019-10-23 11:16:28 │ 2019-10-27 14:16:28 │ -└─────────────────────┴────────────────────────────────────────────────────────┘ -``` - -**Ayrıca Bakınız** - -- [Aralıklı](../../sql-reference/data-types/special-data-types/interval.md) veri türü -- [toİnterval](../../sql-reference/functions/type-conversion-functions.md#function-tointerval) tip dönüştürme işlevleri - -## Mantıksal Olumsuzlama Operatörü {#logical-negation-operator} - -`NOT a` – The `not(a)` İşlev. - -## Mantıksal ve operatör {#logical-and-operator} - -`a AND b` – The`and(a, b)` İşlev. - -## Mantıksal veya operatör {#logical-or-operator} - -`a OR b` – The `or(a, b)` İşlev. - -## Koşullu Operatör {#conditional-operator} - -`a ? b : c` – The `if(a, b, c)` İşlev. - -Not: - -Koşullu işleç B ve c değerlerini hesaplar, ardından a koşulunun karşılanıp karşılanmadığını kontrol eder ve ardından karşılık gelen değeri döndürür. Eğer `b` veya `C` is an [arrayJoin()](../../sql-reference/functions/array-join.md#functions_arrayjoin) işlev, her satır ne olursa olsun çoğaltılır “a” koşul. - -## Koşullu İfade {#operator_case} - -``` sql -CASE [x] - WHEN a THEN b - [WHEN ... THEN ...] - [ELSE c] -END -``` - -Eğer `x` belirtilen sonra `transform(x, [a, ...], [b, ...], c)` function is used. Otherwise – `multiIf(a, b, ..., c)`. - -Eğer herhangi bir `ELSE c` ifadedeki yan tümce, varsayılan değer `NULL`. - -Bu `transform` fonksiyonu ile çalışmıyor `NULL`. - -## Birleştirme Operatörü {#concatenation-operator} - -`s1 || s2` – The `concat(s1, s2) function.` - -## Lambda Oluşturma Operatörü {#lambda-creation-operator} - -`x -> expr` – The `lambda(x, expr) function.` - -Parantez oldukları için aşağıdaki operatörler bir önceliğe sahip değildir: - -## Dizi Oluşturma Operatörü {#array-creation-operator} - -`[x1, ...]` – The `array(x1, ...) function.` - -## Tuple Oluşturma Operatörü {#tuple-creation-operator} - -`(x1, x2, ...)` – The `tuple(x2, x2, ...) function.` - -## İlişkisellik {#associativity} - -Tüm ikili operatörler ilişkisellikten ayrıldı. Mesela, `1 + 2 + 3` dönüştür toülür `plus(plus(1, 2), 3)`. -Bazen bu beklediğiniz gibi çalışmaz. Mesela, `SELECT 4 > 2 > 3` 0 ile sonuç willlanır. - -Verimlilik için, `and` ve `or` işlevler herhangi bir sayıda bağımsız değişkeni kabul eder. İlgili zincirler `AND` ve `OR` operatörler bu işlevlerin tek bir çağrısına dönüştürülür. - -## İçin kontrol `NULL` {#checking-for-null} - -ClickHouse destekler `IS NULL` ve `IS NOT NULL` operatörler. - -### IS NULL {#operator-is-null} - -- İçin [Nullable](../../sql-reference/data-types/nullable.md) türü değerleri `IS NULL` operatör döner: - - `1` değeri ise `NULL`. - - `0` başka. -- Diğer değerler için, `IS NULL` operatör her zaman döner `0`. - - - -``` sql -SELECT x+100 FROM t_null WHERE y IS NULL -``` - -``` text -┌─plus(x, 100)─┐ -│ 101 │ -└──────────────┘ -``` - -### IS NOT NULL {#is-not-null} - -- İçin [Nullable](../../sql-reference/data-types/nullable.md) türü değerleri `IS NOT NULL` operatör döner: - - `0` değeri ise `NULL`. - - `1` başka. -- Diğer değerler için, `IS NOT NULL` operatör her zaman döner `1`. - - - -``` sql -SELECT * FROM t_null WHERE y IS NOT NULL -``` - -``` text -┌─x─┬─y─┐ -│ 2 │ 3 │ -└───┴───┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/operators/) diff --git a/docs/tr/sql-reference/statements/alter.md b/docs/tr/sql-reference/statements/alter.md deleted file mode 100644 index 5acdd1ca2c4..00000000000 --- a/docs/tr/sql-reference/statements/alter.md +++ /dev/null @@ -1,602 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 36 -toc_title: ALTER ---- - -## ALTER {#query_language_queries_alter} - -Bu `ALTER` sorgu yalnızca için desteklenir `*MergeTree` tablo gibi `Merge`ve`Distributed`. Sorgunun çeşitli varyasyonları vardır. - -### Sütun Manipülasyonları {#column-manipulations} - -Tablo yapısını değiştirme. - -``` sql -ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN ... -``` - -Sorguda, bir veya daha fazla virgülle ayrılmış eylemlerin bir listesini belirtin. -Her eylem bir sütun üzerinde bir işlemdir. - -Aşağıdaki eylemler desteklenir: - -- [ADD COLUMN](#alter_add-column) — Adds a new column to the table. -- [DROP COLUMN](#alter_drop-column) — Deletes the column. -- [CLEAR COLUMN](#alter_clear-column) — Resets column values. -- [COMMENT COLUMN](#alter_comment-column) — Adds a text comment to the column. -- [MODIFY COLUMN](#alter_modify-column) — Changes column's type, default expression and TTL. - -Bu eylemler aşağıda ayrıntılı olarak açıklanmıştır. - -#### ADD COLUMN {#alter_add-column} - -``` sql -ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [codec] [AFTER name_after] -``` - -Belirtilen tabloya yeni bir sütun ekler `name`, `type`, [`codec`](create.md#codecs) ve `default_expr` (bkz [Varsayılan ifadeler](create.md#create-default-values)). - -Eğer... `IF NOT EXISTS` yan tümcesi dahil, sütun zaten varsa sorgu bir hata döndürmez. Belirtir specifyseniz `AFTER name_after` (başka bir sütunun adı), sütun tablo sütunları listesinde belirtilen sonra eklenir. Aksi takdirde, sütun tablonun sonuna eklenir. Bir tablonun başına bir sütun eklemek için bir yol olduğunu unutmayın. Bir eylem zinciri için, `name_after` önceki eylemlerden birine eklenen bir sütunun adı olabilir. - -Bir sütun eklemek, verilerle herhangi bir işlem yapmadan tablo yapısını değiştirir. Sonra veriler diskte görünmüyor `ALTER`. Tablodan okurken bir sütun için veri eksikse, varsayılan değerlerle doldurulur (varsa, varsayılan ifadeyi gerçekleştirerek veya sıfır veya boş dizeler kullanarak). Sütun, veri parçalarını birleştirdikten sonra diskte görünür (bkz. [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)). - -Bu yaklaşım bize tamamlamak için izin verir `ALTER` eski verilerin hacmini arttırmadan anında sorgulayın. - -Örnek: - -``` sql -ALTER TABLE visits ADD COLUMN browser String AFTER user_id -``` - -#### DROP COLUMN {#alter_drop-column} - -``` sql -DROP COLUMN [IF EXISTS] name -``` - -Sütun adı ile siler `name`. Eğer... `IF EXISTS` yan tümcesi belirtilir, sütun yoksa sorgu bir hata döndürmez. - -Dosya sisteminden veri siler. Bu, tüm dosyaları sildiğinden, sorgu neredeyse anında tamamlanır. - -Örnek: - -``` sql -ALTER TABLE visits DROP COLUMN browser -``` - -#### CLEAR COLUMN {#alter_clear-column} - -``` sql -CLEAR COLUMN [IF EXISTS] name IN PARTITION partition_name -``` - -Belirtilen bölüm için bir sütundaki tüm verileri sıfırlar. Bölümdeki bölüm adını ayarlama hakkında daha fazla bilgi edinin [Bölüm ifadesi nasıl belirlenir](#alter-how-to-specify-part-expr). - -Eğer... `IF EXISTS` yan tümcesi belirtilir, sütun yoksa sorgu bir hata döndürmez. - -Örnek: - -``` sql -ALTER TABLE visits CLEAR COLUMN browser IN PARTITION tuple() -``` - -#### COMMENT COLUMN {#alter_comment-column} - -``` sql -COMMENT COLUMN [IF EXISTS] name 'comment' -``` - -Sütuna bir yorum ekler. Eğer... `IF EXISTS` yan tümcesi belirtilir, sütun yoksa sorgu bir hata döndürmez. - -Her sütunun bir yorumu olabilir. Sütun için bir yorum zaten varsa, yeni bir yorum önceki yorumun üzerine yazar. - -Yorumlar saklanır `comment_expression` tarafından döndürülen sütun [DESCRIBE TABLE](misc.md#misc-describe-table) sorgu. - -Örnek: - -``` sql -ALTER TABLE visits COMMENT COLUMN browser 'The table shows the browser used for accessing the site.' -``` - -#### MODIFY COLUMN {#alter_modify-column} - -``` sql -MODIFY COLUMN [IF EXISTS] name [type] [default_expr] [TTL] -``` - -Bu sorgu değişiklikleri `name` sütun özellikleri: - -- Tür - -- Varsayılan ifade - -- TTL - - For examples of columns TTL modifying, see [Column TTL](../engines/table_engines/mergetree_family/mergetree.md#mergetree-column-ttl). - -Eğer... `IF EXISTS` yan tümcesi belirtilir, sütun yoksa sorgu bir hata döndürmez. - -Türü değiştirirken, değerler sanki [toType](../../sql-reference/functions/type-conversion-functions.md) fonksiyonlar onlara uygulandı. Yalnızca varsayılan ifade değiştirilirse, sorgu karmaşık bir şey yapmaz ve neredeyse anında tamamlanır. - -Örnek: - -``` sql -ALTER TABLE visits MODIFY COLUMN browser Array(String) -``` - -Changing the column type is the only complex action – it changes the contents of files with data. For large tables, this may take a long time. - -Birkaç işlem aşaması vardır: - -- Geçici (yeni) dosyaları değiştirilmiş verilerle hazırlama. -- Eski dosyaları yeniden adlandırma. -- Geçici (yeni) dosyaları eski adlara yeniden adlandırma. -- Eski dosyaları silme. - -Sadece ilk aşama zaman alır. Bu aşamada bir hata varsa, veriler değişmez. -Ardışık aşamalardan biri sırasında bir hata varsa, veriler el ile geri yüklenebilir. Eski dosyalar dosya sisteminden silindi, ancak yeni dosyaların verileri diske yazılmadı ve kaybolduysa istisnadır. - -Bu `ALTER` sütunları değiştirmek için sorgu çoğaltılır. Talimatlar ZooKeeper kaydedilir, daha sonra her kopya bunları uygular. Tüm `ALTER` sorgular aynı sırada çalıştırılır. Sorgu, diğer yinelemeler üzerinde tamamlanması uygun eylemleri bekler. Ancak, yinelenen bir tablodaki sütunları değiştirmek için bir sorgu kesilebilir ve tüm eylemler zaman uyumsuz olarak gerçekleştirilir. - -#### Sorgu sınırlamalarını değiştir {#alter-query-limitations} - -Bu `ALTER` sorgu oluşturmak ve iç içe veri yapıları, ancak tüm iç içe veri yapıları ayrı öğeleri (sütunlar) silmenizi sağlar. İç içe geçmiş bir veri yapısı eklemek için, aşağıdaki gibi bir ada sahip sütunlar ekleyebilirsiniz `name.nested_name` ve türü `Array(T)`. İç içe geçmiş bir veri yapısı, noktadan önce aynı öneki olan bir ada sahip birden çok dizi sütununa eşdeğerdir. - -Birincil anahtardaki veya örnekleme anahtarındaki sütunları silmek için destek yoktur. `ENGINE` ifade). Birincil anahtarda bulunan sütunların türünü değiştirmek, yalnızca bu değişiklik verilerin değiştirilmesine neden olmazsa mümkündür (örneğin, bir numaraya değer eklemenize veya bir türden değiştirmenize izin verilir `DateTime` -e doğru `UInt32`). - -Eğer... `ALTER` sorgu, ihtiyacınız olan tablo değişikliklerini yapmak için yeterli değildir, yeni bir tablo oluşturabilir, verileri kullanarak kopyalayabilirsiniz. [INSERT SELECT](insert-into.md#insert_query_insert-select) sorgu, daha sonra tabloları kullanarak geçiş [RENAME](misc.md#misc_operations-rename) sorgu ve eski tabloyu silin. Kullanabilirsiniz [clickhouse-fotokopi makinesi](../../operations/utilities/clickhouse-copier.md) bir alternatif olarak `INSERT SELECT` sorgu. - -Bu `ALTER` sorgu tüm okur ve tablo için yazar engeller. Başka bir deyişle, Eğer uzun `SELECT` zamanda çalışıyor `ALTER` sorgu `ALTER` sorgu tamamlanmasını bekleyecektir. Aynı zamanda, aynı tablodaki tüm yeni sorgular bu sırada bekleyecektir `ALTER` çalışıyor. - -Verileri kendileri saklamayan tablolar için (örneğin `Merge` ve `Distributed`), `ALTER` sadece tablo yapısını değiştirir ve alt tabloların yapısını değiştirmez. Örneğin, ALTER for a çalıştırırken `Distributed` tablo, ayrıca çalıştırmak gerekir `ALTER` tüm uzak sunuculardaki tablolar için. - -### Anahtar ifadelerle manipülasyonlar {#manipulations-with-key-expressions} - -Aşağıdaki komut desteklenir: - -``` sql -MODIFY ORDER BY new_expression -``` - -Sadece tablolar için çalışır [`MergeTree`](../../engines/table-engines/mergetree-family/mergetree.md) aile (dahil -[çoğaltıyordu](../../engines/table-engines/mergetree-family/replication.md) Tablolar). Komutu değiştirir -[sıralama anahtarı](../../engines/table-engines/mergetree-family/mergetree.md) tablonun --e doğru `new_expression` (bir ifade veya ifadelerin bir tuple). Birincil anahtar aynı kalır. - -Komut, yalnızca meta verileri değiştirdiği bir anlamda hafiftir. Veri parçası özelliği tutmak için -satırlar sıralama anahtarı ifadesi tarafından sıralanır varolan sütunları içeren ifadeler ekleyemezsiniz -sıralama anahtarına (yalnızca sütun tarafından eklenen `ADD COLUMN` aynı komut `ALTER` sorgu). - -### Veri atlama endeksleri ile manipülasyonlar {#manipulations-with-data-skipping-indices} - -Sadece tablolar için çalışır [`*MergeTree`](../../engines/table-engines/mergetree-family/mergetree.md) aile (dahil -[çoğaltıyordu](../../engines/table-engines/mergetree-family/replication.md) Tablolar). Aşağıdaki işlemler -mevcuttur: - -- `ALTER TABLE [db].name ADD INDEX name expression TYPE type GRANULARITY value AFTER name [AFTER name2]` - Tablolar meta dizin açıklama ekler. - -- `ALTER TABLE [db].name DROP INDEX name` - Tablolar meta dizin açıklama kaldırır ve diskten dizin dosyalarını siler. - -Bu komutlar, yalnızca meta verileri değiştirdikleri veya dosyaları kaldırdıkları bir anlamda hafiftir. -Ayrıca, çoğaltılırlar (ZooKeeper aracılığıyla indeks meta verilerini senkronize etme). - -### Kısıtlamalar ile manipülasyonlar {#manipulations-with-constraints} - -Daha fazla görmek [kısıtlamalar](create.md#constraints) - -Kısıtlamalar eklenebilir veya aşağıdaki sözdizimi kullanılarak silinebilir: - -``` sql -ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression; -ALTER TABLE [db].name DROP CONSTRAINT constraint_name; -``` - -Sorgular eklemek veya hemen işlenir, böylece tablodan kısıtlamaları hakkında meta verileri kaldırın. - -Kısıtlama kontrolü *idam edilm willeyecek* eklen .mişse mevcut ver .ilerde - -Çoğaltılmış tablolardaki tüm değişiklikler Zookeeper'a yayınlanır, bu nedenle diğer kopyalara uygulanır. - -### Bölümler ve parçalar ile manipülasyonlar {#alter_manipulations-with-partitions} - -Aşağıdaki işlemler ile [bölümler](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) mevcuttur: - -- [DETACH PARTITION](#alter_detach-partition) – Moves a partition to the `detached` dizin ve unutun. -- [DROP PARTITION](#alter_drop-partition) – Deletes a partition. -- [ATTACH PART\|PARTITION](#alter_attach-partition) – Adds a part or partition from the `detached` tabloya dizin. -- [ATTACH PARTITION FROM](#alter_attach-partition-from) – Copies the data partition from one table to another and adds. -- [REPLACE PARTITION](#alter_replace-partition) - Veri bölümünü bir tablodan diğerine kopyalar ve değiştirir. -- [MOVE PARTITION TO TABLE](#alter_move_to_table-partition)(#alter_move_to_table-partition) - veri bölümünü bir tablodan diğerine taşıyın. -- [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) - Bir bölümdeki belirtilen sütunun değerini sıfırlar. -- [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) - Bir bölümde belirtilen ikincil dizini sıfırlar. -- [FREEZE PARTITION](#alter_freeze-partition) – Creates a backup of a partition. -- [FETCH PARTITION](#alter_fetch-partition) – Downloads a partition from another server. -- [MOVE PARTITION\|PART](#alter_move-partition) – Move partition/data part to another disk or volume. - - - -#### DETACH PARTITION {#alter_detach-partition} - -``` sql -ALTER TABLE table_name DETACH PARTITION partition_expr -``` - -Belirtilen bölüm için tüm verileri `detached` dizin. Sunucu, yok gibi ayrılmış veri Bölümü hakkında unutur. Sunucu, bu verileri siz yapana kadar bilmeyecektir. [ATTACH](#alter_attach-partition) sorgu. - -Örnek: - -``` sql -ALTER TABLE visits DETACH PARTITION 201901 -``` - -Bir bölümdeki bölüm ifadesini ayarlama hakkında bilgi edinin [Bölüm ifadesi nasıl belirlenir](#alter-how-to-specify-part-expr). - -Sorgu yürütüldükten sonra, veri ile istediğiniz her şeyi yapabilirsiniz `detached` directory — delete it from the file system, or just leave it. - -This query is replicated – it moves the data to the `detached` tüm kopyalarda dizin. Bu sorguyu yalnızca bir lider yinelemesinde yürütebileceğinizi unutmayın. Bir kopya bir lider olup olmadığını öğrenmek için `SELECT` sorgu için [sistem.yinelemeler](../../operations/system-tables.md#system_tables-replicas) Tablo. Alternatif olarak, bir yapmak daha kolaydır `DETACH` tüm yinelemelerde sorgu - tüm yinelemeler, lider yinelemesi dışında bir özel durum oluşturur. - -#### DROP PARTITION {#alter_drop-partition} - -``` sql -ALTER TABLE table_name DROP PARTITION partition_expr -``` - -Belirtilen bölümü tablodan siler. Bu sorgu bölümü etkin olarak etiketler ve verileri tamamen yaklaşık 10 dakika içinde siler. - -Bir bölümdeki bölüm ifadesini ayarlama hakkında bilgi edinin [Bölüm ifadesi nasıl belirlenir](#alter-how-to-specify-part-expr). - -The query is replicated – it deletes data on all replicas. - -#### DROP DETACHED PARTITION\|PART {#alter_drop-detached} - -``` sql -ALTER TABLE table_name DROP DETACHED PARTITION|PART partition_expr -``` - -Belirtilen bölümü veya belirtilen bölümün tüm bölümlerini kaldırır `detached`. -Bir bölümdeki bölüm ifadesini ayarlama hakkında daha fazla bilgi edinin [Bölüm ifadesi nasıl belirlenir](#alter-how-to-specify-part-expr). - -#### ATTACH PARTITION\|PART {#alter_attach-partition} - -``` sql -ALTER TABLE table_name ATTACH PARTITION|PART partition_expr -``` - -Tablodan veri ekler `detached` dizin. Tüm bir bölüm veya ayrı bir bölüm için veri eklemek mümkündür. Örnekler: - -``` sql -ALTER TABLE visits ATTACH PARTITION 201901; -ALTER TABLE visits ATTACH PART 201901_2_2_0; -``` - -Bir bölümdeki bölüm ifadesini ayarlama hakkında daha fazla bilgi edinin [Bölüm ifadesi nasıl belirlenir](#alter-how-to-specify-part-expr). - -Bu sorgu çoğaltılır. Çoğaltma başlatıcısı, veri olup olmadığını denetler. `detached` dizin. Veri varsa, sorgu bütünlüğünü denetler. Her şey doğruysa, sorgu verileri tabloya ekler. Diğer tüm yinelemeler, çoğaltma başlatıcısından verileri karşıdan yükleyin. - -Böylece veri koyabilirsiniz `detached` bir kopya üzerinde dizin ve `ALTER ... ATTACH` tüm yinelemelerde tabloya eklemek için sorgu. - -#### ATTACH PARTITION FROM {#alter_attach-partition-from} - -``` sql -ALTER TABLE table2 ATTACH PARTITION partition_expr FROM table1 -``` - -Bu sorgu, veri bölümünü `table1` -e doğru `table2` exsisting için veri ekler `table2`. Verilerin silinmeyeceğini unutmayın `table1`. - -Sorgunun başarıyla çalışması için aşağıdaki koşulların karşılanması gerekir: - -- Her iki tablo da aynı yapıya sahip olmalıdır. -- Her iki tablo da aynı bölüm anahtarına sahip olmalıdır. - -#### REPLACE PARTITION {#alter_replace-partition} - -``` sql -ALTER TABLE table2 REPLACE PARTITION partition_expr FROM table1 -``` - -Bu sorgu, veri bölümünü `table1` -e doğru `table2` ve mevcut bölümün yerini alır `table2`. Verilerin silinmeyeceğini unutmayın `table1`. - -Sorgunun başarıyla çalışması için aşağıdaki koşulların karşılanması gerekir: - -- Her iki tablo da aynı yapıya sahip olmalıdır. -- Her iki tablo da aynı bölüm anahtarına sahip olmalıdır. - -#### MOVE PARTITION TO TABLE {#alter_move_to_table-partition} - -``` sql -ALTER TABLE table_source MOVE PARTITION partition_expr TO TABLE table_dest -``` - -Bu sorgu, veri bölümünü `table_source` -e doğru `table_dest` verileri silme ile `table_source`. - -Sorgunun başarıyla çalışması için aşağıdaki koşulların karşılanması gerekir: - -- Her iki tablo da aynı yapıya sahip olmalıdır. -- Her iki tablo da aynı bölüm anahtarına sahip olmalıdır. -- Her iki tablo da aynı motor ailesi olmalıdır. (çoğaltılmış veya çoğaltılmamış) -- Her iki tablo da aynı depolama ilkesine sahip olmalıdır. - -#### CLEAR COLUMN IN PARTITION {#alter_clear-column-partition} - -``` sql -ALTER TABLE table_name CLEAR COLUMN column_name IN PARTITION partition_expr -``` - -Bir bölümdeki belirtilen sütundaki tüm değerleri sıfırlar. Eğer... `DEFAULT` bir tablo oluştururken yan tümcesi belirlendi, bu sorgu sütun değerini belirtilen varsayılan değere ayarlar. - -Örnek: - -``` sql -ALTER TABLE visits CLEAR COLUMN hour in PARTITION 201902 -``` - -#### FREEZE PARTITION {#alter_freeze-partition} - -``` sql -ALTER TABLE table_name FREEZE [PARTITION partition_expr] -``` - -Bu sorgu, belirtilen bir bölümün yerel yedeğini oluşturur. Eğer... `PARTITION` yan tümcesi atlandı, sorgu aynı anda tüm bölümlerin yedeğini oluşturur. - -!!! note "Not" - Tüm yedekleme işlemi sunucuyu durdurmadan gerçekleştirilir. - -Eski tarz tablolar için bölüm adının önekini belirtebileceğinizi unutmayın (örneğin, ‘2019’)- daha sonra sorgu tüm ilgili bölümler için yedek oluşturur. Bir bölümdeki bölüm ifadesini ayarlama hakkında bilgi edinin [Bölüm ifadesi nasıl belirlenir](#alter-how-to-specify-part-expr). - -Yürütme sırasında, bir veri anlık görüntüsü için sorgu, bir tablo verilerine sabit bağlantılar oluşturur. Hardlinks dizine yerleştirilir `/var/lib/clickhouse/shadow/N/...`, nere: - -- `/var/lib/clickhouse/` yapılandırmada belirtilen çalışma ClickHouse dizinidir. -- `N` yedeklemenin artımlı sayısıdır. - -!!! note "Not" - Kullanıyorsanız [bir tablodaki veri depolama için disk kümesi](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes), bu `shadow/N` dizin tarafından eşleşen veri parçalarını depolamak, her diskte görünür `PARTITION` İfade. - -Dizinlerin aynı yapısı, içinde olduğu gibi yedek içinde oluşturulur `/var/lib/clickhouse/`. Sorgu gerçekleştirir ‘chmod’ tüm dosyalar için, onlara yazmayı yasaklamak. - -Yedeklemeyi oluşturduktan sonra, verileri `/var/lib/clickhouse/shadow/` uzak sunucuya ve sonra yerel sunucudan silin. Not `ALTER t FREEZE PARTITION` sorgu çoğaltılmaz. Yalnızca yerel sunucuda yerel bir yedekleme oluşturur. - -Sorgu neredeyse anında yedekleme oluşturur (ancak önce geçerli sorguları ilgili tabloya çalışmayı bitirmek için bekler). - -`ALTER TABLE t FREEZE PARTITION` tablo meta verilerini değil, yalnızca verileri kopyalar. Tablo meta verilerinin yedeğini almak için dosyayı kopyalayın `/var/lib/clickhouse/metadata/database/table.sql` - -Bir yedekten veri geri yüklemek için aşağıdakileri yapın: - -1. Yoksa tablo oluşturun. Sorguyu görüntülemek için kullanın .sql dosyası (değiştir `ATTACH` içinde ile `CREATE`). -2. Veri kopyalama `data/database/table/` yedekleme içindeki dizin `/var/lib/clickhouse/data/database/table/detached/` dizin. -3. Koşmak `ALTER TABLE t ATTACH PARTITION` verileri bir tabloya eklemek için sorgular. - -Yedeklemeden geri yükleme, sunucuyu durdurmayı gerektirmez. - -Yedekleme ve geri yükleme verileri hakkında daha fazla bilgi için bkz: [Veri Yedekleme](../../operations/backup.md) bölme. - -#### CLEAR INDEX IN PARTITION {#alter_clear-index-partition} - -``` sql -ALTER TABLE table_name CLEAR INDEX index_name IN PARTITION partition_expr -``` - -Sorgu benzer çalışır `CLEAR COLUMN`, ancak bir sütun verileri yerine bir dizini sıfırlar. - -#### FETCH PARTITION {#alter_fetch-partition} - -``` sql -ALTER TABLE table_name FETCH PARTITION partition_expr FROM 'path-in-zookeeper' -``` - -Başka bir sunucudan bir bölüm indirir. Bu sorgu yalnızca çoğaltılmış tablolar için çalışır. - -Sorgu aşağıdakileri yapar: - -1. Bölümü belirtilen parçadan indirir. İçinde ‘path-in-zookeeper’ zookeeper içinde shard için bir yol belirtmeniz gerekir. -2. Sonra sorgu indirilen verileri `detached` directory of the `table_name` Tablo. Kullan... [ATTACH PARTITION\|PART](#alter_attach-partition) tabloya veri eklemek için sorgu. - -Mesela: - -``` sql -ALTER TABLE users FETCH PARTITION 201902 FROM '/clickhouse/tables/01-01/visits'; -ALTER TABLE users ATTACH PARTITION 201902; -``` - -Not thate that: - -- Bu `ALTER ... FETCH PARTITION` sorgu çoğaltılmaz. Bu bölüm için yerleştirir `detached` yalnızca yerel sunucuda dizin. -- Bu `ALTER TABLE ... ATTACH` sorgu çoğaltılır. Verileri tüm yinelemelere ekler. Veriler, kopyalardan birine eklenir. `detached` dizin ve diğerlerine - komşu kopyalardan. - -İndirmeden önce, sistem bölümün olup olmadığını ve tablo yapısının eşleşip eşleşmediğini kontrol eder. En uygun yineleme, sağlıklı yinelemeler otomatik olarak seçilir. - -Sorgu çağrılsa da `ALTER TABLE`, tablo yapısını değiştirmez ve tabloda bulunan verileri hemen değiştirmez. - -#### MOVE PARTITION\|PART {#alter_move-partition} - -Bölümleri veya veri parçalarını başka bir birime veya diske taşır. `MergeTree`- motor masaları. Görmek [Veri depolama için birden fazla blok cihazı kullanma](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes). - -``` sql -ALTER TABLE table_name MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name' -``` - -Bu `ALTER TABLE t MOVE` sorgu: - -- Çoğaltılamaz, çünkü farklı çoğaltmalar farklı depolama ilkelerine sahip olabilir. -- Belirtilen disk veya birim yapılandırılmamışsa bir hata döndürür. Depolama ilkesinde belirtilen veri taşıma koşulları uygulanamazsa, sorgu da bir hata döndürür. -- Durumda bir hata döndürebilir, taşınacak veriler zaten bir arka plan işlemi tarafından taşındığında, eşzamanlı `ALTER TABLE t MOVE` sorgu veya arka plan veri birleştirme sonucu. Bir kullanıcı bu durumda herhangi bir ek eylem gerçekleştirmemelidir. - -Örnek: - -``` sql -ALTER TABLE hits MOVE PART '20190301_14343_16206_438' TO VOLUME 'slow' -ALTER TABLE hits MOVE PARTITION '2019-09-01' TO DISK 'fast_ssd' -``` - -#### Bölüm ifadesi nasıl ayarlanır {#alter-how-to-specify-part-expr} - -Bölüm ifadesini şu şekilde belirtebilirsiniz `ALTER ... PARTITION` farklı şekillerde sorgular: - -- Bu gibi bir değer `partition` sütun `system.parts` Tablo. Mesela, `ALTER TABLE visits DETACH PARTITION 201901`. -- Tablo sütunundan ifade olarak. Sabitler ve sabit ifadeler desteklenir. Mesela, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`. -- Bölüm kimliğini kullanma. Partition ID, dosya sistemindeki ve Zookeeper'daki bölümlerin adları olarak kullanılan bölümün (mümkünse insan tarafından okunabilir) bir dize tanımlayıcısıdır. Bölüm kimliği belirtilmelidir `PARTITION ID` fık .ra, tek tırnak içinde. Mesela, `ALTER TABLE visits DETACH PARTITION ID '201901'`. -- İn the [ALTER ATTACH PART](#alter_attach-partition) ve [DROP DETACHED PART](#alter_drop-detached) sorgu, bir parçanın adını belirtmek için, bir değer ile dize literal kullanın `name` sütun [sistem.detached_parts](../../operations/system-tables.md#system_tables-detached_parts) Tablo. Mesela, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`. - -Bölüm belirtilirken tırnak kullanımı bölüm ifadesi türüne bağlıdır. Örneğin, için `String` yazın, adını tırnak içinde belirtmeniz gerekir (`'`). İçin `Date` ve `Int*` türleri hiçbir tırnak gereklidir. - -Eski stil tablolar için, bölümü bir sayı olarak belirtebilirsiniz `201901` veya bir dize `'201901'`. Yeni stil tabloları için sözdizimi türleri ile daha sıkı (değerleri giriş biçimi için ayrıştırıcı benzer). - -Yukarıdaki tüm kurallar için de geçerlidir [OPTIMIZE](misc.md#misc_operations-optimize) sorgu. Bölümlenmemiş bir tabloyu en iyi duruma getirirken tek bölümü belirtmeniz gerekiyorsa, ifadeyi ayarlayın `PARTITION tuple()`. Mesela: - -``` sql -OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL; -``` - -Örnekleri `ALTER ... PARTITION` sorgular testlerde gösterilmiştir [`00502_custom_partitioning_local`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_local.sql) ve [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql). - -### Tablo TTL ile manipülasyonlar {#manipulations-with-table-ttl} - -Değiştirebilirsiniz [tablo TTL](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) aşağıdaki formun bir isteği ile: - -``` sql -ALTER TABLE table-name MODIFY TTL ttl-expression -``` - -### Alter sorgularının eşzamanlılığı {#synchronicity-of-alter-queries} - -Replicatable olmayan tablolar için, tüm `ALTER` sorgular eşzamanlı olarak gerçekleştirilir. Replicatable tablolar için, sorgu yalnızca uygun eylemler için yönergeler ekler `ZooKeeper` ve eylemlerin kendileri mümkün olan en kısa sürede gerçekleştirilir. Ancak, sorgu tüm yinelemeler üzerinde tamamlanması için bu eylemleri bekleyebilir. - -İçin `ALTER ... ATTACH|DETACH|DROP` sorgular, kullanabilirsiniz `replication_alter_partitions_sync` bekleyen kurmak için ayarlama. -Olası değerler: `0` – do not wait; `1` – only wait for own execution (default); `2` – wait for all. - -### Mutasyonlar {#alter-mutations} - -Mutasyonlar, bir tablodaki satırların değiştirilmesine veya silinmesine izin veren bir alter query varyantıdır. Standart aksine `UPDATE` ve `DELETE` nokta veri değişikliklerine yönelik sorgular, mutasyonlar, bir tablodaki çok sayıda satırı değiştiren ağır işlemler için tasarlanmıştır. İçin desteklenen `MergeTree` çoğaltma desteği olan motorlar da dahil olmak üzere tablo motorları ailesi. - -Varolan tablolar olduğu gibi mutasyonlar için hazırdır(dönüştürme gerekmez), ancak ilk mutasyon bir tabloya uygulandıktan sonra Meta Veri formatı önceki sunucu sürümleriyle uyumsuz hale gelir ve önceki bir sürüme geri dönmek imkansız hale gelir. - -Şu anda mevcut komutlar: - -``` sql -ALTER TABLE [db.]table DELETE WHERE filter_expr -``` - -Bu `filter_expr` tip olmalıdır `UInt8`. Sorgu, bu ifadenin sıfır olmayan bir değer aldığı tablodaki satırları siler. - -``` sql -ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr -``` - -Bu `filter_expr` tip olmalıdır `UInt8`. Bu sorgu, belirtilen sütunların değerlerini, satırlardaki karşılık gelen ifadelerin değerlerine güncelleştirir. `filter_expr` sıfır olmayan bir değer alır. Değerleri kullanarak sütun türüne döküm `CAST` operatör. Birincil veya bölüm anahtarının hesaplanmasında kullanılan sütunları güncelleştirme desteklenmiyor. - -``` sql -ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name -``` - -Sorgu ikincil dizini yeniden oluşturur `name` bölümünde `partition_name`. - -Bir sorgu virgülle ayrılmış birkaç komut içerebilir. - -\* MergeTree tabloları mutasyonları için tüm veri parçalarını yeniden yazarak yürütün. Atomiklik yoktur-parçalar, hazır oldukları anda mutasyona uğramış parçalar için ikame edilir ve bir `SELECT` bir mutasyon sırasında yürütülmeye başlayan sorgu, henüz mutasyona uğramamış olan parçalardan gelen verilerle birlikte mutasyona uğramış olan parçalardan gelen verileri görecektir. - -Mutasyonlar tamamen yaratılış sırasına göre sıralanır ve her bir parçaya bu sırayla uygulanır. Mutasyonlar da kısmen ekler ile sıralanır-mutasyon gönderilmeden önce tabloya eklenen veriler mutasyona uğrayacak ve bundan sonra eklenen veriler mutasyona uğramayacaktır. Mutasyonların ekleri hiçbir şekilde engellemediğini unutmayın. - -Mutasyon girişi eklendikten hemen sonra bir mutasyon sorgusu döner(çoğaltılmış tablolar Zookeeper'a, çoğaltılmamış tablolar için-dosya sistemine). Mutasyonun kendisi sistem profili ayarlarını kullanarak eşzamansız olarak yürütür. Mutasyonların ilerlemesini izlemek için kullanabilirsiniz [`system.mutations`](../../operations/system-tables.md#system_tables-mutations) Tablo. Başarıyla gönderilen BIR mutasyon, ClickHouse sunucuları yeniden başlatılmış olsa bile yürütmeye devam edecektir. Gönderildikten sonra mutasyonu geri almanın bir yolu yoktur, ancak mutasyon herhangi bir nedenle sıkışmışsa, [`KILL MUTATION`](misc.md#kill-mutation) sorgu. - -Bitmiş mutasyonlar için girişler hemen silinmez (korunmuş girişlerin sayısı, `finished_mutations_to_keep` depolama motoru parametresi). Eski mutasyon girişleri silinir. - -## ALTER USER {#alter-user-statement} - -ClickHouse kullanıcı hesaplarını değiştirir. - -### Sözdizimi {#alter-user-syntax} - -``` sql -ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name] - [RENAME TO new_name] - [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] - [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] - [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] -``` - -### Açıklama {#alter-user-dscr} - -Kullanmak `ALTER USER` sen olmalı [ALTER USER](grant.md#grant-access-management) ayrıcalık. - -### Örnekler {#alter-user-examples} - -Verilen rolleri varsayılan olarak ayarla: - -``` sql -ALTER USER user DEFAULT ROLE role1, role2 -``` - -Roller daha önce bir kullanıcıya verilmezse, ClickHouse bir istisna atar. - -Verilen tüm rolleri varsayılan olarak ayarlayın: - -``` sql -ALTER USER user DEFAULT ROLE ALL -``` - -Gelecekte bir kullanıcıya bir rol verilecekse, otomatik olarak varsayılan hale gelecektir. - -Verilen tüm rolleri varsayılan olarak ayarlama `role1` ve `role2`: - -``` sql -ALTER USER user DEFAULT ROLE ALL EXCEPT role1, role2 -``` - -## ALTER ROLE {#alter-role-statement} - -Rolleri değiştirir. - -### Sözdizimi {#alter-role-syntax} - -``` sql -ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name] - [RENAME TO new_name] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] -``` - -## ALTER ROW POLICY {#alter-row-policy-statement} - -Satır ilkesini değiştirir. - -### Sözdizimi {#alter-row-policy-syntax} - -``` sql -ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]table - [RENAME TO new_name] - [AS {PERMISSIVE | RESTRICTIVE}] - [FOR SELECT] - [USING {condition | NONE}][,...] - [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] -``` - -## ALTER QUOTA {#alter-quota-statement} - -Kotaları değiştirir. - -### Sözdizimi {#alter-quota-syntax} - -``` sql -ALTER QUOTA [IF EXISTS] name [ON CLUSTER cluster_name] - [RENAME TO new_name] - [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] - [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY | WEEK | MONTH | QUARTER | YEAR} - {MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] | - NO LIMITS | TRACKING ONLY} [,...]] - [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] -``` - -## ALTER SETTINGS PROFILE {#alter-settings-profile-statement} - -Kotaları değiştirir. - -### Sözdizimi {#alter-settings-profile-syntax} - -``` sql -ALTER SETTINGS PROFILE [IF EXISTS] name [ON CLUSTER cluster_name] - [RENAME TO new_name] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/alter/) diff --git a/docs/tr/sql-reference/statements/create.md b/docs/tr/sql-reference/statements/create.md deleted file mode 100644 index 78390564880..00000000000 --- a/docs/tr/sql-reference/statements/create.md +++ /dev/null @@ -1,502 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 35 -toc_title: CREATE ---- - -# Sorgu oluştur {#create-queries} - -## CREATE DATABASE {#query-language-create-database} - -Veritabanı oluşturur. - -``` sql -CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] [ENGINE = engine(...)] -``` - -### Yanlar {#clauses} - -- `IF NOT EXISTS` - Eğer... `db_name` veritabanı zaten var, daha sonra ClickHouse yeni bir veritabanı oluşturmuyor ve: - - - If yan tümcesi belirtilmişse bir istisna atmaz. - - Bir istisna atar if yan tümcesi belirtilmemiş. - -- `ON CLUSTER` - ClickHouse oluşturur `db_name` belirtilen bir kümenin tüm sunucularında veritabanı. - -- `ENGINE` - - - [MySQL](../../engines/database-engines/mysql.md) - Uzak MySQL sunucusundan veri almanızı sağlar. - Varsayılan olarak, ClickHouse kendi kullanır [Veritabanı Altyapısı](../../engines/database-engines/index.md). - -## CREATE TABLE {#create-table-query} - -Bu `CREATE TABLE` sorgu çeşitli formlara sahip olabilir. - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [compression_codec] [TTL expr2], - ... -) ENGINE = engine -``` - -Adlı bir tablo oluşturur ‘name’ in the ‘db’ veritabanı veya geçerli veritabanı ise ‘db’ küme değil, parantez içinde belirtilen yapı ve ‘engine’ motor. -Tablonun yapısı sütun açıklamalarının bir listesidir. Dizinler altyapısı tarafından destekleniyorsa, tablo altyapısı için parametreler olarak gösterilir. - -Bir sütun açıklaması `name type` en basit durumda. Örnek: `RegionID UInt32`. -İfadeler varsayılan değerler için de tanımlanabilir (aşağıya bakın). - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2 [ENGINE = engine] -``` - -Başka bir tablo ile aynı yapıya sahip bir tablo oluşturur. Tablo için farklı bir motor belirtebilirsiniz. Motor belirtilmemişse, aynı motor için olduğu gibi kullanılacaktır `db2.name2` Tablo. - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name AS table_function() -``` - -Yapısı ve veri tarafından döndürülen bir tablo oluşturur. [tablo fonksiyonu](../table-functions/index.md#table-functions). - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ... -``` - -Sonucu gibi bir yapıya sahip bir tablo oluşturur `SELECT` Sorgu, ile ‘engine’ motor ve SELECT verilerle doldurur. - -Her durumda, eğer `IF NOT EXISTS` tablo zaten varsa, sorgu bir hata döndürmez. Bu durumda, sorgu hiçbir şey yapmaz. - -Sonra başka maddeler olabilir `ENGINE` sorguda yan tümcesi. Açıklamalarda tabloların nasıl oluşturulacağına ilişkin ayrıntılı belgelere bakın [masa motorları](../../engines/table-engines/index.md#table_engines). - -### Varsayılan Değerler {#create-default-values} - -Sütun açıklaması, aşağıdaki yollardan biriyle varsayılan değer için bir ifade belirtebilir:`DEFAULT expr`, `MATERIALIZED expr`, `ALIAS expr`. -Örnek: `URLDomain String DEFAULT domain(URL)`. - -Varsayılan değer için bir ifade tanımlanmamışsa, varsayılan değerler sayılar için sıfırlar, dizeler için boş dizeler, diziler için boş diziler ve `1970-01-01` tarihler için veya zero unix timestamp zamanla tarihler için. Boş alanlar desteklenmez. - -Varsayılan ifade tanımlanmışsa, sütun türü isteğe bağlıdır. Açıkça tanımlanmış bir tür yoksa, varsayılan ifade türü kullanılır. Örnek: `EventDate DEFAULT toDate(EventTime)` – the ‘Date’ türü için kullanılacak ‘EventDate’ sütun. - -Veri türü ve varsayılan ifade açıkça tanımlanırsa, bu ifade type casting işlevleri kullanılarak belirtilen türe aktarılır. Örnek: `Hits UInt32 DEFAULT 0` aynı şeyi ifade eder `Hits UInt32 DEFAULT toUInt32(0)`. - -Default expressions may be defined as an arbitrary expression from table constants and columns. When creating and changing the table structure, it checks that expressions don't contain loops. For INSERT, it checks that expressions are resolvable – that all columns they can be calculated from have been passed. - -`DEFAULT expr` - -Normal varsayılan değer. INSERT sorgusu karşılık gelen sütunu belirtmezse, ilgili ifadeyi hesaplayarak doldurulur. - -`MATERIALIZED expr` - -Somut ifade. Böyle bir sütun INSERT için belirtilemez, çünkü her zaman hesaplanır. -Sütun listesi olmayan bir ekleme için bu sütunlar dikkate alınmaz. -Buna ek olarak, bir SELECT sorgusunda Yıldız İşareti kullanıldığında bu sütun değiştirilmez. Bu, dökümü kullanarak elde edilen değişmezi korumaktır `SELECT *` sütun listesini belirtmeden INSERT kullanarak tabloya geri eklenebilir. - -`ALIAS expr` - -Eşanlamlı sözcük. Böyle bir sütun tabloda hiç depolanmaz. -Değerleri bir tabloya eklenemez ve bir SELECT sorgusunda Yıldız İşareti kullanılırken değiştirilmez. -Sorgu ayrıştırma sırasında diğer ad genişletilirse, seçimlerde kullanılabilir. - -Yeni sütunlar eklemek için ALTER sorgusunu kullanırken, bu sütunlar için eski veriler yazılmaz. Bunun yerine, yeni sütunlar için değerleri olmayan eski verileri okurken, ifadeler varsayılan olarak anında hesaplanır. Ancak, ifadeleri çalıştırmak sorguda belirtilmeyen farklı sütunlar gerektiriyorsa, bu sütunlar ayrıca okunur, ancak yalnızca buna ihtiyaç duyan veri blokları için okunur. - -Bir tabloya yeni bir sütun eklerseniz, ancak daha sonra varsayılan ifadesini değiştirirseniz, eski veriler için kullanılan değerler değişir (değerlerin diskte depolanmadığı veriler için). Arka plan birleştirmeleri çalıştırırken, birleştirme parçalarından birinde eksik olan sütunların verileri birleştirilmiş parçaya yazıldığını unutmayın. - -İç içe geçmiş veri yapılarındaki öğeler için varsayılan değerleri ayarlamak mümkün değildir. - -### Kısıtlamalar {#constraints} - -Sütun açıklamaları kısıtlamaları ile birlikte tanımlanabilir: - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], - ... - CONSTRAINT constraint_name_1 CHECK boolean_expr_1, - ... -) ENGINE = engine -``` - -`boolean_expr_1` herhangi bir Boole ifadesi ile olabilir. Tablo için kısıtlamalar tanımlanırsa, her biri her satır için kontrol edilir `INSERT` query. If any constraint is not satisfied — server will raise an exception with constraint name and checking expression. - -Büyük miktarda kısıtlama eklemek, büyük `INSERT` sorgular. - -### TTL ifadesi {#ttl-expression} - -Değerler için depolama süresini tanımlar. Sadece MergeTree-family tabloları için belirtilebilir. Ayrıntılı açıklama için, bkz. [Sütunlar ve tablolar için TTL](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl). - -### Sütun Sıkıştırma Kodekleri {#codecs} - -Varsayılan olarak, ClickHouse `lz4` sıkıştırma yöntemi. İçin `MergeTree`- motor ailesi varsayılan sıkıştırma yöntemini değiştirebilirsiniz [sıkıştırma](../../operations/server-configuration-parameters/settings.md#server-settings-compression) bir sunucu yapılandırması bölümü. Her bir sütun için sıkıştırma yöntemini de tanımlayabilirsiniz. `CREATE TABLE` sorgu. - -``` sql -CREATE TABLE codec_example -( - dt Date CODEC(ZSTD), - ts DateTime CODEC(LZ4HC), - float_value Float32 CODEC(NONE), - double_value Float64 CODEC(LZ4HC(9)) - value Float32 CODEC(Delta, ZSTD) -) -ENGINE = -... -``` - -Bir codec bileşeni belirtilmişse, varsayılan codec bileşeni geçerli değildir. Kodekler bir boru hattında birleştirilebilir, örneğin, `CODEC(Delta, ZSTD)`. Projeniz için en iyi codec kombinasyonunu seçmek için, Altınlıkta açıklanana benzer kriterler geçirin [ClickHouse verimliliğini artırmak için yeni Kodlamalar](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) Makale. - -!!! warning "Uyarıcı" - ClickHouse veritabanı dosyalarını harici yardımcı programlarla açamazsınız `lz4`. Bunun yerine, özel kullanın [clickhouse-kompresör](https://github.com/ClickHouse/ClickHouse/tree/master/programs/compressor) program. - -Sıkıştırma Aşağıdaki tablo motorları için desteklenir: - -- [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) aile. Sütun sıkıştırma kodeklerini destekler ve varsayılan sıkıştırma yöntemini seçerek [sıkıştırma](../../operations/server-configuration-parameters/settings.md#server-settings-compression) ayarlar. -- [Günlük](../../engines/table-engines/log-family/index.md) aile. Kullanır `lz4` sıkıştırma yöntemi varsayılan olarak ve sütun sıkıştırma codec destekler. -- [Koymak](../../engines/table-engines/special/set.md). Yalnızca varsayılan sıkıştırmayı destekledi. -- [Katmak](../../engines/table-engines/special/join.md). Yalnızca varsayılan sıkıştırmayı destekledi. - -ClickHouse ortak amaçlı codec ve özel codec destekler. - -#### Özel Kodekler {#create-query-specialized-codecs} - -Bu kodekler, verilerin belirli özelliklerini kullanarak sıkıştırmayı daha etkili hale getirmek için tasarlanmıştır. Bu kodeklerden bazıları verileri kendileri sıkıştırmaz. Bunun yerine, verileri ortak bir amaç için hazırlarlar codec, bu hazırlık olmadan daha iyi sıkıştırır. - -Özel kodekler: - -- `Delta(delta_bytes)` — Compression approach in which raw values are replaced by the difference of two neighboring values, except for the first value that stays unchanged. Up to `delta_bytes` delta değerlerini saklamak için kullanılır, böylece `delta_bytes` ham değerlerin maksimum boyutudur. Mümkün `delta_bytes` değerler: 1, 2, 4, 8. İçin varsayılan değer `delta_bytes` oluyor `sizeof(type)` 1, 2, 4 veya 8'e eşitse. Diğer tüm durumlarda, 1. -- `DoubleDelta` — Calculates delta of deltas and writes it in compact binary form. Optimal compression rates are achieved for monotonic sequences with a constant stride, such as time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. Uses 1 extra bit for 32-byte deltas: 5-bit prefixes instead of 4-bit prefixes. For additional information, see Compressing Time Stamps in [Gorilla: Hızlı, Ölçeklenebilir, Bellek İçi Zaman Serisi Veritabanı](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). -- `Gorilla` — Calculates XOR between current and previous value and writes it in compact binary form. Efficient when storing a series of floating point values that change slowly, because the best compression rate is achieved when neighboring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see Compressing Values in [Gorilla: Hızlı, Ölçeklenebilir, Bellek İçi Zaman Serisi Veritabanı](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). -- `T64` — Compression approach that crops unused high bits of values in integer data types (including `Enum`, `Date` ve `DateTime`). Algoritmasının her adımında, codec 64 değerden oluşan bir blok alır, 64x64 bit matrisine koyar, aktarır, kullanılmayan değer bitlerini kırpar ve gerisini bir dizi olarak döndürür. Kullanılmayan bitler, sıkıştırmanın kullanıldığı tüm veri bölümündeki maksimum ve minimum değerler arasında farklılık göstermeyen bitlerdir. - -`DoubleDelta` ve `Gorilla` kodekler, Gorilla TSDB'DE sıkıştırma algoritmasının bileşenleri olarak kullanılır. Gorilla yaklaşımı, zaman damgaları ile yavaş yavaş değişen değerler dizisi olduğunda senaryolarda etkilidir. Zaman damgaları tarafından etkili bir şekilde sıkıştırılır `DoubleDelta` codec ve değerler etkin bir şekilde sıkıştırılır `Gorilla` codec. Örneğin, etkili bir şekilde saklanan bir tablo elde etmek için, aşağıdaki yapılandırmada oluşturabilirsiniz: - -``` sql -CREATE TABLE codec_example -( - timestamp DateTime CODEC(DoubleDelta), - slow_values Float32 CODEC(Gorilla) -) -ENGINE = MergeTree() -``` - -#### Genel Amaçlı Kodekler {#create-query-general-purpose-codecs} - -Cod codecsec codecs'ler: - -- `NONE` — No compression. -- `LZ4` — Lossless [veri sıkıştırma algoritması](https://github.com/lz4/lz4) varsayılan olarak kullanılır. Lz4 hızlı sıkıştırma uygular. -- `LZ4HC[(level)]` — LZ4 HC (high compression) algorithm with configurable level. Default level: 9. Setting `level <= 0` varsayılan düzeyi uygular. Olası seviyeleri: \[1, 12\]. Önerilen seviye aralığı: \[4, 9\]. -- `ZSTD[(level)]` — [Zstd sıkıştırma algoritması](https://en.wikipedia.org/wiki/Zstandard) yapılandırılabilir ile `level`. Olası seviyeler: \[1, 22\]. Varsayılan değer: 1. - -Yüksek Sıkıştırma seviyeleri asimetrik senaryolar için kullanışlıdır, örneğin bir kez sıkıştırın, tekrar tekrar sıkıştırın. Daha yüksek seviyeler daha iyi sıkıştırma ve daha yüksek CPU kullanımı anlamına gelir. - -## Geçici Tablolar {#temporary-tables} - -ClickHouse aşağıdaki özelliklere sahip geçici tabloları destekler: - -- Bağlantı kaybolursa da dahil olmak üzere oturum sona erdiğinde geçici tablolar kaybolur. -- Geçici bir tablo yalnızca bellek altyapısını kullanır. -- DB geçici bir tablo için belirtilemez. Veritabanları dışında oluşturulur. -- Tüm küme sunucularında dağıtılmış DDL sorgusu ile geçici bir tablo oluşturmak imkansız (kullanarak `ON CLUSTER`): bu tablo yalnızca geçerli oturumda bulunur. -- Geçici bir tablo başka bir ile aynı ada sahip ve bir sorgu DB belirtmeden tablo adını belirtir, geçici tablo kullanılır. -- Dağıtılmış sorgu işleme için bir sorguda kullanılan geçici tablolar uzak sunuculara geçirilir. - -Geçici bir tablo oluşturmak için aşağıdaki sözdizimini kullanın: - -``` sql -CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... -) -``` - -Çoğu durumda, geçici tablolar el ile oluşturulmaz, ancak bir sorgu için veya dağıtılmış için dış verileri kullanırken `(GLOBAL) IN`. Daha fazla bilgi için uygun bölümlere bakın - -İle tabloları kullanmak mümkündür [Motor = bellek](../../engines/table-engines/special/memory.md) geçici tablolar yerine. - -## Dağıtılmış DDL sorguları (küme yan tümcesinde) {#distributed-ddl-queries-on-cluster-clause} - -Bu `CREATE`, `DROP`, `ALTER`, ve `RENAME` sorgular, bir kümede dağıtılmış yürütmeyi destekler. -Örneğin, aşağıdaki sorgu oluşturur `all_hits` `Distributed` her ana bilgisayarda tablo `cluster`: - -``` sql -CREATE TABLE IF NOT EXISTS all_hits ON CLUSTER cluster (p Date, i Int32) ENGINE = Distributed(cluster, default, hits) -``` - -Bu sorguları doğru bir şekilde çalıştırmak için, her ana bilgisayarın aynı küme tanımına sahip olması gerekir (senkronizasyon yapılandırmalarını basitleştirmek için zookeeper'dan değiştirmeleri kullanabilirsiniz). Ayrıca ZooKeeper sunucularına bağlanmaları gerekir. -Bazı ana bilgisayarlar şu anda mevcut olmasa bile, sorgunun yerel sürümü sonunda kümedeki her ana bilgisayarda uygulanır. Tek bir ana makine içinde sorguları yürütme sırası garanti edilir. - -## CREATE VIEW {#create-view} - -``` sql -CREATE [MATERIALIZED] VIEW [IF NOT EXISTS] [db.]table_name [TO[db.]name] [ENGINE = engine] [POPULATE] AS SELECT ... -``` - -Bir görünüm oluşturur. İki tür görüş vardır: normal ve SOMUTLAŞTIRILMIŞ. - -Normal görünümler herhangi bir veri depolamaz, ancak başka bir tablodan bir okuma gerçekleştirir. Başka bir deyişle, normal bir görünüm kaydedilmiş bir sorgudan başka bir şey değildir. Bir görünümden okurken, bu kaydedilmiş sorgu FROM yan tümcesinde bir alt sorgu olarak kullanılır. - -Örnek olarak, bir görünüm oluşturduğunuzu varsayalım: - -``` sql -CREATE VIEW view AS SELECT ... -``` - -ve bir sorgu yazdı: - -``` sql -SELECT a, b, c FROM view -``` - -Bu sorgu, alt sorguyu kullanmaya tam olarak eşdeğerdir: - -``` sql -SELECT a, b, c FROM (SELECT ...) -``` - -Materialized görünümler, ilgili SELECT sorgusu tarafından dönüştürülmüş verileri depolar. - -Olmadan hayata bir görünüm oluştururken `TO [db].[table]`, you must specify ENGINE – the table engine for storing data. - -İle somutlaştırılmış bir görünüm oluştururken `TO [db].[table]`, kullanma mustmalısınız `POPULATE`. - -Materialized görünüm aşağıdaki gibi düzenlenmiştir: SELECT belirtilen tabloya veri eklerken, eklenen verilerin bir kısmı bu SELECT sorgusu tarafından dönüştürülür ve sonuç görünümde eklenir. - -Doldur belirtirseniz, varolan tablo verilerini oluştururken görünümde, sanki bir `CREATE TABLE ... AS SELECT ...` . Aksi takdirde, sorgu yalnızca görünümü oluşturduktan sonra tabloya eklenen verileri içerir. Görünüm oluşturma sırasında tabloya eklenen veriler EKLENMEYECEĞİNDEN, doldur kullanmanızı önermiyoruz. - -A `SELECT` sorgu içerebilir `DISTINCT`, `GROUP BY`, `ORDER BY`, `LIMIT`… Note that the corresponding conversions are performed independently on each block of inserted data. For example, if `GROUP BY` ayarlanır, veri ekleme sırasında toplanır, ancak yalnızca tek bir eklenen veri paketi içinde toplanır. Veriler daha fazla toplanmayacaktır. Özel durum, bağımsız olarak veri toplama, gibi gerçekleştiren bir motor kullanırken olur `SummingMergeTree`. - -Yürütme `ALTER` somut görünümlerle ilgili sorgular tam olarak geliştirilmemiştir, bu nedenle rahatsız edici olabilirler. Eğer hayata görünüm inşaat kullanıyorsa `TO [db.]name` yapabilirsiniz `DETACH` the view, run `ALTER` hedef tablo için ve sonra `ATTACH` daha önce müstakil (`DETACH`) görünüm. - -Görünümler normal tablolarla aynı görünür. Örneğin, bunlar sonucu listelenir `SHOW TABLES` sorgu. - -Görünümleri silmek için ayrı bir sorgu yok. Bir görünümü silmek için şunları kullanın `DROP TABLE`. - -## CREATE DICTIONARY {#create-dictionary-query} - -``` sql -CREATE DICTIONARY [IF NOT EXISTS] [db.]dictionary_name [ON CLUSTER cluster] -( - key1 type1 [DEFAULT|EXPRESSION expr1] [HIERARCHICAL|INJECTIVE|IS_OBJECT_ID], - key2 type2 [DEFAULT|EXPRESSION expr2] [HIERARCHICAL|INJECTIVE|IS_OBJECT_ID], - attr1 type2 [DEFAULT|EXPRESSION expr3], - attr2 type2 [DEFAULT|EXPRESSION expr4] -) -PRIMARY KEY key1, key2 -SOURCE(SOURCE_NAME([param1 value1 ... paramN valueN])) -LAYOUT(LAYOUT_NAME([param_name param_value])) -LIFETIME({MIN min_val MAX max_val | max_val}) -``` - -Oluşturuyor [dış sözlük](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) verilen ile [yapılı](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md), [kaynaklı](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md), [düzen](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md) ve [ömür](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md). - -Dış sözlük yapısı özniteliklerden oluşur. Sözlük öznitelikleri tablo sütunlarına benzer şekilde belirtilir. Tek gerekli öznitelik özelliği türüdür, diğer tüm özelliklerin varsayılan değerleri olabilir. - -Sözlüğe bağlı olarak [düzen](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md) bir veya daha fazla öznitelik sözlük anahtarları olarak belirtilebilir. - -Daha fazla bilgi için, bkz. [Dış Söz Dictionarieslükler](../dictionaries/external-dictionaries/external-dicts.md) bölme. - -## CREATE USER {#create-user-statement} - -Oluşturur bir [kullanıcı hesabı](../../operations/access-rights.md#user-account-management). - -### Sözdizimi {#create-user-syntax} - -``` sql -CREATE USER [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name] - [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}] - [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] - [DEFAULT ROLE role [,...]] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] -``` - -#### Tanıma {#identification} - -Kullanıcı tanımlama birden çok yolu vardır: - -- `IDENTIFIED WITH no_password` -- `IDENTIFIED WITH plaintext_password BY 'qwerty'` -- `IDENTIFIED WITH sha256_password BY 'qwerty'` veya `IDENTIFIED BY 'password'` -- `IDENTIFIED WITH sha256_hash BY 'hash'` -- `IDENTIFIED WITH double_sha1_password BY 'qwerty'` -- `IDENTIFIED WITH double_sha1_hash BY 'hash'` - -#### Kullanıcı Host {#user-host} - -Kullanıcı ana bilgisayar, ClickHouse sunucusuna bağlantı kurulabilen bir ana bilgisayardır. Ev sahibi belirtilebilir `HOST` aşağıdaki yollarla sorgu bölümü: - -- `HOST IP 'ip_address_or_subnetwork'` — User can connect to ClickHouse server only from the specified IP address or a [alt ağ](https://en.wikipedia.org/wiki/Subnetwork). Örnekler: `HOST IP '192.168.0.0/16'`, `HOST IP '2001:DB8::/32'`. Üretimde kullanım için, sadece belirtin `HOST IP` elemanları (IP adresleri ve maskeleri), kullanıl ,dığından beri `host` ve `host_regexp` ekstra gecikmeye neden olabilir. -- `HOST ANY` — User can connect from any location. This is default option. -- `HOST LOCAL` — User can connect only locally. -- `HOST NAME 'fqdn'` — User host can be specified as FQDN. For example, `HOST NAME 'mysite.com'`. -- `HOST NAME REGEXP 'regexp'` — You can use [pcre](http://www.pcre.org/) kullanıcı ana bilgisayarlarını belirtirken düzenli ifadeler. Mesela, `HOST NAME REGEXP '.*\.mysite\.com'`. -- `HOST LIKE 'template'` — Allows you use the [LIKE](../functions/string-search-functions.md#function-like) kullanıcı ana filtrelemek için operatör. Mesela, `HOST LIKE '%'` eşdeğ toer equivalentdir `HOST ANY`, `HOST LIKE '%.mysite.com'` tüm host filtersları filtreler `mysite.com` etki. - -Host belirtme başka bir yolu kullanmaktır `@` kullanıcı adı ile sözdizimi. Örnekler: - -- `CREATE USER mira@'127.0.0.1'` — Equivalent to the `HOST IP` sözdizimi. -- `CREATE USER mira@'localhost'` — Equivalent to the `HOST LOCAL` sözdizimi. -- `CREATE USER mira@'192.168.%.%'` — Equivalent to the `HOST LIKE` sözdizimi. - -!!! info "Uyarıcı" - ClickHouse davranır `user_name@'address'` bir bütün olarak bir kullanıcı adı olarak. Böylece, teknik olarak birden fazla kullanıcı oluşturabilirsiniz `user_name` ve sonra farklı yapılar `@`. Ben bunu tavsiye etmiyoruz. - -### Örnekler {#create-user-examples} - -Kullanıcı hesabı oluşturma `mira` şifre ile korunmaktadır `qwerty`: - -``` sql -CREATE USER mira HOST IP '127.0.0.1' IDENTIFIED WITH sha256_password BY 'qwerty' -``` - -`mira` ClickHouse sunucusunun çalıştığı ana bilgisayarda istemci uygulamasını başlatmalıdır. - -Kullanıcı hesabı oluşturma `john`, ona roller atayın ve bu rolleri varsayılan yapın: - -``` sql -CREATE USER john DEFAULT ROLE role1, role2 -``` - -Kullanıcı hesabı oluşturma `john` ve gelecekteki tüm rollerini varsayılan hale getirin: - -``` sql -ALTER USER user DEFAULT ROLE ALL -``` - -Ne zaman bazı rol atanacak `john` gelecekte otomatik olarak varsayılan hale gelecektir. - -Kullanıcı hesabı oluşturma `john` ve gelecekteki tüm rollerini varsayılan olarak yapın `role1` ve `role2`: - -``` sql -ALTER USER john DEFAULT ROLE ALL EXCEPT role1, role2 -``` - -## CREATE ROLE {#create-role-statement} - -Oluşturur bir [rol](../../operations/access-rights.md#role-management). - -### Sözdizimi {#create-role-syntax} - -``` sql -CREATE ROLE [IF NOT EXISTS | OR REPLACE] name - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] -``` - -### Açıklama {#create-role-description} - -Rol bir dizi [ayrıcalıklar](grant.md#grant-privileges). Bir rolle verilen bir kullanıcı, bu rolün tüm ayrıcalıklarını alır. - -Bir kullanıcı birden çok rolle atanabilir. Kullanıcılar tarafından keyfi kombinasyonlarda verilen rolleri uygulayabilirsiniz [SET ROLE](misc.md#set-role-statement) deyim. Ayrıcalıkların son kapsamı, uygulanan tüm rollerin tüm ayrıcalıklarının birleştirilmiş kümesidir. Bir kullanıcının doğrudan kullanıcı hesabına verilen ayrıcalıkları varsa, bunlar roller tarafından verilen ayrıcalıklarla da birleştirilir. - -Kullanıcı, kullanıcı girişinde geçerli olan varsayılan rollere sahip olabilir. Varsayılan rolleri ayarlamak için [SET DEFAULT ROLE](misc.md#set-default-role-statement) beyan veya [ALTER USER](alter.md#alter-user-statement) deyim. - -Bir rolü iptal etmek için [REVOKE](revoke.md) deyim. - -Rolü silmek için [DROP ROLE](misc.md#drop-role-statement) deyim. Silinen rol, kendisine verilen tüm kullanıcılardan ve rollerden otomatik olarak iptal edilir. - -### Örnekler {#create-role-examples} - -``` sql -CREATE ROLE accountant; -GRANT SELECT ON db.* TO accountant; -``` - -Bu sorgu sırası rolü oluşturur `accountant` bu veri okuma ayrıcalığına sahip `accounting` veritabanı. - -Kullanıcıya rol verilmesi `mira`: - -``` sql -GRANT accountant TO mira; -``` - -Rol verildikten sonra kullanıcı bunu kullanabilir ve izin verilen sorguları gerçekleştirebilir. Mesela: - -``` sql -SET ROLE accountant; -SELECT * FROM db.*; -``` - -## CREATE ROW POLICY {#create-row-policy-statement} - -Oluşturur bir [satırlar için filtre](../../operations/access-rights.md#row-policy-management), bir kullanıcı bir tablodan okuyabilir. - -### Sözdizimi {#create-row-policy-syntax} - -``` sql -CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name [ON CLUSTER cluster_name] ON [db.]table - [AS {PERMISSIVE | RESTRICTIVE}] - [FOR SELECT] - [USING condition] - [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] -``` - -#### Bölüm olarak {#create-row-policy-as} - -Bu bölümü kullanarak izin veren veya kısıtlayıcı ilkeler oluşturabilirsiniz. - -İzin verme ilkesi satırlara erişim sağlar. Aynı tabloya uygulanan izin veren politikalar boolean kullanılarak birlikte birleştirilir `OR` operatör. İlkeler varsayılan olarak izinlidir. - -Kısıtlayıcı ilke satıra erişimi kısıtlar. Aynı tabloya uygulanan kısıtlayıcı ilkeler, boolean kullanılarak birlikte birleştirilir `AND` operatör. - -Kısıtlayıcı ilkeler, izin veren süzgeçleri geçen satırlara uygulanır. Kısıtlayıcı ilkeler ayarlarsanız, ancak izin veren ilkeler yoksa, kullanıcı tablodan herhangi bir satır alamaz. - -#### Bölüm için {#create-row-policy-to} - -Bölümünde `TO` örneğin, rollerin ve kullanıcıların karışık bir listesini verebilirsiniz, `CREATE ROW POLICY ... TO accountant, john@localhost`. - -Kelime `ALL` geçerli kullanıcı dahil olmak üzere tüm ClickHouse kullanıcıları anlamına gelir. Kelimeler `ALL EXCEPT` bazı kullanıcıları tüm kullanıcılar listesinden çıkarmak için izin ver, örneğin `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost` - -### Örnekler {#examples} - -- `CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO accountant, john@localhost` -- `CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO ALL EXCEPT mira` - -## CREATE QUOTA {#create-quota-statement} - -Oluşturur bir [kota](../../operations/access-rights.md#quotas-management) bu bir kullanıcıya veya bir role atanabilir. - -### Sözdizimi {#create-quota-syntax} - -``` sql -CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name] - [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] - [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY | WEEK | MONTH | QUARTER | YEAR} - {MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] | - NO LIMITS | TRACKING ONLY} [,...]] - [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] -``` - -### Örnek {#create-quota-example} - -15 ay içinde 123 Sorgu ile geçerli kullanıcı için sorgu sayısını sınır constraintlayın: - -``` sql -CREATE QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 123 TO CURRENT_USER -``` - -## CREATE SETTINGS PROFILE {#create-settings-profile-statement} - -Oluşturur bir [ayarlar profili](../../operations/access-rights.md#settings-profiles-management) bu bir kullanıcıya veya bir role atanabilir. - -### Sözdizimi {#create-settings-profile-syntax} - -``` sql -CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] -``` - -# Örnek {#create-settings-profile-syntax} - -Create the `max_memory_usage_profile` ayar profili için değer ve kısıtlamalarla `max_memory_usage` ayar. At itayın `robin`: - -``` sql -CREATE SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/create/) diff --git a/docs/tr/sql-reference/statements/grant.md b/docs/tr/sql-reference/statements/grant.md deleted file mode 100644 index 61a40a297fc..00000000000 --- a/docs/tr/sql-reference/statements/grant.md +++ /dev/null @@ -1,476 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: cbd8aa9052361a7ee11c209560cff7175c2b8e42 -toc_priority: 39 -toc_title: GRANT ---- - -# GRANT {#grant} - -- Veriyor [ayrıcalıklar](#grant-privileges) kullanıcı hesaplarını veya rollerini tıklamak için. -- Rolleri kullanıcı hesaplarına veya diğer rollere atar. - -Ayrıcalıkları iptal etmek için [REVOKE](../../sql-reference/statements/revoke.md) deyim. Ayrıca, verilen ayrıcalıkları listeleyebilirsiniz [SHOW GRANTS](../../sql-reference/statements/show.md#show-grants-statement) deyim. - -## Ayrıcalık Sözdizimi Verme {#grant-privigele-syntax} - -``` sql -GRANT [ON CLUSTER cluster_name] privilege[(column_name [,...])] [,...] ON {db.table|db.*|*.*|table|*} TO {user | role | CURRENT_USER} [,...] [WITH GRANT OPTION] -``` - -- `privilege` — Type of privilege. -- `role` — ClickHouse user role. -- `user` — ClickHouse user account. - -Bu `WITH GRANT OPTION` madde hibeleri `user` veya `role` yürütme izni ile `GRANT` sorgu. Kullanıcılar sahip oldukları ve daha az olan aynı kapsamdaki ayrıcalıkları verebilir. - -## Rol Sözdizimi Atama {#assign-role-syntax} - -``` sql -GRANT [ON CLUSTER cluster_name] role [,...] TO {user | another_role | CURRENT_USER} [,...] [WITH ADMIN OPTION] -``` - -- `role` — ClickHouse user role. -- `user` — ClickHouse user account. - -Bu `WITH ADMIN OPTION` madde hibeleri [ADMIN OPTION](#admin-option-privilege) ayrıcalık `user` veya `role`. - -## Kullanma {#grant-usage} - -Kullanmak `GRANT` hesabınız olmalıdır var `GRANT OPTION` ayrıcalık. Ayrıcalıkları yalnızca hesap ayrıcalıklarınız kapsamında verebilirsiniz. - -Örneğin, yönetici için ayrıcalıklar verdi `john` sorguya göre hesap: - -``` sql -GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION -``` - -Demek ki `john` yürütme izni var mı: - -- `SELECT x,y FROM db.table`. -- `SELECT x FROM db.table`. -- `SELECT y FROM db.table`. - -`john` idam edilemiyor `SELECT z FROM db.table`. Bu `SELECT * FROM db.table` ayrıca mevcut değildir. Bu sorguyu işlerken, ClickHouse bile herhangi bir veri döndürmez `x` ve `y`. Tek istisna, bir tablo yalnızca `x` ve `y` sütun. Bu durumda ClickHouse tüm verileri döndürür. - -Ayrıca `john` vardır `GRANT OPTION` ayrıcalık, böylece diğer kullanıcılara aynı veya daha küçük kapsamdaki ayrıcalıklar verebilir. - -Ayrıcalıkları belirtme Yıldız İşareti kullanabilirsiniz (`*`) bir tablo veya veritabanı adı yerine. Örneğin, `GRANT SELECT ON db.* TO john` sorgu sağlar `john` yürütmek için `SELECT` tüm tablolar üzerinde sorgu `db` veritabanı. Ayrıca, veritabanı adı atlayabilirsiniz. Bu durumda, geçerli veritabanı için ayrıcalıklar verilir. Mesela, `GRANT SELECT ON * TO john` geçerli veritabanındaki tüm tablolarda ayrıcalık verir, `GRANT SELECT ON mytable TO john` bu ayrıcalığı verir `mytable` geçerli veritabanındaki tablo. - -Erişim `system` veritabanına her zaman izin verilir (bu veritabanı sorguları işlemek için kullanıldığından). - -Tek bir sorguda birden çok hesaba birden çok ayrıcalık verebilirsiniz. Sorgu `GRANT SELECT, INSERT ON *.* TO john, robin` hesaplara izin verir `john` ve `robin` yürütmek için `INSERT` ve `SELECT` sunucudaki tüm veritabanlarındaki tüm tablolar üzerinden sorgular. - -## Ayrıcalıklar {#grant-privileges} - -Ayrıcalık, belirli bir sorgu türünü yürütme iznidir. - -Ayrıcalıklar hiyerarşik bir yapıya sahiptir. İzin verilen sorgular kümesi ayrıcalık kapsamına bağlıdır. - -Ayrıcalıkların hiyerarşisi: - -- [SELECT](#grant-select) -- [INSERT](#grant-insert) -- [ALTER](#grant-alter) - - `ALTER TABLE` - - `ALTER UPDATE` - - `ALTER DELETE` - - `ALTER COLUMN` - - `ALTER ADD COLUMN` - - `ALTER DROP COLUMN` - - `ALTER MODIFY COLUMN` - - `ALTER COMMENT COLUMN` - - `ALTER CLEAR COLUMN` - - `ALTER RENAME COLUMN` - - `ALTER INDEX` - - `ALTER ORDER BY` - - `ALTER ADD INDEX` - - `ALTER DROP INDEX` - - `ALTER MATERIALIZE INDEX` - - `ALTER CLEAR INDEX` - - `ALTER CONSTRAINT` - - `ALTER ADD CONSTRAINT` - - `ALTER DROP CONSTRAINT` - - `ALTER TTL` - - `ALTER MATERIALIZE TTL` - - `ALTER SETTINGS` - - `ALTER MOVE PARTITION` - - `ALTER FETCH PARTITION` - - `ALTER FREEZE PARTITION` - - `ALTER VIEW` - - `ALTER VIEW REFRESH` - - `ALTER VIEW MODIFY QUERY` -- [CREATE](#grant-create) - - `CREATE DATABASE` - - `CREATE TABLE` - - `CREATE VIEW` - - `CREATE DICTIONARY` - - `CREATE TEMPORARY TABLE` -- [DROP](#grant-drop) - - `DROP DATABASE` - - `DROP TABLE` - - `DROP VIEW` - - `DROP DICTIONARY` -- [TRUNCATE](#grant-truncate) -- [OPTIMIZE](#grant-optimize) -- [SHOW](#grant-show) - - `SHOW DATABASES` - - `SHOW TABLES` - - `SHOW COLUMNS` - - `SHOW DICTIONARIES` -- [KILL QUERY](#grant-kill-query) -- [ACCESS MANAGEMENT](#grant-access-management) - - `CREATE USER` - - `ALTER USER` - - `DROP USER` - - `CREATE ROLE` - - `ALTER ROLE` - - `DROP ROLE` - - `CREATE ROW POLICY` - - `ALTER ROW POLICY` - - `DROP ROW POLICY` - - `CREATE QUOTA` - - `ALTER QUOTA` - - `DROP QUOTA` - - `CREATE SETTINGS PROFILE` - - `ALTER SETTINGS PROFILE` - - `DROP SETTINGS PROFILE` - - `SHOW ACCESS` - - `SHOW_USERS` - - `SHOW_ROLES` - - `SHOW_ROW_POLICIES` - - `SHOW_QUOTAS` - - `SHOW_SETTINGS_PROFILES` - - `ROLE ADMIN` -- [SYSTEM](#grant-system) - - `SYSTEM SHUTDOWN` - - `SYSTEM DROP CACHE` - - `SYSTEM DROP DNS CACHE` - - `SYSTEM DROP MARK CACHE` - - `SYSTEM DROP UNCOMPRESSED CACHE` - - `SYSTEM RELOAD` - - `SYSTEM RELOAD CONFIG` - - `SYSTEM RELOAD DICTIONARY` - - `SYSTEM RELOAD EMBEDDED DICTIONARIES` - - `SYSTEM MERGES` - - `SYSTEM TTL MERGES` - - `SYSTEM FETCHES` - - `SYSTEM MOVES` - - `SYSTEM SENDS` - - `SYSTEM DISTRIBUTED SENDS` - - `SYSTEM REPLICATED SENDS` - - `SYSTEM REPLICATION QUEUES` - - `SYSTEM SYNC REPLICA` - - `SYSTEM RESTART REPLICA` - - `SYSTEM FLUSH` - - `SYSTEM FLUSH DISTRIBUTED` - - `SYSTEM FLUSH LOGS` -- [INTROSPECTION](#grant-introspection) - - `addressToLine` - - `addressToSymbol` - - `demangle` -- [SOURCES](#grant-sources) - - `FILE` - - `URL` - - `REMOTE` - - `YSQL` - - `ODBC` - - `JDBC` - - `HDFS` - - `S3` -- [dictGet](#grant-dictget) - -Bu hiyerarşinin nasıl ele alındığına dair örnekler: - -- Bu `ALTER` ayrıcalık diğer her şeyi içerir `ALTER*` ayrıcalıklar. -- `ALTER CONSTRAINT` içerir `ALTER ADD CONSTRAINT` ve `ALTER DROP CONSTRAINT` ayrıcalıklar. - -Ayrıcalıklar farklı düzeylerde uygulanır. Bir seviyeyi bilmek, ayrıcalık için mevcut sözdizimini önerir. - -Seviyeler (düşükten yükseğe): - -- `COLUMN` — Privilege can be granted for column, table, database, or globally. -- `TABLE` — Privilege can be granted for table, database, or globally. -- `VIEW` — Privilege can be granted for view, database, or globally. -- `DICTIONARY` — Privilege can be granted for dictionary, database, or globally. -- `DATABASE` — Privilege can be granted for database or globally. -- `GLOBAL` — Privilege can be granted only globally. -- `GROUP` — Groups privileges of different levels. When `GROUP`- seviye ayrıcalığı verilir, yalnızca kullanılan sözdizimine karşılık gelen grup ayrıcalıkları verilir. - -İzin verilen sözdizimi örnekleri: - -- `GRANT SELECT(x) ON db.table TO user` -- `GRANT SELECT ON db.* TO user` - -İzin verilmeyen sözdizimi örnekleri: - -- `GRANT CREATE USER(x) ON db.table TO user` -- `GRANT CREATE USER ON db.* TO user` - -Özel ayrıcalık [ALL](#grant-all) bir kullanıcı hesabı veya rol için tüm ayrıcalıkları verir. - -Varsayılan olarak, bir kullanıcı hesabının veya rolün ayrıcalıkları yoktur. - -Eğer bir kullanıcı veya rolü herhangi bir imtiyaz varsa, görüntülenen bu [NONE](#grant-none) ayrıcalık. - -Bazı sorgular uygulamalarıyla bir dizi ayrıcalık gerektirir. Örneğin, yürütmek için [RENAME](../../sql-reference/statements/misc.md#misc_operations-rename) sorgu aşağıdaki ayrıcalıklara ihtiyacınız var: `SELECT`, `CREATE TABLE`, `INSERT` ve `DROP TABLE`. - -### SELECT {#grant-select} - -Çalıştırmaya izin verir [SELECT](../../sql-reference/statements/select/index.md) sorgular. - -Ayrıcalık düzeyi: `COLUMN`. - -**Açıklama** - -Bu ayrıcalığa sahip bir kullanıcı yürütebilir `SELECT` belirtilen tablo ve veritabanındaki sütunların belirtilen bir liste üzerinde sorgular. Kullanıcı başka sütunlar içeriyorsa, belirtilen sorgu herhangi bir veri döndürmez. - -Aşağıdaki ayrıcalığı göz önünde bulundurun: - -``` sql -GRANT SELECT(x,y) ON db.table TO john -``` - -Bu ayrıcalık sağlar `john` herhangi bir yürütmek için `SELECT` veri içeren sorgu `x` ve / veya `y` içindeki sütunlar `db.table`, mesela, `SELECT x FROM db.table`. `john` idam edilemiyor `SELECT z FROM db.table`. Bu `SELECT * FROM db.table` ayrıca mevcut değildir. Bu sorguyu işlerken, ClickHouse bile herhangi bir veri döndürmez `x` ve `y`. Tek istisna, bir tablo yalnızca `x` ve `y` sütunlar, bu durumda ClickHouse tüm verileri döndürür. - -### INSERT {#grant-insert} - -Çalıştırmaya izin verir [INSERT](../../sql-reference/statements/insert-into.md) sorgular. - -Ayrıcalık düzeyi: `COLUMN`. - -**Açıklama** - -Bu ayrıcalığa sahip bir kullanıcı yürütebilir `INSERT` belirtilen tablo ve veritabanındaki sütunların belirtilen bir liste üzerinde sorgular. Kullanıcı başka sütunlar içeriyorsa, belirtilen sorgu herhangi bir veri eklemez. - -**Örnek** - -``` sql -GRANT INSERT(x,y) ON db.table TO john -``` - -Verilen ayrıcalık izin verir `john` veri eklemek için `x` ve / veya `y` içindeki sütunlar `db.table`. - -### ALTER {#grant-alter} - -Çalıştırmaya izin verir [ALTER](../../sql-reference/statements/alter.md) aşağıdaki ayrıcalıklar hiyerarşisine göre sorgular: - -- `ALTER`. Düzey: `COLUMN`. - - `ALTER TABLE`. Düzey: `GROUP` - - `ALTER UPDATE`. Düzey: `COLUMN`. Takma ad: `UPDATE` - - `ALTER DELETE`. Düzey: `COLUMN`. Takma ad: `DELETE` - - `ALTER COLUMN`. Düzey: `GROUP` - - `ALTER ADD COLUMN`. Düzey: `COLUMN`. Takma ad: `ADD COLUMN` - - `ALTER DROP COLUMN`. Düzey: `COLUMN`. Takma ad: `DROP COLUMN` - - `ALTER MODIFY COLUMN`. Düzey: `COLUMN`. Takma ad: `MODIFY COLUMN` - - `ALTER COMMENT COLUMN`. Düzey: `COLUMN`. Takma ad: `COMMENT COLUMN` - - `ALTER CLEAR COLUMN`. Düzey: `COLUMN`. Takma ad: `CLEAR COLUMN` - - `ALTER RENAME COLUMN`. Düzey: `COLUMN`. Takma ad: `RENAME COLUMN` - - `ALTER INDEX`. Düzey: `GROUP`. Takma ad: `INDEX` - - `ALTER ORDER BY`. Düzey: `TABLE`. Takma ad: `ALTER MODIFY ORDER BY`, `MODIFY ORDER BY` - - `ALTER ADD INDEX`. Düzey: `TABLE`. Takma ad: `ADD INDEX` - - `ALTER DROP INDEX`. Düzey: `TABLE`. Takma ad: `DROP INDEX` - - `ALTER MATERIALIZE INDEX`. Düzey: `TABLE`. Takma ad: `MATERIALIZE INDEX` - - `ALTER CLEAR INDEX`. Düzey: `TABLE`. Takma ad: `CLEAR INDEX` - - `ALTER CONSTRAINT`. Düzey: `GROUP`. Takma ad: `CONSTRAINT` - - `ALTER ADD CONSTRAINT`. Düzey: `TABLE`. Takma ad: `ADD CONSTRAINT` - - `ALTER DROP CONSTRAINT`. Düzey: `TABLE`. Takma ad: `DROP CONSTRAINT` - - `ALTER TTL`. Düzey: `TABLE`. Takma ad: `ALTER MODIFY TTL`, `MODIFY TTL` - - `ALTER MATERIALIZE TTL`. Düzey: `TABLE`. Takma ad: `MATERIALIZE TTL` - - `ALTER SETTINGS`. Düzey: `TABLE`. Takma ad: `ALTER SETTING`, `ALTER MODIFY SETTING`, `MODIFY SETTING` - - `ALTER MOVE PARTITION`. Düzey: `TABLE`. Takma ad: `ALTER MOVE PART`, `MOVE PARTITION`, `MOVE PART` - - `ALTER FETCH PARTITION`. Düzey: `TABLE`. Takma ad: `FETCH PARTITION` - - `ALTER FREEZE PARTITION`. Düzey: `TABLE`. Takma ad: `FREEZE PARTITION` - - `ALTER VIEW` Düzey: `GROUP` - - `ALTER VIEW REFRESH`. Düzey: `VIEW`. Takma ad: `ALTER LIVE VIEW REFRESH`, `REFRESH VIEW` - - `ALTER VIEW MODIFY QUERY`. Düzey: `VIEW`. Takma ad: `ALTER TABLE MODIFY QUERY` - -Bu hiyerarşinin nasıl ele alındığına dair örnekler: - -- Bu `ALTER` ayrıcalık diğer her şeyi içerir `ALTER*` ayrıcalıklar. -- `ALTER CONSTRAINT` içerir `ALTER ADD CONSTRAINT` ve `ALTER DROP CONSTRAINT` ayrıcalıklar. - -**Not** - -- Bu `MODIFY SETTING` ayrıcalık, tablo altyapısı ayarlarını değiştirmenize izin verir. Ayarları veya sunucu yapılandırma parametrelerini etkilemez. -- Bu `ATTACH` operasyon ihtiyacı [CREATE](#grant-create) ayrıcalık. -- Bu `DETACH` operasyon ihtiyacı [DROP](#grant-drop) ayrıcalık. -- Mutasyonu durdurmak için [KILL MUTATION](../../sql-reference/statements/misc.md#kill-mutation) sorgu, bu mutasyonu başlatmak için bir ayrıcalığa sahip olmanız gerekir. Örneğin, durdurmak istiyorsanız `ALTER UPDATE` sorgu, ihtiyacınız olan `ALTER UPDATE`, `ALTER TABLE`, veya `ALTER` ayrıcalık. - -### CREATE {#grant-create} - -Çalıştırmaya izin verir [CREATE](../../sql-reference/statements/create.md) ve [ATTACH](../../sql-reference/statements/misc.md#attach) DDL-aşağıdaki ayrıcalıklar hiyerarşisine göre sorgular: - -- `CREATE`. Düzey: `GROUP` - - `CREATE DATABASE`. Düzey: `DATABASE` - - `CREATE TABLE`. Düzey: `TABLE` - - `CREATE VIEW`. Düzey: `VIEW` - - `CREATE DICTIONARY`. Düzey: `DICTIONARY` - - `CREATE TEMPORARY TABLE`. Düzey: `GLOBAL` - -**Not** - -- Oluşturulan tabloyu silmek için bir kullanıcının ihtiyacı vardır [DROP](#grant-drop). - -### DROP {#grant-drop} - -Çalıştırmaya izin verir [DROP](../../sql-reference/statements/misc.md#drop) ve [DETACH](../../sql-reference/statements/misc.md#detach) aşağıdaki ayrıcalıklar hiyerarşisine göre sorgular: - -- `DROP`. Düzey: - - `DROP DATABASE`. Düzey: `DATABASE` - - `DROP TABLE`. Düzey: `TABLE` - - `DROP VIEW`. Düzey: `VIEW` - - `DROP DICTIONARY`. Düzey: `DICTIONARY` - -### TRUNCATE {#grant-truncate} - -Çalıştırmaya izin verir [TRUNCATE](../../sql-reference/statements/misc.md#truncate-statement) sorgular. - -Ayrıcalık düzeyi: `TABLE`. - -### OPTIMIZE {#grant-optimize} - -Çalıştırmaya izin verir [OPTIMIZE TABLE](../../sql-reference/statements/misc.md#misc_operations-optimize) sorgular. - -Ayrıcalık düzeyi: `TABLE`. - -### SHOW {#grant-show} - -Çalıştırmaya izin verir `SHOW`, `DESCRIBE`, `USE`, ve `EXISTS` aşağıdaki ayrıcalıklar hiyerarşisine göre sorgular: - -- `SHOW`. Düzey: `GROUP` - - `SHOW DATABASES`. Düzey: `DATABASE`. Yürütmek için izin verir `SHOW DATABASES`, `SHOW CREATE DATABASE`, `USE ` sorgular. - - `SHOW TABLES`. Düzey: `TABLE`. Yürütmek için izin verir `SHOW TABLES`, `EXISTS `, `CHECK
` sorgular. - - `SHOW COLUMNS`. Düzey: `COLUMN`. Yürütmek için izin verir `SHOW CREATE TABLE`, `DESCRIBE` sorgular. - - `SHOW DICTIONARIES`. Düzey: `DICTIONARY`. Yürütmek için izin verir `SHOW DICTIONARIES`, `SHOW CREATE DICTIONARY`, `EXISTS ` sorgular. - -**Not** - -Bir kullanıcı aşağıdaki özelliklere sahiptir `SHOW` belirtilen tablo, sözlük veya veritabanı ile ilgili başka bir ayrıcalık varsa ayrıcalık. - -### KILL QUERY {#grant-kill-query} - -Çalıştırmaya izin verir [KILL](../../sql-reference/statements/misc.md#kill-query-statement) aşağıdaki ayrıcalıklar hiyerarşisine göre sorgular: - -Ayrıcalık düzeyi: `GLOBAL`. - -**Not** - -`KILL QUERY` ayrıcalık, bir kullanıcının diğer kullanıcıların sorgularını öldürmesine izin verir. - -### ACCESS MANAGEMENT {#grant-access-management} - -Bir kullanıcının kullanıcıları, rolleri ve satır ilkelerini yöneten sorguları yürütmesine izin verir. - -- `ACCESS MANAGEMENT`. Düzey: `GROUP` - - `CREATE USER`. Düzey: `GLOBAL` - - `ALTER USER`. Düzey: `GLOBAL` - - `DROP USER`. Düzey: `GLOBAL` - - `CREATE ROLE`. Düzey: `GLOBAL` - - `ALTER ROLE`. Düzey: `GLOBAL` - - `DROP ROLE`. Düzey: `GLOBAL` - - `ROLE ADMIN`. Düzey: `GLOBAL` - - `CREATE ROW POLICY`. Düzey: `GLOBAL`. Takma ad: `CREATE POLICY` - - `ALTER ROW POLICY`. Düzey: `GLOBAL`. Takma ad: `ALTER POLICY` - - `DROP ROW POLICY`. Düzey: `GLOBAL`. Takma ad: `DROP POLICY` - - `CREATE QUOTA`. Düzey: `GLOBAL` - - `ALTER QUOTA`. Düzey: `GLOBAL` - - `DROP QUOTA`. Düzey: `GLOBAL` - - `CREATE SETTINGS PROFILE`. Düzey: `GLOBAL`. Takma ad: `CREATE PROFILE` - - `ALTER SETTINGS PROFILE`. Düzey: `GLOBAL`. Takma ad: `ALTER PROFILE` - - `DROP SETTINGS PROFILE`. Düzey: `GLOBAL`. Takma ad: `DROP PROFILE` - - `SHOW ACCESS`. Düzey: `GROUP` - - `SHOW_USERS`. Düzey: `GLOBAL`. Takma ad: `SHOW CREATE USER` - - `SHOW_ROLES`. Düzey: `GLOBAL`. Takma ad: `SHOW CREATE ROLE` - - `SHOW_ROW_POLICIES`. Düzey: `GLOBAL`. Takma ad: `SHOW POLICIES`, `SHOW CREATE ROW POLICY`, `SHOW CREATE POLICY` - - `SHOW_QUOTAS`. Düzey: `GLOBAL`. Takma ad: `SHOW CREATE QUOTA` - - `SHOW_SETTINGS_PROFILES`. Düzey: `GLOBAL`. Takma ad: `SHOW PROFILES`, `SHOW CREATE SETTINGS PROFILE`, `SHOW CREATE PROFILE` - -Bu `ROLE ADMIN` ayrıcalık, kullanıcının yönetici seçeneğiyle kullanıcıya atanmayanlar da dahil olmak üzere herhangi bir rol atamasına ve iptal etmesine izin verir. - -### SYSTEM {#grant-system} - -Bir kullanıcının yürütmesine izin verir [SYSTEM](../../sql-reference/statements/system.md) aşağıdaki ayrıcalıklar hiyerarşisine göre sorgular. - -- `SYSTEM`. Düzey: `GROUP` - - `SYSTEM SHUTDOWN`. Düzey: `GLOBAL`. Takma ad: `SYSTEM KILL`, `SHUTDOWN` - - `SYSTEM DROP CACHE`. Takma ad: `DROP CACHE` - - `SYSTEM DROP DNS CACHE`. Düzey: `GLOBAL`. Takma ad: `SYSTEM DROP DNS`, `DROP DNS CACHE`, `DROP DNS` - - `SYSTEM DROP MARK CACHE`. Düzey: `GLOBAL`. Takma ad: `SYSTEM DROP MARK`, `DROP MARK CACHE`, `DROP MARKS` - - `SYSTEM DROP UNCOMPRESSED CACHE`. Düzey: `GLOBAL`. Takma ad: `SYSTEM DROP UNCOMPRESSED`, `DROP UNCOMPRESSED CACHE`, `DROP UNCOMPRESSED` - - `SYSTEM RELOAD`. Düzey: `GROUP` - - `SYSTEM RELOAD CONFIG`. Düzey: `GLOBAL`. Takma ad: `RELOAD CONFIG` - - `SYSTEM RELOAD DICTIONARY`. Düzey: `GLOBAL`. Takma ad: `SYSTEM RELOAD DICTIONARIES`, `RELOAD DICTIONARY`, `RELOAD DICTIONARIES` - - `SYSTEM RELOAD EMBEDDED DICTIONARIES`. Düzey: `GLOBAL`. Takma Adlar: R`ELOAD EMBEDDED DICTIONARIES` - - `SYSTEM MERGES`. Düzey: `TABLE`. Takma ad: `SYSTEM STOP MERGES`, `SYSTEM START MERGES`, `STOP MERGES`, `START MERGES` - - `SYSTEM TTL MERGES`. Düzey: `TABLE`. Takma ad: `SYSTEM STOP TTL MERGES`, `SYSTEM START TTL MERGES`, `STOP TTL MERGES`, `START TTL MERGES` - - `SYSTEM FETCHES`. Düzey: `TABLE`. Takma ad: `SYSTEM STOP FETCHES`, `SYSTEM START FETCHES`, `STOP FETCHES`, `START FETCHES` - - `SYSTEM MOVES`. Düzey: `TABLE`. Takma ad: `SYSTEM STOP MOVES`, `SYSTEM START MOVES`, `STOP MOVES`, `START MOVES` - - `SYSTEM SENDS`. Düzey: `GROUP`. Takma ad: `SYSTEM STOP SENDS`, `SYSTEM START SENDS`, `STOP SENDS`, `START SENDS` - - `SYSTEM DISTRIBUTED SENDS`. Düzey: `TABLE`. Takma ad: `SYSTEM STOP DISTRIBUTED SENDS`, `SYSTEM START DISTRIBUTED SENDS`, `STOP DISTRIBUTED SENDS`, `START DISTRIBUTED SENDS` - - `SYSTEM REPLICATED SENDS`. Düzey: `TABLE`. Takma ad: `SYSTEM STOP REPLICATED SENDS`, `SYSTEM START REPLICATED SENDS`, `STOP REPLICATED SENDS`, `START REPLICATED SENDS` - - `SYSTEM REPLICATION QUEUES`. Düzey: `TABLE`. Takma ad: `SYSTEM STOP REPLICATION QUEUES`, `SYSTEM START REPLICATION QUEUES`, `STOP REPLICATION QUEUES`, `START REPLICATION QUEUES` - - `SYSTEM SYNC REPLICA`. Düzey: `TABLE`. Takma ad: `SYNC REPLICA` - - `SYSTEM RESTART REPLICA`. Düzey: `TABLE`. Takma ad: `RESTART REPLICA` - - `SYSTEM FLUSH`. Düzey: `GROUP` - - `SYSTEM FLUSH DISTRIBUTED`. Düzey: `TABLE`. Takma ad: `FLUSH DISTRIBUTED` - - `SYSTEM FLUSH LOGS`. Düzey: `GLOBAL`. Takma ad: `FLUSH LOGS` - -Bu `SYSTEM RELOAD EMBEDDED DICTIONARIES` tarafından örtülü olarak verilen ayrıcalık `SYSTEM RELOAD DICTIONARY ON *.*` ayrıcalık. - -### INTROSPECTION {#grant-introspection} - -Kullanmanıza izin verir [içgözlem](../../operations/optimizing-performance/sampling-query-profiler.md) işlevler. - -- `INTROSPECTION`. Düzey: `GROUP`. Takma ad: `INTROSPECTION FUNCTIONS` - - `addressToLine`. Düzey: `GLOBAL` - - `addressToSymbol`. Düzey: `GLOBAL` - - `demangle`. Düzey: `GLOBAL` - -### SOURCES {#grant-sources} - -Harici veri kaynaklarının kullanılmasına izin verir. İçin geçerlidir [masa motorları](../../engines/table-engines/index.md) ve [tablo işlevleri](../../sql-reference/table-functions/index.md#table-functions). - -- `SOURCES`. Düzey: `GROUP` - - `FILE`. Düzey: `GLOBAL` - - `URL`. Düzey: `GLOBAL` - - `REMOTE`. Düzey: `GLOBAL` - - `YSQL`. Düzey: `GLOBAL` - - `ODBC`. Düzey: `GLOBAL` - - `JDBC`. Düzey: `GLOBAL` - - `HDFS`. Düzey: `GLOBAL` - - `S3`. Düzey: `GLOBAL` - -Bu `SOURCES` ayrıcalık, tüm kaynakların kullanılmasına izin verir. Ayrıca, her kaynak için ayrı ayrı bir ayrıcalık verebilirsiniz. Kaynakları kullanmak için ek ayrıcalıklara ihtiyacınız var. - -Örnekler: - -- İle bir tablo oluşturmak için [MySQL tablo motoru](../../engines/table-engines/integrations/mysql.md) ihtiyacınız `CREATE TABLE (ON db.table_name)` ve `MYSQL` ayrıcalıklar. -- Kullanmak için [mysql tablo işlevi](../../sql-reference/table-functions/mysql.md) ihtiyacınız `CREATE TEMPORARY TABLE` ve `MYSQL` ayrıcalıklar. - -### dictGet {#grant-dictget} - -- `dictGet`. Takma ad: `dictHas`, `dictGetHierarchy`, `dictIsIn` - -Bir kullanıcının yürütmesine izin verir [dictGet](../../sql-reference/functions/ext-dict-functions.md#dictget), [dictHas](../../sql-reference/functions/ext-dict-functions.md#dicthas), [dictGetHierarchy](../../sql-reference/functions/ext-dict-functions.md#dictgethierarchy), [dictİsİn](../../sql-reference/functions/ext-dict-functions.md#dictisin) işlevler. - -Ayrıcalık düzeyi: `DICTIONARY`. - -**Örnekler** - -- `GRANT dictGet ON mydb.mydictionary TO john` -- `GRANT dictGet ON mydictionary TO john` - -### ALL {#grant-all} - -Düzenlenmiş varlık üzerindeki tüm ayrıcalıkları bir kullanıcı hesabına veya bir role verir. - -### NONE {#grant-none} - -Herhangi bir ayrıcalık vermez. - -### ADMIN OPTION {#admin-option-privilege} - -Bu `ADMIN OPTION` ayrıcalık, kullanıcının rolünü başka bir kullanıcıya vermesine izin verir. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/grant/) diff --git a/docs/tr/sql-reference/statements/index.md b/docs/tr/sql-reference/statements/index.md deleted file mode 100644 index c3fb5aa47c6..00000000000 --- a/docs/tr/sql-reference/statements/index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: Deyimler -toc_priority: 31 ---- - - diff --git a/docs/tr/sql-reference/statements/insert-into.md b/docs/tr/sql-reference/statements/insert-into.md deleted file mode 100644 index dd5dec73efe..00000000000 --- a/docs/tr/sql-reference/statements/insert-into.md +++ /dev/null @@ -1,80 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 34 -toc_title: INSERT INTO ---- - -## INSERT {#insert} - -Veri ekleme. - -Temel sorgu biçimi: - -``` sql -INSERT INTO [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), ... -``` - -Sorgu eklemek için sütunların bir listesini belirtebilirsiniz `[(c1, c2, c3)]`. Bu durumda, sütunların geri kalanı ile doldurulur: - -- Hesaplanan değerler `DEFAULT` tablo tanımında belirtilen ifadeler. -- Sıfırlar ve boş dizeler, eğer `DEFAULT` ifadeler tanımlanmamıştır. - -Eğer [strict_ınsert_defaults = 1](../../operations/settings/settings.md), sahip olmayan sütunlar `DEFAULT` tanımlanan sorguda listelenmelidir. - -Veri herhangi bir İNSERT geçirilebilir [biçimli](../../interfaces/formats.md#formats) ClickHouse tarafından desteklenmektedir. Biçim sorguda açıkça belirtilmelidir: - -``` sql -INSERT INTO [db.]table [(c1, c2, c3)] FORMAT format_name data_set -``` - -For example, the following query format is identical to the basic version of INSERT … VALUES: - -``` sql -INSERT INTO [db.]table [(c1, c2, c3)] FORMAT Values (v11, v12, v13), (v21, v22, v23), ... -``` - -ClickHouse, veriden önce tüm boşlukları ve bir satır beslemesini (varsa) kaldırır. Bir sorgu oluştururken, sorgu işleçlerinden sonra verileri yeni bir satıra koymanızı öneririz (veriler boşluklarla başlarsa bu önemlidir). - -Örnek: - -``` sql -INSERT INTO t FORMAT TabSeparated -11 Hello, world! -22 Qwerty -``` - -Komut satırı istemcisini veya HTTP arabirimini kullanarak verileri sorgudan ayrı olarak ekleyebilirsiniz. Daha fazla bilgi için bölüme bakın “[Arabirimler](../../interfaces/index.md#interfaces)”. - -### Kısıtlamalar {#constraints} - -Tablo varsa [kısıtlamalar](create.md#constraints), their expressions will be checked for each row of inserted data. If any of those constraints is not satisfied — server will raise an exception containing constraint name and expression, the query will be stopped. - -### Sonuçları ekleme `SELECT` {#insert_query_insert-select} - -``` sql -INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... -``` - -Sütunlar, SELECT yan tümcesindeki konumlarına göre eşleştirilir. Ancak, SELECT ifadesi ve INSERT için tablo adları farklı olabilir. Gerekirse, tip döküm yapılır. - -Değerler dışındaki veri biçimlerinin hiçbiri, aşağıdaki gibi ifadelere değerler ayarlamasına izin vermez `now()`, `1 + 2` ve bu yüzden. Değerler biçimi, ifadelerin sınırlı kullanımına izin verir, ancak bu önerilmez, çünkü bu durumda verimsiz kod yürütme için kullanılır. - -Veri bölümlerini değiştirmek için diğer sorgular desteklenmiyor: `UPDATE`, `DELETE`, `REPLACE`, `MERGE`, `UPSERT`, `INSERT UPDATE`. -Ancak, eski verileri kullanarak silebilirsiniz `ALTER TABLE ... DROP PARTITION`. - -`FORMAT` yan tümcesi sorgu sonunda belirtilmelidir eğer `SELECT` yan tümcesi tablo işlevi içerir [girdi()](../table-functions/input.md). - -### Performans Konuları {#performance-considerations} - -`INSERT` giriş verilerini birincil anahtarla sıralar ve bunları bir bölüm anahtarı ile bölümlere ayırır. Bir kerede birkaç bölüme veri eklerseniz, bu veri tabanının performansını önemli ölçüde azaltabilir. `INSERT` sorgu. Bunu önlemek için: - -- Bir seferde 100.000 satır gibi oldukça büyük gruplar halinde veri ekleyin. -- Clickhouse'a yüklemeden önce verileri bir bölüm anahtarıyla gruplandırın. - -Eğer performans azalmaz: - -- Veri gerçek zamanlı olarak eklenir. -- Genellikle zamana göre sıralanır veri yükleyin. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/insert_into/) diff --git a/docs/tr/sql-reference/statements/misc.md b/docs/tr/sql-reference/statements/misc.md deleted file mode 100644 index 328bb83a049..00000000000 --- a/docs/tr/sql-reference/statements/misc.md +++ /dev/null @@ -1,358 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 41 -toc_title: "Di\u011Fer" ---- - -# Çeşitli Sorgular {#miscellaneous-queries} - -## ATTACH {#attach} - -Bu sorgu tam olarak aynıdır `CREATE`, ama - -- Kelime yerine `CREATE` kelime kullanır `ATTACH`. -- Sorgu diskte veri oluşturmaz, ancak verilerin zaten uygun yerlerde olduğunu ve yalnızca tablo hakkında bilgi sunucuya eklediğini varsayar. - Bir ekleme sorgusu çalıştırdıktan sonra, sunucu tablonun varlığı hakkında bilgi sahibi olacaktır. - -Tablo daha önce ayrılmış olsaydı (`DETACH`), yapısının bilindiği anlamına gelir, yapıyı tanımlamadan steno kullanabilirsiniz. - -``` sql -ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster] -``` - -Bu sorgu, sunucu başlatılırken kullanılır. Sunucu, tablo meta verilerini dosyalar olarak depolar `ATTACH` başlangıçta çalıştığı sorgular (sunucuda açıkça oluşturulan sistem tabloları hariç). - -## CHECK TABLE {#check-table} - -Tablodaki verilerin bozuk olup olmadığını denetler. - -``` sql -CHECK TABLE [db.]name -``` - -Bu `CHECK TABLE` sorgu, gerçek dosya boyutlarını sunucuda depolanan beklenen değerlerle karşılaştırır. Dosya boyutları depolanan değerlerle eşleşmiyorsa, verilerin bozuk olduğu anlamına gelir. Bu, örneğin, sorgu yürütme sırasında bir sistem çökmesine neden olabilir. - -Sorgu yanıtı içerir `result` tek satırlı sütun. Satır bir değere sahiptir -[Boeanoleanean](../../sql-reference/data-types/boolean.md) tür: - -- 0-tablodaki veriler bozuk. -- 1 - veri bütünlüğünü korur. - -Bu `CHECK TABLE` sorgu Aşağıdaki tablo motorlarını destekler: - -- [Günlük](../../engines/table-engines/log-family/log.md) -- [TinyLog](../../engines/table-engines/log-family/tinylog.md) -- [StripeLog](../../engines/table-engines/log-family/stripelog.md) -- [MergeTree ailesi](../../engines/table-engines/mergetree-family/mergetree.md) - -Başka bir tablo motorları ile tablolar üzerinde gerçekleştirilen bir özel duruma neden olur. - -Motor fromlardan `*Log` aile başarısızlık otomatik veri kurtarma sağlamaz. Kullan... `CHECK TABLE` veri kaybını zamanında izlemek için sorgu. - -İçin `MergeTree` aile motorları, `CHECK TABLE` sorgu, yerel sunucudaki bir tablonun her bir veri bölümü için bir kontrol durumunu gösterir. - -**Veri bozuksa** - -Tablo bozuksa, bozuk olmayan verileri başka bir tabloya kopyalayabilirsiniz. Bunu yapmak için : - -1. Bozuk tablo ile aynı yapıya sahip yeni bir tablo oluşturun. Bunu yapmak için sorguyu yürütün `CREATE TABLE AS `. -2. Ayarla... [max_threads](../../operations/settings/settings.md#settings-max_threads) bir sonraki sorguyu tek bir iş parçacığında işlemek için 1 değeri. Bunu yapmak için sorguyu çalıştırın `SET max_threads = 1`. -3. Sorgu yürütme `INSERT INTO SELECT * FROM `. Bu istek bozuk olmayan verileri bozuk tablodan başka bir tabloya kopyalar. Yalnızca bozuk parçadan önceki veriler kopyalanır. -4. Yeniden Başlat `clickhouse-client` sıfırlamak için `max_threads` değer. - -## DESCRIBE TABLE {#misc-describe-table} - -``` sql -DESC|DESCRIBE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] -``` - -Aşağıdaki döndürür `String` sütun tipi: - -- `name` — Column name. -- `type`— Column type. -- `default_type` — Clause that is used in [varsayılan ifade](create.md#create-default-values) (`DEFAULT`, `MATERIALIZED` veya `ALIAS`). Varsayılan ifade belirtilmemişse, sütun boş bir dize içerir. -- `default_expression` — Value specified in the `DEFAULT` yan. -- `comment_expression` — Comment text. - -İç içe veri yapıları çıktı “expanded” biçimli. Her sütun ayrı ayrı gösterilir, bir noktadan sonra adı ile. - -## DETACH {#detach} - -Hakkında bilgi siler ‘name’ sunucudan tablo. Sunucu, tablonun varlığını bilmeyi durdurur. - -``` sql -DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] -``` - -Bu tablonun veri veya meta verileri silmez. Bir sonraki sunucu lansmanında, sunucu meta verileri okuyacak ve tablo hakkında tekrar bilgi edinecektir. -Benzer şekilde, bir “detached” tablo kullanılarak yeniden eklenebilir `ATTACH` sorgu (bunlar için depolanan meta verilere sahip olmayan sistem tabloları hariç). - -Hiç yok... `DETACH DATABASE` sorgu. - -## DROP {#drop} - -Bu sorgu iki türü vardır: `DROP DATABASE` ve `DROP TABLE`. - -``` sql -DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] -``` - -İçindeki tüm tabloları siler ‘db’ veritabanı, daha sonra siler ‘db’ veritabanı kendisi. -Eğer `IF EXISTS` belirtilen, veritabanı yoksa bir hata döndürmez. - -``` sql -DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] -``` - -Tabloyu siler. -Eğer `IF EXISTS` belirtilmişse, tablo yoksa veya veritabanı yoksa bir hata döndürmez. - - DROP DICTIONARY [IF EXISTS] [db.]name - -Sözlük Delets. -Eğer `IF EXISTS` belirtilmişse, tablo yoksa veya veritabanı yoksa bir hata döndürmez. - -## DROP USER {#drop-user-statement} - -Bir kullanıcıyı siler. - -### Sözdizimi {#drop-user-syntax} - -``` sql -DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] -``` - -## DROP ROLE {#drop-role-statement} - -Bir rolü siler. - -Silinen rol, verildiği tüm varlıklardan iptal edilir. - -### Sözdizimi {#drop-role-syntax} - -``` sql -DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] -``` - -## DROP ROW POLICY {#drop-row-policy-statement} - -Bir satır ilkesi siler. - -Silinen satır ilkesi, atandığı tüm varlıklardan iptal edilir. - -### Sözdizimi {#drop-row-policy-syntax} - -``` sql -DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] -``` - -## DROP QUOTA {#drop-quota-statement} - -Bir kota siler. - -Silinen kota, atandığı tüm varlıklardan iptal edilir. - -### Sözdizimi {#drop-quota-syntax} - -``` sql -DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name] -``` - -## DROP SETTINGS PROFILE {#drop-settings-profile-statement} - -Bir kota siler. - -Silinen kota, atandığı tüm varlıklardan iptal edilir. - -### Sözdizimi {#drop-settings-profile-syntax} - -``` sql -DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] -``` - -## EXISTS {#exists-statement} - -``` sql -EXISTS [TEMPORARY] [TABLE|DICTIONARY] [db.]name [INTO OUTFILE filename] [FORMAT format] -``` - -Bir tek döndürür `UInt8`- tek değeri içeren sütun yazın `0` tablo veya veritabanı yoksa veya `1` tablo belirtilen veritabanında varsa. - -## KILL QUERY {#kill-query-statement} - -``` sql -KILL QUERY [ON CLUSTER cluster] - WHERE - [SYNC|ASYNC|TEST] - [FORMAT format] -``` - -Şu anda çalışan sorguları zorla sonlandırmaya çalışır. -Sonlandırılacak sorgular sistemden seçilir.tanımlanan kriterleri kullanarak işlemler tablosu `WHERE` fıkra ofsı `KILL` sorgu. - -Örnekler: - -``` sql --- Forcibly terminates all queries with the specified query_id: -KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90' - --- Synchronously terminates all queries run by 'username': -KILL QUERY WHERE user='username' SYNC -``` - -Salt okunur kullanıcılar yalnızca kendi sorgularını durdurabilir. - -Varsayılan olarak, sorguların zaman uyumsuz sürümü kullanılır (`ASYNC`), sorguların durduğuna dair onay beklemez. - -Senkron versiyonu (`SYNC`) tüm sorguların durmasını bekler ve durduğunda her işlem hakkında bilgi görüntüler. -Yanıt içerir `kill_status` aşağıdaki değerleri alabilen sütun: - -1. ‘finished’ – The query was terminated successfully. -2. ‘waiting’ – Waiting for the query to end after sending it a signal to terminate. -3. The other values ​​explain why the query can't be stopped. - -Bir test sorgusu (`TEST`) yalnızca kullanıcının haklarını denetler ve durdurulacak sorguların bir listesini görüntüler. - -## KILL MUTATION {#kill-mutation} - -``` sql -KILL MUTATION [ON CLUSTER cluster] - WHERE - [TEST] - [FORMAT format] -``` - -İptal etmek ve kaldırmak için çalışır [mutasyonlar](alter.md#alter-mutations) şu anda yürütülüyor. İptal etmek için mutationsasyonlar seçilir [`system.mutations`](../../operations/system-tables.md#system_tables-mutations) tablo tarafından belirtilen filtreyi kullanarak `WHERE` fıkra ofsı `KILL` sorgu. - -Bir test sorgusu (`TEST`) yalnızca kullanıcının haklarını denetler ve durdurulacak sorguların bir listesini görüntüler. - -Örnekler: - -``` sql --- Cancel and remove all mutations of the single table: -KILL MUTATION WHERE database = 'default' AND table = 'table' - --- Cancel the specific mutation: -KILL MUTATION WHERE database = 'default' AND table = 'table' AND mutation_id = 'mutation_3.txt' -``` - -The query is useful when a mutation is stuck and cannot finish (e.g. if some function in the mutation query throws an exception when applied to the data contained in the table). - -Mutasyon tarafından yapılan değişiklikler geri alınmaz. - -## OPTIMIZE {#misc_operations-optimize} - -``` sql -OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] [DEDUPLICATE] -``` - -Bu sorgu, bir tablo altyapısı ile tablolar için veri parçaları planlanmamış birleştirme başlatmaya çalışır. [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) aile. - -Bu `OPTMIZE` sorgu için de desteklenmektedir [MaterializedView](../../engines/table-engines/special/materializedview.md) ve... [Arabellek](../../engines/table-engines/special/buffer.md) motorlar. Diğer tablo motorları desteklenmiyor. - -Ne zaman `OPTIMIZE` ile kullanılır [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) Tablo motorları ailesi, ClickHouse birleştirme için bir görev oluşturur ve tüm düğümlerde yürütülmeyi bekler (eğer `replication_alter_partitions_sync` ayar etkinse) ' dir. - -- Eğer `OPTIMIZE` herhangi bir nedenle bir birleştirme gerçekleştirmez, müşteriye bildirmez. Bildirimleri etkinleştirmek için [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) ayar. -- Belirtir aseniz bir `PARTITION`, sadece belirtilen bölüm optimize edilmiştir. [Bölüm ifadesi nasıl ayarlanır](alter.md#alter-how-to-specify-part-expr). -- Belirtir specifyseniz `FINAL`, optimizasyon, tüm veriler zaten bir parçada olsa bile gerçekleştirilir. -- Belirtir specifyseniz `DEDUPLICATE`, sonra tamamen aynı satırlar tekilleştirilecektir (tüm sütunlar karşılaştırılır), sadece MergeTree motoru için anlamlıdır. - -!!! warning "Uyarıcı" - `OPTIMIZE` Düzelt canemiyorum “Too many parts” hatasız. - -## RENAME {#misc_operations-rename} - -Bir veya daha fazla tabloyu yeniden adlandırır. - -``` sql -RENAME TABLE [db11.]name11 TO [db12.]name12, [db21.]name21 TO [db22.]name22, ... [ON CLUSTER cluster] -``` - -Tüm tablolar genel kilitleme altında yeniden adlandırılır. Tabloları yeniden adlandırma hafif bir işlemdir. İÇİN'DEN sonra başka bir veritabanı belirttiyseniz, tablo bu veritabanına taşınacaktır. Ancak, veritabanlarına sahip dizinlerin aynı dosya sisteminde bulunması gerekir (aksi takdirde bir hata döndürülür). - -## SET {#query-set} - -``` sql -SET param = value -``` - -Atıyor `value` to the `param` [ayar](../../operations/settings/index.md) geçerli oturum için. Değiştiremezsiniz [sunucu ayarları](../../operations/server-configuration-parameters/index.md) bu şekilde. - -Belirtilen ayarlar profilindeki tüm değerleri tek bir sorguda da ayarlayabilirsiniz. - -``` sql -SET profile = 'profile-name-from-the-settings-file' -``` - -Daha fazla bilgi için, bkz. [Ayarlar](../../operations/settings/settings.md). - -## SET ROLE {#set-role-statement} - -Geçerli kullanıcı için rolleri etkinleştirir. - -### Sözdizimi {#set-role-syntax} - -``` sql -SET ROLE {DEFAULT | NONE | role [,...] | ALL | ALL EXCEPT role [,...]} -``` - -## SET DEFAULT ROLE {#set-default-role-statement} - -Varsayılan rolleri bir kullanıcıya ayarlar. - -Varsayılan roller kullanıcı girişinde otomatik olarak etkinleştirilir. Varsayılan olarak yalnızca önceden verilen rolleri ayarlayabilirsiniz. Rol bir kullanıcıya verilmezse, ClickHouse bir istisna atar. - -### Sözdizimi {#set-default-role-syntax} - -``` sql -SET DEFAULT ROLE {NONE | role [,...] | ALL | ALL EXCEPT role [,...]} TO {user|CURRENT_USER} [,...] -``` - -### Örnekler {#set-default-role-examples} - -Bir kullanıcıya birden çok varsayılan rol ayarlama: - -``` sql -SET DEFAULT ROLE role1, role2, ... TO user -``` - -Verilen tüm rolleri bir kullanıcıya varsayılan olarak ayarlayın: - -``` sql -SET DEFAULT ROLE ALL TO user -``` - -Bir kullanıcıdan varsayılan rolleri temizleme: - -``` sql -SET DEFAULT ROLE NONE TO user -``` - -Verilen tüm rolleri bazıları hariç varsayılan olarak ayarlayın: - -``` sql -SET DEFAULT ROLE ALL EXCEPT role1, role2 TO user -``` - -## TRUNCATE {#truncate-statement} - -``` sql -TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] -``` - -Bir tablodaki tüm verileri kaldırır. Fık thera ne zaman `IF EXISTS` tablo yoksa, sorgu bir hata döndürür. - -Bu `TRUNCATE` sorgu için desteklenmiyor [Görünüm](../../engines/table-engines/special/view.md), [Dosya](../../engines/table-engines/special/file.md), [URL](../../engines/table-engines/special/url.md) ve [Boş](../../engines/table-engines/special/null.md) masa motorları. - -## USE {#use} - -``` sql -USE db -``` - -Oturum için geçerli veritabanını ayarlamanızı sağlar. -Geçerli veritabanı, veritabanı sorguda tablo adından önce bir nokta ile açıkça tanımlanmamışsa, tabloları aramak için kullanılır. -Bir oturum kavramı olmadığından, bu sorgu HTTP protokolünü kullanırken yapılamaz. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/misc/) diff --git a/docs/tr/sql-reference/statements/revoke.md b/docs/tr/sql-reference/statements/revoke.md deleted file mode 120000 index 4321fdb14a7..00000000000 --- a/docs/tr/sql-reference/statements/revoke.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/sql-reference/statements/revoke.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/array-join.md b/docs/tr/sql-reference/statements/select/array-join.md deleted file mode 120000 index c341801e419..00000000000 --- a/docs/tr/sql-reference/statements/select/array-join.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/array-join.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/distinct.md b/docs/tr/sql-reference/statements/select/distinct.md deleted file mode 120000 index 59319557dc1..00000000000 --- a/docs/tr/sql-reference/statements/select/distinct.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/distinct.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/format.md b/docs/tr/sql-reference/statements/select/format.md deleted file mode 120000 index 106b2d9ebbc..00000000000 --- a/docs/tr/sql-reference/statements/select/format.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/format.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/from.md b/docs/tr/sql-reference/statements/select/from.md deleted file mode 120000 index f8ebfe655cc..00000000000 --- a/docs/tr/sql-reference/statements/select/from.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/from.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/group-by.md b/docs/tr/sql-reference/statements/select/group-by.md deleted file mode 120000 index cf519ad7781..00000000000 --- a/docs/tr/sql-reference/statements/select/group-by.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/group-by.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/having.md b/docs/tr/sql-reference/statements/select/having.md deleted file mode 120000 index 4a038beb126..00000000000 --- a/docs/tr/sql-reference/statements/select/having.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/having.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/index.md b/docs/tr/sql-reference/statements/select/index.md deleted file mode 120000 index 9c649322c82..00000000000 --- a/docs/tr/sql-reference/statements/select/index.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/index.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/into-outfile.md b/docs/tr/sql-reference/statements/select/into-outfile.md deleted file mode 120000 index 2c9c812b3d5..00000000000 --- a/docs/tr/sql-reference/statements/select/into-outfile.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/into-outfile.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/join.md b/docs/tr/sql-reference/statements/select/join.md deleted file mode 120000 index 5951a105137..00000000000 --- a/docs/tr/sql-reference/statements/select/join.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/join.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/limit-by.md b/docs/tr/sql-reference/statements/select/limit-by.md deleted file mode 120000 index f3a63e9fe22..00000000000 --- a/docs/tr/sql-reference/statements/select/limit-by.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/limit-by.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/limit.md b/docs/tr/sql-reference/statements/select/limit.md deleted file mode 120000 index e0a0c632dac..00000000000 --- a/docs/tr/sql-reference/statements/select/limit.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/limit.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/order-by.md b/docs/tr/sql-reference/statements/select/order-by.md deleted file mode 120000 index cc2567bce0b..00000000000 --- a/docs/tr/sql-reference/statements/select/order-by.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/order-by.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/prewhere.md b/docs/tr/sql-reference/statements/select/prewhere.md deleted file mode 120000 index 567fc95356f..00000000000 --- a/docs/tr/sql-reference/statements/select/prewhere.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/prewhere.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/sample.md b/docs/tr/sql-reference/statements/select/sample.md deleted file mode 120000 index 9df6e25d0f3..00000000000 --- a/docs/tr/sql-reference/statements/select/sample.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/sample.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/union.md b/docs/tr/sql-reference/statements/select/union.md deleted file mode 100644 index 0eb8db0be7a..00000000000 --- a/docs/tr/sql-reference/statements/select/union.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/union.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/where.md b/docs/tr/sql-reference/statements/select/where.md deleted file mode 120000 index 8ba28926879..00000000000 --- a/docs/tr/sql-reference/statements/select/where.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/where.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/select/with.md b/docs/tr/sql-reference/statements/select/with.md deleted file mode 120000 index 8b7ea4db44c..00000000000 --- a/docs/tr/sql-reference/statements/select/with.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/select/with.md \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/show.md b/docs/tr/sql-reference/statements/show.md deleted file mode 100644 index 69e8b492971..00000000000 --- a/docs/tr/sql-reference/statements/show.md +++ /dev/null @@ -1,169 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 38 -toc_title: SHOW ---- - -# Sorguları göster {#show-queries} - -## SHOW CREATE TABLE {#show-create-table} - -``` sql -SHOW CREATE [TEMPORARY] [TABLE|DICTIONARY] [db.]table [INTO OUTFILE filename] [FORMAT format] -``` - -Bir tek döndürür `String`-tür ‘statement’ column, which contains a single value – the `CREATE` belirtilen nesneyi oluşturmak için kullanılan sorgu. - -## SHOW DATABASES {#show-databases} - -``` sql -SHOW DATABASES [INTO OUTFILE filename] [FORMAT format] -``` - -Tüm veritabanlarının bir listesini yazdırır. -Bu sorgu ile aynıdır `SELECT name FROM system.databases [INTO OUTFILE filename] [FORMAT format]`. - -## SHOW PROCESSLIST {#show-processlist} - -``` sql -SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format] -``` - -İçeriği verir [sistem.işleyişler](../../operations/system-tables.md#system_tables-processes) şu anda işlenmekte olan sorguların bir listesini içeren tablo, hariç `SHOW PROCESSLIST` sorgular. - -Bu `SELECT * FROM system.processes` sorgu, geçerli tüm sorgular hakkında veri döndürür. - -İpucu (konsolda Yürüt): - -``` bash -$ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'" -``` - -## SHOW TABLES {#show-tables} - -Tablo listesini görüntüler. - -``` sql -SHOW [TEMPORARY] TABLES [{FROM | IN} ] [LIKE '' | WHERE expr] [LIMIT ] [INTO OUTFILE ] [FORMAT ] -``` - -Eğer... `FROM` yan tümcesi belirtilmemiş, sorgu geçerli veritabanından tabloların listesini döndürür. - -Aynı sonuçları elde edebilirsiniz `SHOW TABLES` aşağıdaki şekilde sorgu: - -``` sql -SELECT name FROM system.tables WHERE database = [AND name LIKE ] [LIMIT ] [INTO OUTFILE ] [FORMAT ] -``` - -**Örnek** - -Aşağıdaki sorgu, tablo listesinden ilk iki satırı seçer. `system` adları içeren veritabanı `co`. - -``` sql -SHOW TABLES FROM system LIKE '%co%' LIMIT 2 -``` - -``` text -┌─name───────────────────────────┐ -│ aggregate_function_combinators │ -│ collations │ -└────────────────────────────────┘ -``` - -## SHOW DICTIONARIES {#show-dictionaries} - -Bir listesini görüntüler [dış söz dictionarieslükler](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md). - -``` sql -SHOW DICTIONARIES [FROM ] [LIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] -``` - -Eğer... `FROM` yan tümcesi belirtilmemiş, sorgu geçerli veritabanından sözlükler listesini döndürür. - -Aynı sonuçları elde edebilirsiniz `SHOW DICTIONARIES` aşağıdaki şekilde sorgu: - -``` sql -SELECT name FROM system.dictionaries WHERE database = [AND name LIKE ] [LIMIT ] [INTO OUTFILE ] [FORMAT ] -``` - -**Örnek** - -Aşağıdaki sorgu, tablo listesinden ilk iki satırı seçer. `system` adları içeren veritabanı `reg`. - -``` sql -SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2 -``` - -``` text -┌─name─────────┐ -│ regions │ -│ region_names │ -└──────────────┘ -``` - -## SHOW GRANTS {#show-grants-statement} - -Bir kullanıcı için ayrıcalıkları gösterir. - -### Sözdizimi {#show-grants-syntax} - -``` sql -SHOW GRANTS [FOR user] -``` - -Kullanıcı belirtilmezse, sorgu geçerli kullanıcı için ayrıcalıklar döndürür. - -## SHOW CREATE USER {#show-create-user-statement} - -Kullanılan parametreleri gösterir. [kullanıcı oluşturma](create.md#create-user-statement). - -`SHOW CREATE USER` kullanıcı şifreleri çıkmıyor. - -### Sözdizimi {#show-create-user-syntax} - -``` sql -SHOW CREATE USER [name | CURRENT_USER] -``` - -## SHOW CREATE ROLE {#show-create-role-statement} - -Kullanılan parametreleri gösterir. [rol oluşturma](create.md#create-role-statement) - -### Sözdizimi {#show-create-role-syntax} - -``` sql -SHOW CREATE ROLE name -``` - -## SHOW CREATE ROW POLICY {#show-create-row-policy-statement} - -Kullanılan parametreleri gösterir. [satır ilkesi oluşturma](create.md#create-row-policy-statement) - -### Sözdizimi {#show-create-row-policy-syntax} - -``` sql -SHOW CREATE [ROW] POLICY name ON [database.]table -``` - -## SHOW CREATE QUOTA {#show-create-quota-statement} - -Kullanılan parametreleri gösterir. [Ko creationta oluşturma](create.md#create-quota-statement) - -### Sözdizimi {#show-create-row-policy-syntax} - -``` sql -SHOW CREATE QUOTA [name | CURRENT] -``` - -## SHOW CREATE SETTINGS PROFILE {#show-create-settings-profile-statement} - -Kullanılan parametreleri gösterir. [ayarlar profil oluşturma](create.md#create-settings-profile-statement) - -### Sözdizimi {#show-create-row-policy-syntax} - -``` sql -SHOW CREATE [SETTINGS] PROFILE name -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/show/) diff --git a/docs/tr/sql-reference/statements/system.md b/docs/tr/sql-reference/statements/system.md deleted file mode 100644 index 6a81c78ac40..00000000000 --- a/docs/tr/sql-reference/statements/system.md +++ /dev/null @@ -1,113 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 37 -toc_title: SYSTEM ---- - -# Sistem sorguları {#query-language-system} - -- [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries) -- [RELOAD DICTIONARY](#query_language-system-reload-dictionary) -- [DROP DNS CACHE](#query_language-system-drop-dns-cache) -- [DROP MARK CACHE](#query_language-system-drop-mark-cache) -- [FLUSH LOGS](#query_language-system-flush_logs) -- [RELOAD CONFIG](#query_language-system-reload-config) -- [SHUTDOWN](#query_language-system-shutdown) -- [KILL](#query_language-system-kill) -- [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends) -- [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) -- [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) -- [STOP MERGES](#query_language-system-stop-merges) -- [START MERGES](#query_language-system-start-merges) - -## RELOAD DICTIONARIES {#query_language-system-reload-dictionaries} - -Daha önce başarıyla yüklenen tüm sözlükleri yeniden yükler. -Varsayılan olarak, sözlükler tembel yüklenir (bkz [dictionaries_lazy_load](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_lazy_load)), bu nedenle başlangıçta otomatik olarak yüklenmek yerine, dictGet işlevi aracılığıyla ilk erişimde başlatılır veya ENGİNE = Dictionary ile tablolardan seçim yapılır. Bu `SYSTEM RELOAD DICTIONARIES` sorgu bu sözlükleri yeniden yükler (yüklü). -Her zaman döner `Ok.` sözlük güncellemesinin sonucu ne olursa olsun. - -## Sözlük Dictionary_name yeniden yükle {#query_language-system-reload-dictionary} - -Tamamen bir sözlük reloads `dictionary_name`, sözlük durumuna bakılmaksızın (LOADED / NOT_LOADED / FAİLED). -Her zaman döner `Ok.` ne olursa olsun sözlük güncelleme sonucu. -Sözlüğün durumu sorgulanarak kontrol edilebilir `system.dictionaries` Tablo. - -``` sql -SELECT name, status FROM system.dictionaries; -``` - -## DROP DNS CACHE {#query_language-system-drop-dns-cache} - -Clickhouse'un iç DNS önbelleğini sıfırlar. Bazen (eski ClickHouse sürümleri için) altyapıyı değiştirirken (başka bir ClickHouse sunucusunun IP adresini veya sözlükler tarafından kullanılan sunucuyu değiştirirken) bu komutu kullanmak gerekir. - -Daha uygun (otomatik) önbellek yönetimi için bkz: disable_internal_dns_cache, dns_cache_update_period parametreleri. - -## DROP MARK CACHE {#query_language-system-drop-mark-cache} - -İşaret önbelleğini sıfırlar. ClickHouse ve performans testlerinin geliştirilmesinde kullanılır. - -## FLUSH LOGS {#query_language-system-flush_logs} - -Flushes buffers of log messages to system tables (e.g. system.query_log). Allows you to not wait 7.5 seconds when debugging. - -## RELOAD CONFIG {#query_language-system-reload-config} - -ClickHouse yapılandırmasını yeniden yükler. Yapılandırma ZooKeeeper saklandığında kullanılır. - -## SHUTDOWN {#query_language-system-shutdown} - -Normalde Clickhouse'u kapatır (gibi `service clickhouse-server stop` / `kill {$pid_clickhouse-server}`) - -## KILL {#query_language-system-kill} - -ClickHouse işlemini iptal eder (gibi `kill -9 {$ pid_clickhouse-server}`) - -## Dağıtılmış Tabloları Yönetme {#query-language-system-distributed} - -ClickHouse yönetebilir [dağılı](../../engines/table-engines/special/distributed.md) Tablolar. Bir kullanıcı bu tablolara veri eklediğinde, ClickHouse önce küme düğümlerine gönderilmesi gereken verilerin bir sırası oluşturur, sonra zaman uyumsuz olarak gönderir. İle kuyruk işleme yönetebilirsiniz [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends), [FLUSH DISTRIBUTED](#query_language-system-flush-distributed), ve [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) sorgular. Ayrıca, dağıtılmış verileri eşzamanlı olarak `insert_distributed_sync` ayar. - -### STOP DISTRIBUTED SENDS {#query_language-system-stop-distributed-sends} - -Dağıtılmış tablolara veri eklerken arka plan veri dağıtımını devre dışı bırakır. - -``` sql -SYSTEM STOP DISTRIBUTED SENDS [db.] -``` - -### FLUSH DISTRIBUTED {#query_language-system-flush-distributed} - -Küme düğümlerine eşzamanlı olarak veri göndermek için Clickhouse'u zorlar. Herhangi bir düğüm kullanılamıyorsa, ClickHouse bir özel durum atar ve sorgu yürütülmesini durdurur. Tüm düğümler tekrar çevrimiçi olduğunda gerçekleşecek olan başarılı olana kadar sorguyu yeniden deneyebilirsiniz. - -``` sql -SYSTEM FLUSH DISTRIBUTED [db.] -``` - -### START DISTRIBUTED SENDS {#query_language-system-start-distributed-sends} - -Dağıtılmış tablolara veri eklerken arka plan veri dağıtımını etkinleştirir. - -``` sql -SYSTEM START DISTRIBUTED SENDS [db.] -``` - -### STOP MERGES {#query_language-system-stop-merges} - -MergeTree ailesindeki tablolar için arka plan birleşmelerini durdurma imkanı sağlar: - -``` sql -SYSTEM STOP MERGES [[db.]merge_tree_family_table_name] -``` - -!!! note "Not" - `DETACH / ATTACH` tablo, daha önce tüm MergeTree tabloları için birleştirmeler durdurulduğunda bile tablo için arka plan birleştirmelerini başlatır. - -### START MERGES {#query_language-system-start-merges} - -MergeTree ailesindeki tablolar için arka plan birleştirmelerini başlatma imkanı sağlar: - -``` sql -SYSTEM START MERGES [[db.]merge_tree_family_table_name] -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/system/) diff --git a/docs/tr/sql-reference/syntax.md b/docs/tr/sql-reference/syntax.md deleted file mode 100644 index c9cc6bd078f..00000000000 --- a/docs/tr/sql-reference/syntax.md +++ /dev/null @@ -1,187 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 31 -toc_title: "S\xF6zdizimi" ---- - -# Sözdizimi {#syntax} - -Sistemde iki tür ayrıştırıcı vardır: tam SQL ayrıştırıcısı (özyinelemeli bir iniş ayrıştırıcısı) ve veri biçimi ayrıştırıcısı (hızlı akış ayrıştırıcısı). -Dışında her durumda `INSERT` sorgu, sadece tam SQL ayrıştırıcı kullanılır. -Bu `INSERT` sorgu her iki ayrıştırıcıyı da kullanır: - -``` sql -INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') -``` - -Bu `INSERT INTO t VALUES` parça tam ayrıştırıcı tarafından ayrıştırılır ve veriler `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` hızlı akış ayrıştırıcısı tarafından ayrıştırılır. Ayrıca kullanarak veriler için tam ayrıştırıcı açabilirsiniz [ınput_format_values_interpret_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) ayar. Ne zaman `input_format_values_interpret_expressions = 1`, ClickHouse önce hızlı akış ayrıştırıcısı ile değerleri ayrıştırmaya çalışır. Başarısız olursa, ClickHouse veriler için tam ayrıştırıcıyı kullanmaya çalışır ve bir SQL gibi davranır [ifade](#syntax-expressions). - -Veri herhangi bir biçime sahip olabilir. Bir sorgu alındığında, sunucu daha fazla hesaplar [max_query_size](../operations/settings/settings.md#settings-max_query_size) istek bayt RAM (varsayılan olarak, 1 MB) ve geri kalanı akış ayrıştırılır. -Bu büyük sorunları önlemek için izin verir `INSERT` sorgular. - -Kullanırken `Values` biçim içinde bir `INSERT` sorgu, verilerin bir ifadedeki ifadelerle aynı şekilde ayrıştırıldığı görünebilir `SELECT` sorgu, ancak bu doğru değil. Bu `Values` biçim çok daha sınırlıdır. - -Bu makalenin geri kalanı tam çözümleyici kapsar. Biçim ayrıştırıcıları hakkında daha fazla bilgi için bkz: [Biçimliler](../interfaces/formats.md) bölme. - -## Alanlar {#spaces} - -Sözdizimsel yapılar arasında (bir sorgunun başlangıcı ve sonu dahil) herhangi bir sayıda boşluk simgesi olabilir. Boşluk sembolleri boşluk, sekme, satır beslemesi, CR ve form beslemesini içerir. - -## Yorumlar {#comments} - -ClickHouse, SQL stili ve C stili yorumlarını destekler. -SQL tarzı yorumlar ile başlar `--` ve hattın sonuna kadar devam, bir boşluk sonra `--` atlanmış olabilir. -C-style dan `/*` -e doğru `*/`ve çok satırlı olabilir, boşluklar da gerekli değildir. - -## Kelimeler {#syntax-keywords} - -Anahtar kelimeler karşılık geldiğinde büyük / küçük harf duyarsızdır: - -- SQL standardı. Mesela, `SELECT`, `select` ve `SeLeCt` hepsi geçerlidir. -- Bazı popüler DBMS'DE (MySQL veya Postgres) uygulama. Mesela, `DateTime` ile aynıdır `datetime`. - -Veri türü adı büyük / küçük harf duyarlı olup olmadığını denetlenebilir `system.data_type_families` Tablo. - -Standart SQL'İN aksine, diğer tüm anahtar kelimeler (işlev adları dahil) şunlardır **büyük küçük harf duyarlı**. - -Anahtar kelimeler ayrılmış değildir; sadece karşılık gelen bağlamda bu şekilde ele alınır. Kullanıyorsanız [tanıtıcılar](#syntax-identifiers) anahtar kelimelerle aynı ada sahip olarak, bunları çift tırnak veya backticks içine alın. Örneğin, sorgu `SELECT "FROM" FROM table_name` tablo geçerli ise `table_name` adı ile sütun vardır `"FROM"`. - -## Tanıtıcılar {#syntax-identifiers} - -Tanımlay areıcılar: - -- Küme, veritabanı, tablo, bölüm ve sütun adları. -- İşlevler. -- Veri türleri. -- [İfade takma adları](#syntax-expression_aliases). - -Tanımlayıcılar alıntılanabilir veya alıntılanamaz. İkincisi tercih edilir. - -Alıntılanmamış tanımlayıcılar regex ile eşleşmelidir `^[a-zA-Z_][0-9a-zA-Z_]*$` ve eşit olamaz [kelimeler](#syntax-keywords). Örnekler: `x, _1, X_y__Z123_.` - -Tanımlayıcıları anahtar kelimelerle aynı şekilde kullanmak istiyorsanız veya tanımlayıcılarda başka semboller kullanmak istiyorsanız, örneğin çift tırnak işaretleri veya backticks kullanarak alıntı yapın, `"id"`, `` `id` ``. - -## Harfler {#literals} - -Sayısal, dize, bileşik ve `NULL` harfler. - -### Sayısal {#numeric} - -Sayısal literal ayrıştırılmaya çalışılıyor: - -- İlk olarak, 64-bit imzalı bir sayı olarak, [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) İşlev. -- Başarısız olursa, 64-bit imzasız bir sayı olarak, [strtoll](https://en.cppreference.com/w/cpp/string/byte/strtol) İşlev. -- Başarısız olursa, kayan noktalı sayı olarak [strtod](https://en.cppreference.com/w/cpp/string/byte/strtof) İşlev. -- Aksi takdirde, bir hata döndürür. - -Hazır bilgi değeri, değerin sığdığı en küçük türe sahiptir. -Örneğin, 1 olarak ayrıştırılır `UInt8`, ancak 256 olarak ayrıştırılır `UInt16`. Daha fazla bilgi için, bkz. [Veri türleri](../sql-reference/data-types/index.md). - -Örnekler: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`. - -### Dize {#syntax-string-literal} - -Tek tırnak yalnızca dize değişmezleri desteklenir. Kapalı karakterler ters eğik çizgi kaçabilir. Aşağıdaki kaçış dizileri karşılık gelen özel bir değere sahiptir: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. Diğer tüm durumlarda, çıkış dizileri biçiminde `\c`, nere `c` herhangi bir karakter, dönüştürülür `c`. Bu dizileri kullanabileceğiniz anlamına gelir `\'`ve`\\`. Değeri olacak [Dize](../sql-reference/data-types/string.md) tür. - -Dize değişmezlerinde, en azından kaçmanız gerekir `'` ve `\`. Tek tırnak tek Alıntı ile kaçabilir, değişmez `'It\'s'` ve `'It''s'` eşittir. - -### Bileşik {#compound} - -Diziler köşeli parantez ile inşa edilmiştir `[1, 2, 3]`. Nuples yuvarlak parantez ile inşa edilmiştir `(1, 'Hello, world!', 2)`. -Teknik olarak bunlar değişmezler değil, sırasıyla dizi oluşturma işleci ve tuple oluşturma işleci ile ifadeler. -Bir dizi en az bir öğeden oluşmalı ve bir tuple en az iki öğeye sahip olmalıdır. -İçinde tuples göründüğünde ayrı bir durum var `IN` CLA ause of a `SELECT` sorgu. Sorgu sonuçları tuples içerebilir, ancak tuples bir veritabanına kaydedilemez (tablolar hariç [Hafıza](../engines/table-engines/special/memory.md) motor). - -### NULL {#null-literal} - -Değerin eksik olduğunu gösterir. - -Saklamak için `NULL` bir tablo alanında, bu olmalıdır [Nullable](../sql-reference/data-types/nullable.md) tür. - -Veri formatına bağlı olarak (giriş veya çıkış), `NULL` farklı bir temsili olabilir. Daha fazla bilgi için belgelere bakın [veri formatları](../interfaces/formats.md#formats). - -İşleme için birçok nüans var `NULL`. Örneğin, bir karşılaştırma işleminin argümanlarından en az biri ise `NULL`, bu işlemin sonucu da `NULL`. Aynı şey çarpma, toplama ve diğer işlemler için de geçerlidir. Daha fazla bilgi için her işlem için belgeleri okuyun. - -Sorgularda, kontrol edebilirsiniz `NULL` kullanarak [IS NULL](operators/index.md#operator-is-null) ve [IS NOT NULL](operators/index.md) operatörler ve ilgili fonksiyonlar `isNull` ve `isNotNull`. - -## İşlevler {#functions} - -İşlev çağrıları, yuvarlak parantez içinde bir argüman listesi (muhtemelen boş) olan bir tanımlayıcı gibi yazılır. Standart SQL'İN aksine, boş bir argüman listesi için bile parantezler gereklidir. Örnek: `now()`. -Düzenli ve agrega işlevleri vardır (bkz. “Aggregate functions”). Bazı toplama işlevleri parantez içinde iki bağımsız değişken listesi içerebilir. Örnek: `quantile (0.9) (x)`. Bu toplama fonksiyonları denir “parametric” fonksiyonlar ve ilk listedeki argümanlar çağrılır “parameters”. Parametresiz toplama işlevlerinin sözdizimi, normal işlevlerle aynıdır. - -## Operatörler {#operators} - -Operatörler, sorgu ayrıştırma sırasında önceliklerini ve ilişkilendirmelerini dikkate alarak karşılık gelen işlevlerine dönüştürülür. -Örneğin, ifade `1 + 2 * 3 + 4` dönüştür toülür `plus(plus(1, multiply(2, 3)), 4)`. - -## Veri türleri ve veritabanı tablosu motorları {#data_types-and-database-table-engines} - -Veri türleri ve tablo motorları `CREATE` sorgu tanımlayıcıları veya işlevleri aynı şekilde yazılır. Başka bir deyişle, parantez içinde bir argüman listesi içerebilir veya içermeyebilir. Daha fazla bilgi için bölümlere bakın “Data types,” “Table engines,” ve “CREATE”. - -## İfade Takma Adları {#syntax-expression_aliases} - -Diğer ad, sorgudaki ifade için kullanıcı tanımlı bir addır. - -``` sql -expr AS alias -``` - -- `AS` — The keyword for defining aliases. You can define the alias for a table name or a column name in a `SELECT` kullanmadan fık thera `AS` kelime. - - For example, `SELECT table_name_alias.column_name FROM table_name table_name_alias`. - - In the [CAST](sql_reference/functions/type_conversion_functions.md#type_conversion_function-cast) function, the `AS` keyword has another meaning. See the description of the function. - -- `expr` — Any expression supported by ClickHouse. - - For example, `SELECT column_name * 2 AS double FROM some_table`. - -- `alias` — Name for `expr`. Takma adlar ile uyumlu olmalıdır [tanıtıcılar](#syntax-identifiers) sözdizimi. - - For example, `SELECT "table t".column_name FROM table_name AS "table t"`. - -### Kullanımı ile ilgili notlar {#notes-on-usage} - -Diğer adlar bir sorgu veya alt sorgu için geneldir ve herhangi bir ifade için sorgunun herhangi bir bölümünde bir diğer ad tanımlayabilirsiniz. Mesela, `SELECT (1 AS n) + 2, n`. - -Diğer adlar alt sorgularda ve alt sorgular arasında görünmez. Örneğin, sorgu yürütülürken `SELECT (SELECT sum(b.a) + num FROM b) - a.a AS num FROM a` ClickHouse istisna oluşturur `Unknown identifier: num`. - -Sonuç sütunları için bir diğer ad tanımlanmışsa `SELECT` bir alt sorgunun yan tümcesi, bu sütunlar dış sorguda görülebilir. Mesela, `SELECT n + m FROM (SELECT 1 AS n, 2 AS m)`. - -Sütun veya tablo adlarıyla aynı olan diğer adlara dikkat edin. Aşağıdaki örneği ele alalım: - -``` sql -CREATE TABLE t -( - a Int, - b Int -) -ENGINE = TinyLog() -``` - -``` sql -SELECT - argMax(a, b), - sum(b) AS b -FROM t -``` - -``` text -Received exception from server (version 18.14.17): -Code: 184. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: Aggregate function sum(b) is found inside another aggregate function in query. -``` - -Bu örnekte, tablo ilan ettik `t` sütun ile `b`. Ardından, veri seçerken, `sum(b) AS b` takma ad. Takma adlar küresel olduğundan, ClickHouse literal yerine `b` ifad theesinde `argMax(a, b)` ifad theesiyle `sum(b)`. Bu ikame istisnaya neden oldu. - -## Yıldız işareti {#asterisk} - -İn a `SELECT` sorgu, bir yıldız ifadesinin yerini alabilir. Daha fazla bilgi için bölüme bakın “SELECT”. - -## İfadeler {#syntax-expressions} - -Bir ifade, bir işlev, tanımlayıcı, değişmez, bir operatörün uygulaması, parantez içindeki ifade, alt sorgu veya yıldız işaretidir. Ayrıca bir takma ad içerebilir. -İfadelerin listesi, virgülle ayrılmış bir veya daha fazla ifadedir. -Fonksiyonlar ve operatörler, sırayla, argüman olarak ifadelere sahip olabilirler. - -[Orijinal makale](https://clickhouse.tech/docs/en/sql_reference/syntax/) diff --git a/docs/tr/sql-reference/table-functions/file.md b/docs/tr/sql-reference/table-functions/file.md deleted file mode 100644 index 834a2b13cb1..00000000000 --- a/docs/tr/sql-reference/table-functions/file.md +++ /dev/null @@ -1,121 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 37 -toc_title: Dosya ---- - -# Dosya {#file} - -Bir dosyadan bir tablo oluşturur. Bu tablo işlevi benzer [url](url.md) ve [hdf'ler](hdfs.md) biri. - -``` sql -file(path, format, structure) -``` - -**Giriş parametreleri** - -- `path` — The relative path to the file from [user_files_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Readonly modunda glob'ları takip eden dosya desteğine giden yol: `*`, `?`, `{abc,def}` ve `{N..M}` nerede `N`, `M` — numbers, \``'abc', 'def'` — strings. -- `format` — The [biçimli](../../interfaces/formats.md#formats) dosya. -- `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`. - -**Döndürülen değer** - -Belirtilen dosyada veri okumak veya yazmak için belirtilen yapıya sahip bir tablo. - -**Örnek** - -Ayar `user_files_path` ve dosyanın içeriği `test.csv`: - -``` bash -$ grep user_files_path /etc/clickhouse-server/config.xml - /var/lib/clickhouse/user_files/ - -$ cat /var/lib/clickhouse/user_files/test.csv - 1,2,3 - 3,2,1 - 78,43,45 -``` - -Tablo fromdan`test.csv` ve ondan ilk iki satır seçimi: - -``` sql -SELECT * -FROM file('test.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32') -LIMIT 2 -``` - -``` text -┌─column1─┬─column2─┬─column3─┐ -│ 1 │ 2 │ 3 │ -│ 3 │ 2 │ 1 │ -└─────────┴─────────┴─────────┘ -``` - -``` sql --- getting the first 10 lines of a table that contains 3 columns of UInt32 type from a CSV file -SELECT * FROM file('test.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32') LIMIT 10 -``` - -**Yolda Globs** - -Birden çok yol bileşenleri globs olabilir. İşlenmek için dosya var olmalı ve tüm yol deseniyle eşleşmelidir (sadece sonek veya önek değil). - -- `*` — Substitutes any number of any characters except `/` boş dize dahil. -- `?` — Substitutes any single character. -- `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. -- `{N..M}` — Substitutes any number in range from N to M including both borders. - -İle yapılar `{}` benzer olan [uzaktan masa fonksiyonu](../../sql-reference/table-functions/remote.md)). - -**Örnek** - -1. Aşağıdaki göreli yollara sahip birkaç dosyamız olduğunu varsayalım: - -- ‘some_dir/some_file_1’ -- ‘some_dir/some_file_2’ -- ‘some_dir/some_file_3’ -- ‘another_dir/some_file_1’ -- ‘another_dir/some_file_2’ -- ‘another_dir/some_file_3’ - -1. Bu dosyalardaki satır miktarını sorgula: - - - -``` sql -SELECT count(*) -FROM file('{some,another}_dir/some_file_{1..3}', 'TSV', 'name String, value UInt32') -``` - -1. Bu iki dizinin tüm dosyalarındaki satır miktarını sorgula: - - - -``` sql -SELECT count(*) -FROM file('{some,another}_dir/*', 'TSV', 'name String, value UInt32') -``` - -!!! warning "Uyarıcı" - Dosya listenizde önde gelen sıfırlar içeren sayı aralıkları varsa, her basamak için parantez içeren yapıyı ayrı ayrı kullanın veya kullanın `?`. - -**Örnek** - -Adlı dosy thealardan verileri sorgu thelamak `file000`, `file001`, … , `file999`: - -``` sql -SELECT count(*) -FROM file('big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String, value UInt32') -``` - -## Sanal Sütunlar {#virtual-columns} - -- `_path` — Path to the file. -- `_file` — Name of the file. - -**Ayrıca Bakınız** - -- [Sanal sütunlar](https://clickhouse.tech/docs/en/operations/table_engines/#table_engines-virtual_columns) - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/file/) diff --git a/docs/tr/sql-reference/table-functions/generate.md b/docs/tr/sql-reference/table-functions/generate.md deleted file mode 100644 index 9259854e69f..00000000000 --- a/docs/tr/sql-reference/table-functions/generate.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 47 -toc_title: generateRandom ---- - -# generateRandom {#generaterandom} - -Verilen şema ile rastgele veri üretir. -Test tablolarını verilerle doldurmaya izin verir. -Dışında tabloda saklanabilir tüm veri türlerini destekler `LowCardinality` ve `AggregateFunction`. - -``` sql -generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_string_length'[, 'max_array_length']]]); -``` - -**Parametre** - -- `name` — Name of corresponding column. -- `TypeName` — Type of corresponding column. -- `max_array_length` — Maximum array length for all generated arrays. Defaults to `10`. -- `max_string_length` — Maximum string length for all generated strings. Defaults to `10`. -- `random_seed` — Specify random seed manually to produce stable results. If NULL — seed is randomly generated. - -**Döndürülen Değer** - -Istenen şema ile bir tablo nesnesi. - -## Kullanım Örneği {#usage-example} - -``` sql -SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 1, 10, 2) LIMIT 3; -``` - -``` text -┌─a────────┬────────────d─┬─c──────────────────────────────────────────────────────────────────┐ -│ [77] │ -124167.6723 │ ('2061-04-17 21:59:44.573','3f72f405-ec3e-13c8-44ca-66ef335f7835') │ -│ [32,110] │ -141397.7312 │ ('1979-02-09 03:43:48.526','982486d1-5a5d-a308-e525-7bd8b80ffa73') │ -│ [68] │ -67417.0770 │ ('2080-03-12 14:17:31.269','110425e5-413f-10a6-05ba-fa6b3e929f15') │ -└──────────┴──────────────┴────────────────────────────────────────────────────────────────────┘ -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/generate/) diff --git a/docs/tr/sql-reference/table-functions/hdfs.md b/docs/tr/sql-reference/table-functions/hdfs.md deleted file mode 100644 index 438a2bfe00e..00000000000 --- a/docs/tr/sql-reference/table-functions/hdfs.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 45 -toc_title: hdf'ler ---- - -# hdf'ler {#hdfs} - -Hdfs'deki dosyalardan bir tablo oluşturur. Bu tablo işlevi benzer [url](url.md) ve [Dosya](file.md) biri. - -``` sql -hdfs(URI, format, structure) -``` - -**Giriş parametreleri** - -- `URI` — The relative URI to the file in HDFS. Path to file support following globs in readonly mode: `*`, `?`, `{abc,def}` ve `{N..M}` nerede `N`, `M` — numbers, \``'abc', 'def'` — strings. -- `format` — The [biçimli](../../interfaces/formats.md#formats) dosya. -- `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`. - -**Döndürülen değer** - -Belirtilen dosyada veri okumak veya yazmak için belirtilen yapıya sahip bir tablo. - -**Örnek** - -Tablo fromdan `hdfs://hdfs1:9000/test` ve ondan ilk iki satır seçimi: - -``` sql -SELECT * -FROM hdfs('hdfs://hdfs1:9000/test', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') -LIMIT 2 -``` - -``` text -┌─column1─┬─column2─┬─column3─┐ -│ 1 │ 2 │ 3 │ -│ 3 │ 2 │ 1 │ -└─────────┴─────────┴─────────┘ -``` - -**Yolda Globs** - -Birden çok yol bileşenleri globs olabilir. İşlenmek için dosya var olmalı ve tüm yol deseniyle eşleşmelidir (sadece sonek veya önek değil). - -- `*` — Substitutes any number of any characters except `/` boş dize dahil. -- `?` — Substitutes any single character. -- `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. -- `{N..M}` — Substitutes any number in range from N to M including both borders. - -İle yapılar `{}` benzer olan [uzaktan masa fonksiyonu](../../sql-reference/table-functions/remote.md)). - -**Örnek** - -1. HDFS'DE aşağıdaki Urı'lere sahip birkaç dosyamız olduğunu varsayalım: - -- ‘hdfs://hdfs1:9000/some_dir/some_file_1’ -- ‘hdfs://hdfs1:9000/some_dir/some_file_2’ -- ‘hdfs://hdfs1:9000/some_dir/some_file_3’ -- ‘hdfs://hdfs1:9000/another_dir/some_file_1’ -- ‘hdfs://hdfs1:9000/another_dir/some_file_2’ -- ‘hdfs://hdfs1:9000/another_dir/some_file_3’ - -1. Bu dosyalardaki satır miktarını sorgula: - - - -``` sql -SELECT count(*) -FROM hdfs('hdfs://hdfs1:9000/{some,another}_dir/some_file_{1..3}', 'TSV', 'name String, value UInt32') -``` - -1. Bu iki dizinin tüm dosyalarındaki satır miktarını sorgula: - - - -``` sql -SELECT count(*) -FROM hdfs('hdfs://hdfs1:9000/{some,another}_dir/*', 'TSV', 'name String, value UInt32') -``` - -!!! warning "Uyarıcı" - Dosya listenizde önde gelen sıfırlar içeren sayı aralıkları varsa, her basamak için parantez içeren yapıyı ayrı ayrı kullanın veya kullanın `?`. - -**Örnek** - -Adlı dosy thealardan verileri sorgu thelamak `file000`, `file001`, … , `file999`: - -``` sql -SELECT count(*) -FROM hdfs('hdfs://hdfs1:9000/big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String, value UInt32') -``` - -## Sanal Sütunlar {#virtual-columns} - -- `_path` — Path to the file. -- `_file` — Name of the file. - -**Ayrıca Bakınız** - -- [Sanal sütunlar](https://clickhouse.tech/docs/en/operations/table_engines/#table_engines-virtual_columns) - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/hdfs/) diff --git a/docs/tr/sql-reference/table-functions/index.md b/docs/tr/sql-reference/table-functions/index.md deleted file mode 100644 index 3d9715be66a..00000000000 --- a/docs/tr/sql-reference/table-functions/index.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "Tablo Fonksiyonlar\u0131" -toc_priority: 34 -toc_title: "Giri\u015F" ---- - -# Tablo Fonksiyonları {#table-functions} - -Tablo işlevleri tabloları oluşturmak için yöntemlerdir. - -Tablo işlevlerini kullanabilirsiniz: - -- [FROM](../statements/select/from.md) fıkra ofsı `SELECT` sorgu. - - The method for creating a temporary table that is available only in the current query. The table is deleted when the query finishes. - -- [Tablo oluştur \](../statements/create.md#create-table-query) sorgu. - - It's one of the methods of creating a table. - -!!! warning "Uyarıcı" - Eğer tablo işlevlerini kullanamazsınız [allow_ddl](../../operations/settings/permissions-for-queries.md#settings_allow_ddl) ayarı devre dışı. - -| İşlev | Açıklama | -|--------------------------|-----------------------------------------------------------------------------------------------------------------------------| -| [Dosya](file.md) | Oluşturur bir [Dosya](../../engines/table-engines/special/file.md)- motor masası. | -| [birleştirmek](merge.md) | Oluşturur bir [Birleştirmek](../../engines/table-engines/special/merge.md)- motor masası. | -| [şiir](numbers.md) | Tamsayı sayılarla dolu tek bir sütun içeren bir tablo oluşturur. | -| [uzak](remote.md) | Oluşturmadan uzak sunuculara erişmenizi sağlar. [Dağılı](../../engines/table-engines/special/distributed.md)- motor masası. | -| [url](url.md) | Oluşturur bir [Url](../../engines/table-engines/special/url.md)- motor masası. | -| [mysql](mysql.md) | Oluşturur bir [MySQL](../../engines/table-engines/integrations/mysql.md)- motor masası. | -| [jdbc](jdbc.md) | Oluşturur bir [JDBC](../../engines/table-engines/integrations/jdbc.md)- motor masası. | -| [odbc](odbc.md) | Oluşturur bir [ODBC](../../engines/table-engines/integrations/odbc.md)- motor masası. | -| [hdf'ler](hdfs.md) | Oluşturur bir [HDFS](../../engines/table-engines/integrations/hdfs.md)- motor masası. | - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/) diff --git a/docs/tr/sql-reference/table-functions/input.md b/docs/tr/sql-reference/table-functions/input.md deleted file mode 100644 index cf6d65f1723..00000000000 --- a/docs/tr/sql-reference/table-functions/input.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 46 -toc_title: girdi ---- - -# girdi {#input} - -`input(structure)` - etkin bir şekilde dönüştürmek ve veri eklemek sağlar tablo fonksiyonu gönderilen -başka bir yapıya sahip tabloya verilen yapıya sahip sunucu. - -`structure` - aşağıdaki formatta sunucuya gönderilen verilerin yapısı `'column1_name column1_type, column2_name column2_type, ...'`. -Mesela, `'id UInt32, name String'`. - -Bu işlev yalnızca kullanılabilir `INSERT SELECT` sorgu ve sadece bir kez ama aksi takdirde sıradan tablo işlevi gibi davranır -(örneğin, alt sorguda vb.kullanılabilir.). - -Veri sıradan gibi herhangi bir şekilde gönderilebilir `INSERT` sorgu ve herhangi bir kullanılabilir geçti [biçimli](../../interfaces/formats.md#formats) -bu sorgu sonunda belirtilmelidir (sıradan aksine `INSERT SELECT`). - -Bu işlevin ana özelliği, sunucu istemciden veri aldığında aynı anda onu dönüştürmesidir -ifadeler listesine göre `SELECT` yan tümcesi ve hedef tabloya ekler. Geçici tablo -aktarılan tüm veriler ile oluşturulmaz. - -**Örnekler** - -- L letet the `test` tablo aşağıdaki yapıya sahiptir `(a String, b String)` - ve veri `data.csv` farklı bir yapıya sahiptir `(col1 String, col2 Date, col3 Int32)`. Insert sorgusu - bu verileri `data.csv` içine `test` eşzamanlı dönüşüm ile tablo şöyle görünüyor: - - - -``` bash -$ cat data.csv | clickhouse-client --query="INSERT INTO test SELECT lower(col1), col3 * col3 FROM input('col1 String, col2 Date, col3 Int32') FORMAT CSV"; -``` - -- Eğer `data.csv` aynı yapının verilerini içerir `test_structure` tablo olarak `test` sonra bu iki sorgu eşittir: - - - -``` bash -$ cat data.csv | clickhouse-client --query="INSERT INTO test FORMAT CSV" -$ cat data.csv | clickhouse-client --query="INSERT INTO test SELECT * FROM input('test_structure') FORMAT CSV" -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/input/) diff --git a/docs/tr/sql-reference/table-functions/jdbc.md b/docs/tr/sql-reference/table-functions/jdbc.md deleted file mode 100644 index bd664a093ac..00000000000 --- a/docs/tr/sql-reference/table-functions/jdbc.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 43 -toc_title: jdbc ---- - -# jdbc {#table-function-jdbc} - -`jdbc(jdbc_connection_uri, schema, table)` - JDBC sürücüsü ile bağlı döner tablo. - -Bu tablo işlevi ayrı gerektirir `clickhouse-jdbc-bridge` program çalıştırılacak. -Bu (sorgulanan uzak tablonun DDL dayalı) null türleri destekler. - -**Örnekler** - -``` sql -SELECT * FROM jdbc('jdbc:mysql://localhost:3306/?user=root&password=root', 'schema', 'table') -``` - -``` sql -SELECT * FROM jdbc('mysql://localhost:3306/?user=root&password=root', 'schema', 'table') -``` - -``` sql -SELECT * FROM jdbc('datasource://mysql-local', 'schema', 'table') -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/jdbc/) diff --git a/docs/tr/sql-reference/table-functions/merge.md b/docs/tr/sql-reference/table-functions/merge.md deleted file mode 100644 index 867dc0c3205..00000000000 --- a/docs/tr/sql-reference/table-functions/merge.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 38 -toc_title: "birle\u015Ftirmek" ---- - -# birleştirmek {#merge} - -`merge(db_name, 'tables_regexp')` – Creates a temporary Merge table. For more information, see the section “Table engines, Merge”. - -Tablo yapısı, normal ifadeyle eşleşen ilk tablodan alınır. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/merge/) diff --git a/docs/tr/sql-reference/table-functions/mysql.md b/docs/tr/sql-reference/table-functions/mysql.md deleted file mode 100644 index 24e9d0b68c5..00000000000 --- a/docs/tr/sql-reference/table-functions/mysql.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 42 -toc_title: mysql ---- - -# mysql {#mysql} - -Veriyor `SELECT` uzak bir MySQL sunucusunda depolanan veriler üzerinde gerçekleştirilecek sorgular. - -``` sql -mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']); -``` - -**Parametre** - -- `host:port` — MySQL server address. - -- `database` — Remote database name. - -- `table` — Remote table name. - -- `user` — MySQL user. - -- `password` — User password. - -- `replace_query` — Flag that converts `INSERT INTO` için sorgular `REPLACE INTO`. Eğer `replace_query=1`, sorgu değiştirilir. - -- `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` eklenen ifade `INSERT` sorgu. - - Example: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, where `on_duplicate_clause` is `UPDATE c2 = c2 + 1`. See the MySQL documentation to find which `on_duplicate_clause` you can use with the `ON DUPLICATE KEY` clause. - - To specify `on_duplicate_clause` you need to pass `0` to the `replace_query` parameter. If you simultaneously pass `replace_query = 1` and `on_duplicate_clause`, ClickHouse generates an exception. - -Basit `WHERE` gibi maddeler `=, !=, >, >=, <, <=` şu anda MySQL sunucusunda yürütülür. - -Geri kalan şartlar ve `LIMIT` örnekleme kısıtlaması, yalnızca MySQL sorgusu bittikten sonra Clickhouse'da yürütülür. - -**Döndürülen Değer** - -Orijinal MySQL tablosu ile aynı sütunlara sahip bir tablo nesnesi. - -## Kullanım Örneği {#usage-example} - -MySQL tablo: - -``` text -mysql> CREATE TABLE `test`.`test` ( - -> `int_id` INT NOT NULL AUTO_INCREMENT, - -> `int_nullable` INT NULL DEFAULT NULL, - -> `float` FLOAT NOT NULL, - -> `float_nullable` FLOAT NULL DEFAULT NULL, - -> PRIMARY KEY (`int_id`)); -Query OK, 0 rows affected (0,09 sec) - -mysql> insert into test (`int_id`, `float`) VALUES (1,2); -Query OK, 1 row affected (0,00 sec) - -mysql> select * from test; -+------+----------+-----+----------+ -| int_id | int_nullable | float | float_nullable | -+------+----------+-----+----------+ -| 1 | NULL | 2 | NULL | -+------+----------+-----+----------+ -1 row in set (0,00 sec) -``` - -Clickhouse'dan veri seçme: - -``` sql -SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123') -``` - -``` text -┌─int_id─┬─int_nullable─┬─float─┬─float_nullable─┐ -│ 1 │ ᴺᵁᴸᴸ │ 2 │ ᴺᵁᴸᴸ │ -└────────┴──────────────┴───────┴────────────────┘ -``` - -## Ayrıca Bakınız {#see-also} - -- [Bu ‘MySQL’ masa motoru](../../engines/table-engines/integrations/mysql.md) -- [Harici sözlük kaynağı olarak MySQL kullanma](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-mysql) - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/mysql/) diff --git a/docs/tr/sql-reference/table-functions/numbers.md b/docs/tr/sql-reference/table-functions/numbers.md deleted file mode 100644 index f475483f6dd..00000000000 --- a/docs/tr/sql-reference/table-functions/numbers.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 39 -toc_title: "\u015Fiir" ---- - -# şiir {#numbers} - -`numbers(N)` – Returns a table with the single ‘number’ 0'dan n-1'e kadar tamsayılar içeren sütun (Uİnt64). -`numbers(N, M)` - Tek bir tablo döndürür ‘number’ n'den (n + M - 1) tamsayıları içeren sütun (Uİnt64). - -Benzer `system.numbers` tablo, ardışık değerleri test etmek ve üretmek için kullanılabilir, `numbers(N, M)` daha verimli `system.numbers`. - -Aşağıdaki sorgular eşdeğerdir: - -``` sql -SELECT * FROM numbers(10); -SELECT * FROM numbers(0, 10); -SELECT * FROM system.numbers LIMIT 10; -``` - -Örnekler: - -``` sql --- Generate a sequence of dates from 2010-01-01 to 2010-12-31 -select toDate('2010-01-01') + number as d FROM numbers(365); -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/numbers/) diff --git a/docs/tr/sql-reference/table-functions/odbc.md b/docs/tr/sql-reference/table-functions/odbc.md deleted file mode 100644 index e932efbcd6a..00000000000 --- a/docs/tr/sql-reference/table-functions/odbc.md +++ /dev/null @@ -1,108 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 44 -toc_title: odbc ---- - -# odbc {#table-functions-odbc} - -Üzerinden bağlanan tabloyu döndürür [ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity). - -``` sql -odbc(connection_settings, external_database, external_table) -``` - -Parametre: - -- `connection_settings` — Name of the section with connection settings in the `odbc.ini` Dosya. -- `external_database` — Name of a database in an external DBMS. -- `external_table` — Name of a table in the `external_database`. - -ODBC bağlantılarını güvenli bir şekilde uygulamak için ClickHouse ayrı bir program kullanır `clickhouse-odbc-bridge`. ODBC sürücüsü doğrudan yüklenmişse `clickhouse-server`, sürücü sorunları ClickHouse sunucu çökmesine neden olabilir. ClickHouse otomatik olarak başlar `clickhouse-odbc-bridge` gerekli olduğunda. ODBC Köprüsü programı aynı paketten yüklenir `clickhouse-server`. - -Alanları ile `NULL` dış tablodaki değerler, temel veri türü için varsayılan değerlere dönüştürülür. Örneğin, uzak bir MySQL tablo alanı `INT NULL` yazın 0'a dönüştürülür (ClickHouse için varsayılan değer `Int32` veri türü). - -## Kullanım Örneği {#usage-example} - -**ODBC üzerinden yerel MySQL kurulumundan veri alma** - -Bu örnek Ubuntu Linux 18.04 ve MySQL server 5.7 için kontrol edilir. - -UnixODBC ve MySQL Connector yüklü olduğundan emin olun. - -Varsayılan olarak (paketlerden yüklüyse), ClickHouse kullanıcı olarak başlar `clickhouse`. Bu nedenle, bu kullanıcıyı MySQL sunucusunda oluşturmanız ve yapılandırmanız gerekir. - -``` bash -$ sudo mysql -``` - -``` sql -mysql> CREATE USER 'clickhouse'@'localhost' IDENTIFIED BY 'clickhouse'; -mysql> GRANT ALL PRIVILEGES ON *.* TO 'clickhouse'@'clickhouse' WITH GRANT OPTION; -``` - -Sonra bağlantıyı yapılandırın `/etc/odbc.ini`. - -``` bash -$ cat /etc/odbc.ini -[mysqlconn] -DRIVER = /usr/local/lib/libmyodbc5w.so -SERVER = 127.0.0.1 -PORT = 3306 -DATABASE = test -USERNAME = clickhouse -PASSWORD = clickhouse -``` - -Kullanarak bağlantıyı kontrol edebilirsiniz `isql` unixodbc yüklemesinden yardımcı program. - -``` bash -$ isql -v mysqlconn -+-------------------------+ -| Connected! | -| | -... -``` - -MySQL tablo: - -``` text -mysql> CREATE TABLE `test`.`test` ( - -> `int_id` INT NOT NULL AUTO_INCREMENT, - -> `int_nullable` INT NULL DEFAULT NULL, - -> `float` FLOAT NOT NULL, - -> `float_nullable` FLOAT NULL DEFAULT NULL, - -> PRIMARY KEY (`int_id`)); -Query OK, 0 rows affected (0,09 sec) - -mysql> insert into test (`int_id`, `float`) VALUES (1,2); -Query OK, 1 row affected (0,00 sec) - -mysql> select * from test; -+------+----------+-----+----------+ -| int_id | int_nullable | float | float_nullable | -+------+----------+-----+----------+ -| 1 | NULL | 2 | NULL | -+------+----------+-----+----------+ -1 row in set (0,00 sec) -``` - -Clickhouse'daki MySQL tablosundan veri alma: - -``` sql -SELECT * FROM odbc('DSN=mysqlconn', 'test', 'test') -``` - -``` text -┌─int_id─┬─int_nullable─┬─float─┬─float_nullable─┐ -│ 1 │ 0 │ 2 │ 0 │ -└────────┴──────────────┴───────┴────────────────┘ -``` - -## Ayrıca Bakınız {#see-also} - -- [ODBC harici sözlükler](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-odbc) -- [ODBC tablo motoru](../../engines/table-engines/integrations/odbc.md). - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/jdbc/) diff --git a/docs/tr/sql-reference/table-functions/remote.md b/docs/tr/sql-reference/table-functions/remote.md deleted file mode 100644 index 8d5427c91e4..00000000000 --- a/docs/tr/sql-reference/table-functions/remote.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 40 -toc_title: uzak ---- - -# uzaktan, remoteSecure {#remote-remotesecure} - -Oluşturmadan uzak sunuculara erişmenizi sağlar. `Distributed` Tablo. - -İmzalar: - -``` sql -remote('addresses_expr', db, table[, 'user'[, 'password']]) -remote('addresses_expr', db.table[, 'user'[, 'password']]) -remoteSecure('addresses_expr', db, table[, 'user'[, 'password']]) -remoteSecure('addresses_expr', db.table[, 'user'[, 'password']]) -``` - -`addresses_expr` – An expression that generates addresses of remote servers. This may be just one server address. The server address is `host:port` ya da sadece `host`. Ana bilgisayar sunucu adı veya IPv4 veya IPv6 adresi olarak belirtilebilir. Köşeli parantez içinde bir IPv6 adresi belirtilir. Bağlantı noktası, uzak sunucudaki TCP bağlantı noktasıdır. Bağlantı noktası atlanırsa, kullanır `tcp_port` sunucunun yapılandırma dosyasından (varsayılan olarak, 9000). - -!!! important "Önemli" - Bir IPv6 adresi için bağlantı noktası gereklidir. - -Örnekler: - -``` text -example01-01-1 -example01-01-1:9000 -localhost -127.0.0.1 -[::]:9000 -[2a02:6b8:0:1111::11]:9000 -``` - -Birden çok Adres virgülle ayrılmış olabilir. Bu durumda, ClickHouse dağıtılmış işleme kullanır, bu nedenle sorguyu belirtilen tüm adreslere gönderir (farklı verilerle kırıklar gibi). - -Örnek: - -``` text -example01-01-1,example01-02-1 -``` - -İfadenin bir kısmı kıvırcık parantez içinde belirtilebilir. Önceki örnek aşağıdaki gibi yazılabilir: - -``` text -example01-0{1,2}-1 -``` - -Kıvırcık parantez iki nokta (negatif olmayan tamsayılar) ile ayrılmış bir sayı aralığı içerebilir. Bu durumda, Aralık, shard adresleri üreten bir değer kümesine genişletilir. İlk sayı sıfır ile başlarsa, değerler aynı sıfır hizalamasıyla oluşturulur. Önceki örnek aşağıdaki gibi yazılabilir: - -``` text -example01-{01..02}-1 -``` - -Birden fazla kıvırcık parantez çiftiniz varsa, ilgili kümelerin doğrudan ürününü oluşturur. - -Adresler ve kıvırcık parantez içindeki adreslerin parçaları boru sembolü (\|) ile ayrılabilir. Bu durumda, karşılık gelen Adres kümeleri yinelemeler olarak yorumlanır ve sorgu ilk sağlıklı yinelemeye gönderilir. Ancak, yinelemeler şu anda ayarlanmış sırayla yinelenir [dengeleme](../../operations/settings/settings.md) ayar. - -Örnek: - -``` text -example01-{01..02}-{1|2} -``` - -Bu örnek, her birinin iki kopyası olan iki parçayı belirtir. - -Oluşturulan Adres sayısı bir sabit tarafından sınırlıdır. Şu anda bu 1000 Adres. - -Kullanarak `remote` tablo işlevi, bir `Distributed` tablo, çünkü bu durumda, her istek için sunucu bağlantısı yeniden kurulur. Buna ek olarak, ana bilgisayar adları ayarlanmışsa, adlar giderilir ve çeşitli yinelemelerle çalışırken hatalar sayılmaz. Çok sayıda sorgu işlerken, her zaman `Distributed` masa vaktinden önce, ve kullanmayın `remote` tablo işlevi. - -Bu `remote` tablo işlevi aşağıdaki durumlarda yararlı olabilir: - -- Veri karşılaştırma, hata ayıklama ve sınama için belirli bir sunucuya erişme. -- Araştırma amaçlı çeşitli ClickHouse kümeleri arasındaki sorgular. -- El ile yapılan seyrek dağıtılmış istekler. -- Sunucu kümesinin her seferinde yeniden tanımlandığı dağıtılmış istekler. - -Kullanıcı belirtilmemişse, `default` kullanılır. -Parola belirtilmezse, boş bir parola kullanılır. - -`remoteSecure` - aynı `remote` but with secured connection. Default port — [tcp_port_secure](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port_secure) yapılandırma veya 9440'ten. - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/remote/) diff --git a/docs/tr/sql-reference/table-functions/url.md b/docs/tr/sql-reference/table-functions/url.md deleted file mode 100644 index ba1277c69d0..00000000000 --- a/docs/tr/sql-reference/table-functions/url.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 41 -toc_title: url ---- - -# url {#url} - -`url(URL, format, structure)` - oluşturulan bir tablo döndürür `URL` verilen ile -`format` ve `structure`. - -Kabul edebilen URL - HTTP veya HTTPS sunucu adresi `GET` ve / veya `POST` istemler. - -biçimli - [biçimli](../../interfaces/formats.md#formats) verilerin. - -yapı-tablo yapısı `'UserID UInt64, Name String'` biçimli. Sütun adlarını ve türlerini belirler. - -**Örnek** - -``` sql --- getting the first 3 lines of a table that contains columns of String and UInt32 type from HTTP-server which answers in CSV format. -SELECT * FROM url('http://127.0.0.1:12345/', CSV, 'column1 String, column2 UInt32') LIMIT 3 -``` - -[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/url/) diff --git a/docs/tr/whats-new/changelog/2017.md b/docs/tr/whats-new/changelog/2017.md deleted file mode 120000 index d581cbbb422..00000000000 --- a/docs/tr/whats-new/changelog/2017.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/whats-new/changelog/2017.md \ No newline at end of file diff --git a/docs/tr/whats-new/changelog/2018.md b/docs/tr/whats-new/changelog/2018.md deleted file mode 120000 index 22874fcae85..00000000000 --- a/docs/tr/whats-new/changelog/2018.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/whats-new/changelog/2018.md \ No newline at end of file diff --git a/docs/tr/whats-new/changelog/2019.md b/docs/tr/whats-new/changelog/2019.md deleted file mode 120000 index 0f3f095f8a1..00000000000 --- a/docs/tr/whats-new/changelog/2019.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/whats-new/changelog/2019.md \ No newline at end of file diff --git a/docs/tr/whats-new/changelog/index.md b/docs/tr/whats-new/changelog/index.md deleted file mode 120000 index 5461b93ec8c..00000000000 --- a/docs/tr/whats-new/changelog/index.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/whats-new/changelog/index.md \ No newline at end of file diff --git a/docs/tr/whats-new/index.md b/docs/tr/whats-new/index.md deleted file mode 100644 index fa00602d6be..00000000000 --- a/docs/tr/whats-new/index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: Ne yeni -toc_priority: 72 ---- - - diff --git a/docs/tr/whats-new/roadmap.md b/docs/tr/whats-new/roadmap.md deleted file mode 100644 index adf05d821e8..00000000000 --- a/docs/tr/whats-new/roadmap.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 74 -toc_title: "Yol haritas\u0131" ---- - -# Yol haritası {#roadmap} - -## Q1 2020 {#q1-2020} - -- Rol tabanlı erişim denetimi - -## Q2 2020 {#q2-2020} - -- Dış kimlik doğrulama hizmetleri ile entegrasyon -- Kullanıcılar arasında küme kapasitesinin daha hassas dağılımı için kaynak havuzları - -{## [Orijinal makale](https://clickhouse.tech/docs/en/roadmap/) ##} diff --git a/docs/tr/whats-new/security-changelog.md b/docs/tr/whats-new/security-changelog.md deleted file mode 100644 index 8e93723aa71..00000000000 --- a/docs/tr/whats-new/security-changelog.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 76 -toc_title: "G\xFCvenlik Changelog" ---- - -## ClickHouse sürümünde düzeltildi 19.14.3.3, 2019-09-10 {#fixed-in-clickhouse-release-19-14-3-3-2019-09-10} - -### CVE-2019-15024 {#cve-2019-15024} - -Аn attacker that has write access to ZooKeeper and who ican run a custom server available from the network where ClickHouse runs, can create a custom-built malicious server that will act as a ClickHouse replica and register it in ZooKeeper. When another replica will fetch data part from the malicious replica, it can force clickhouse-server to write to arbitrary path on filesystem. - -Kredi: Yandex Bilgi Güvenliği ekibinden Eldar Zaitov - -### CVE-2019-16535 {#cve-2019-16535} - -Аn OOB read, OOB write and integer underflow in decompression algorithms can be used to achieve RCE or DoS via native protocol. - -Kredi: Yandex Bilgi Güvenliği ekibinden Eldar Zaitov - -### CVE-2019-16536 {#cve-2019-16536} - -DOS'A giden yığın taşması, kötü amaçlı kimliği doğrulanmış bir istemci tarafından tetiklenebilir. - -Kredi: Yandex Bilgi Güvenliği ekibinden Eldar Zaitov - -## ClickHouse sürümü 19.13.6.1, 2019-09-20'de düzeltildi {#fixed-in-clickhouse-release-19-13-6-1-2019-09-20} - -### CVE-2019-18657 {#cve-2019-18657} - -Tablo fonksiyonu `url` güvenlik açığı saldırganın istekte rasgele HTTP üstbilgileri enjekte etmesine izin vermişti. - -Krediler: [Nikita Tikhomirov](https://github.com/NSTikhomirov) - -## ClickHouse sürümünde sabit 18.12.13, 2018-09-10 {#fixed-in-clickhouse-release-18-12-13-2018-09-10} - -### CVE-2018-14672 {#cve-2018-14672} - -Catboost modellerini yüklemek için işlevler, yol geçişine izin verdi ve hata mesajları aracılığıyla keyfi dosyaları okudu. - -Kredi: Yandex Bilgi Güvenliği ekibinden Andrey Krasichkov - -## ClickHouse sürüm 18.10.3, 2018-08-13 sabit {#fixed-in-clickhouse-release-18-10-3-2018-08-13} - -### CVE-2018-14671 {#cve-2018-14671} - -unixODBC, dosya sisteminden rasgele paylaşılan nesnelerin yüklenmesine izin verdi ve bu da uzaktan kod yürütme güvenlik açığına yol açtı. - -Kredi: Yandex Bilgi Güvenliği ekibinden Andrey Krasichkov ve Evgeny Sidorov - -## ClickHouse sürüm 1.1.54388, 2018-06-28 sabit {#fixed-in-clickhouse-release-1-1-54388-2018-06-28} - -### CVE-2018-14668 {#cve-2018-14668} - -“remote” tablo fonksiyonu izin keyfi semboller “user”, “password” ve “default_database” çapraz Protokol isteği sahtecilik saldırılarına yol açan alanlar. - -Kredi: Yandex Bilgi Güvenliği ekibinden Andrey Krasichkov - -## ClickHouse sürüm 1.1.54390, 2018-07-06 sabit {#fixed-in-clickhouse-release-1-1-54390-2018-07-06} - -### CVE-2018-14669 {#cve-2018-14669} - -ClickHouse MySQL istemcisi vardı “LOAD DATA LOCAL INFILE” işlevsellik, kötü niyetli bir MySQL veritabanının bağlı ClickHouse sunucusundan rasgele dosyaları okumasına izin verdi. - -Kredi: Yandex Bilgi Güvenliği ekibinden Andrey Krasichkov ve Evgeny Sidorov - -## ClickHouse sürüm 1.1.54131, 2017-01-10 sabit {#fixed-in-clickhouse-release-1-1-54131-2017-01-10} - -### CVE-2018-14670 {#cve-2018-14670} - -Deb paketindeki yanlış yapılandırma, veritabanının yetkisiz kullanımına neden olabilir. - -Kredi: İngiltere'nin Ulusal siber güvenlik merkezi (NCSC) - -{## [Orijinal makale](https://clickhouse.tech/docs/en/security_changelog/) ##} From 52686198e3e0da89edf8fc0a5ab1b0707b13bb34 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Thu, 31 Dec 2020 00:49:52 +0300 Subject: [PATCH 270/284] Note added, syntax error fixed --- docs/en/sql-reference/table-functions/mysql.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/table-functions/mysql.md b/docs/en/sql-reference/table-functions/mysql.md index 0af9ae415c2..b28afa4f2ef 100644 --- a/docs/en/sql-reference/table-functions/mysql.md +++ b/docs/en/sql-reference/table-functions/mysql.md @@ -43,6 +43,9 @@ The rest of the conditions and the `LIMIT` sampling constraint are executed in C A table object with the same columns as the original MySQL table. +!!! info "Note" + In the `INSERT` query to distinguish table function `mysql(...)` from table name with column names list you must use keywords `FUNCTION` or `TABLE FUNCTION`. See examples below. + **Examples** Table in MySQL: @@ -78,8 +81,8 @@ SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123'); Replacing and inserting: ```sql -INSERT INTO mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 1) ('int_id', 'float') VALUES (1, 3); -INSERT INTO mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 0, 'UPDATE int_id = int_id + 1') ('int_id', 'float') VALUES (1, 4); +INSERT INTO FUNCTION mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 1) ('int_id', 'float') VALUES (1, 3); +INSERT INTO TABLE FUNCTION mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 0, 'UPDATE int_id = int_id + 1') ('int_id', 'float') VALUES (1, 4); SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123'); ``` From aa2e89611d9ca60f2984ea1db98277bdd4d21917 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 31 Dec 2020 02:22:04 +0300 Subject: [PATCH 271/284] Remove sumburConsistentHash function --- contrib/CMakeLists.txt | 1 - .../consistent-hashing-sumbur/CMakeLists.txt | 2 - contrib/consistent-hashing-sumbur/sumbur.cpp | 113 ------------------ contrib/consistent-hashing-sumbur/sumbur.h | 28 ----- src/Functions/CMakeLists.txt | 1 - .../registerFunctionsConsistentHashing.cpp | 8 -- src/Functions/sumburConsistentHash.cpp | 38 ------ src/Functions/ya.make.in | 2 +- utils/check-style/check-include | 1 - 9 files changed, 1 insertion(+), 193 deletions(-) delete mode 100644 contrib/consistent-hashing-sumbur/CMakeLists.txt delete mode 100644 contrib/consistent-hashing-sumbur/sumbur.cpp delete mode 100644 contrib/consistent-hashing-sumbur/sumbur.h delete mode 100644 src/Functions/sumburConsistentHash.cpp diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 3bd08de01b2..be3d3f86348 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -31,7 +31,6 @@ set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1) add_subdirectory (antlr4-runtime-cmake) add_subdirectory (boost-cmake) add_subdirectory (cctz-cmake) -add_subdirectory (consistent-hashing-sumbur) add_subdirectory (consistent-hashing) add_subdirectory (dragonbox-cmake) add_subdirectory (FastMemcpy) diff --git a/contrib/consistent-hashing-sumbur/CMakeLists.txt b/contrib/consistent-hashing-sumbur/CMakeLists.txt deleted file mode 100644 index f11b5095f51..00000000000 --- a/contrib/consistent-hashing-sumbur/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_library(consistent-hashing-sumbur sumbur.cpp) -target_include_directories(consistent-hashing-sumbur PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/contrib/consistent-hashing-sumbur/sumbur.cpp b/contrib/consistent-hashing-sumbur/sumbur.cpp deleted file mode 100644 index 78f59ca875f..00000000000 --- a/contrib/consistent-hashing-sumbur/sumbur.cpp +++ /dev/null @@ -1,113 +0,0 @@ -//Copyright (c) 2011-2012 Mail.RU -//Copyright (c) 2011-2012 Maksim Kalinchenko -//Copyright (c) 2012 Sokolov Yura aka funny-falcon -// -//MIT License -// -//Permission is hereby granted, free of charge, to any person obtaining -//a copy of this software and associated documentation files (the -//"Software"), to deal in the Software without restriction, including -//without limitation the rights to use, copy, modify, merge, publish, -//distribute, sublicense, and/or sell copies of the Software, and to -//permit persons to whom the Software is furnished to do so, subject to -//the following conditions: -// -//The above copyright notice and this permission notice shall be -//included in all copies or substantial portions of the Software. -// -//THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -//EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -//MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -//NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -//LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -//OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -//WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -#include - - -#define L 0xFFFFFFFF - -static unsigned int L27_38[] = {L / 27, L / 28, L / 29, L / 30, L / 31, L / 32, - L / 33, L / 34, L / 35, L / 36, L / 37, L / 38, - L / 39, L / 40, L / 41, L / 42, L / 43, L / 44, - L / 45, L / 46, L / 47, L / 48, L / 49, L / 50, - L / 51, L / 52, L / 53, L / 54, L / 55, L / 56, - L / 57, L / 58, L / 59, L / 60, L / 61, L / 62 - }; -static unsigned int LL27_38[] = {L/(26*27), L/(27*28), L/(28*29), L/(29*30), L/(30*31), L/(31*32), - L/(32*33), L/(33*34), L/(34*35), L/(35*36), L/(36*37), L/(37*38), - L/(38*39), L/(39*40), L/(40*41), L/(41*42), L/(42*43), L/(43*44), - L/(44*45), L/(45*46), L/(46*47), L/(47*48), L/(48*49), L/(49*50), - L/(50*51), L/(51*52), L/(52*53), L/(53*54), L/(54*55), L/(55*56), - L/(56*57), L/(57*58), L/(58*59), L/(59*60), L/(60*61), L/(61*62) - }; - -unsigned int sumburConsistentHash(unsigned int hashed_int, unsigned int capacity) -{ - unsigned int h = hashed_int; - unsigned int capa = capacity; - unsigned int part, n, i, c; - - if (capa == 0) - throw std::runtime_error("Sumbur is not applicable to empty cluster"); - - part = L / capa; - - if (L - h < part) return 0; - - n = 1; - - do { - if (h >= L / 2) h -= L / 2; - else { - n = 2; - if (L / 2 - h < part) return 1; - } - if (capa == 2) return 1; - -#define curslice(i) (L / (i * (i - 1))) -#define unroll(i) \ - if (curslice(i) <= h) h -= curslice(i); \ - else { \ - h += curslice(i) * (i - n - 1); \ - n = i; \ - if (L / i - h < part) return n-1; \ - } \ - if (capa == i) return (n-1) - - unroll(3); unroll(4); unroll(5); - unroll(6); unroll(7); unroll(8); - unroll(9); unroll(10); unroll(11); - unroll(12); unroll(13); unroll(14); - unroll(15); unroll(16); unroll(17); - unroll(18); unroll(19); unroll(20); - unroll(21); unroll(22); unroll(23); - unroll(24); unroll(25); unroll(26); - - for (i = 27; i <= capa && i <= 62; i++) { - c = LL27_38[i-27]; - if (c <= h) { - h -= c; - } - else { - h += c * (i - n - 1); - n = i; - if (L27_38[i-27] - h < part) return n-1; - } - } - - for(i = 63; i <= capa; i++) { - c = L / (i * (i - 1)); - if (c <= h) { - h -= c; - } - else { - h += c * (i - n - 1); - n = i; - if (L / i - h < part) return n - 1; - } - } - } while(false); - return n - 1; -} diff --git a/contrib/consistent-hashing-sumbur/sumbur.h b/contrib/consistent-hashing-sumbur/sumbur.h deleted file mode 100644 index 1632665a073..00000000000 --- a/contrib/consistent-hashing-sumbur/sumbur.h +++ /dev/null @@ -1,28 +0,0 @@ -//Copyright (c) 2011-2012 Mail.RU -//Copyright (c) 2011-2012 Maksim Kalinchenko -//Copyright (c) 2012 Sokolov Yura aka funny-falcon -// -//MIT License -// -//Permission is hereby granted, free of charge, to any person obtaining -//a copy of this software and associated documentation files (the -//"Software"), to deal in the Software without restriction, including -//without limitation the rights to use, copy, modify, merge, publish, -//distribute, sublicense, and/or sell copies of the Software, and to -//permit persons to whom the Software is furnished to do so, subject to -//the following conditions: -// -//The above copyright notice and this permission notice shall be -//included in all copies or substantial portions of the Software. -// -//THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -//EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -//MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -//NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -//LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -//OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -//WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -/// Source code: https://github.com/mailru/sumbur-ruby/blob/master/ext/sumbur/sumbur.c - -unsigned int sumburConsistentHash(unsigned int hashed_int, unsigned int capacity); diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 126e4c7c57d..321aa5e2196 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -18,7 +18,6 @@ target_link_libraries(clickhouse_functions clickhouse_dictionaries_embedded clickhouse_parsers consistent-hashing - consistent-hashing-sumbur dbms metrohash murmurhash diff --git a/src/Functions/registerFunctionsConsistentHashing.cpp b/src/Functions/registerFunctionsConsistentHashing.cpp index ceec6dca5e6..d4d740bc92f 100644 --- a/src/Functions/registerFunctionsConsistentHashing.cpp +++ b/src/Functions/registerFunctionsConsistentHashing.cpp @@ -1,22 +1,14 @@ namespace DB - { class FunctionFactory; void registerFunctionYandexConsistentHash(FunctionFactory & factory); void registerFunctionJumpConsistentHash(FunctionFactory & factory); -#if !defined(ARCADIA_BUILD) -void registerFunctionSumburConsistentHash(FunctionFactory & factory); -#endif - void registerFunctionsConsistentHashing(FunctionFactory & factory) { registerFunctionYandexConsistentHash(factory); registerFunctionJumpConsistentHash(factory); -#if !defined(ARCADIA_BUILD) - registerFunctionSumburConsistentHash(factory); -#endif } } diff --git a/src/Functions/sumburConsistentHash.cpp b/src/Functions/sumburConsistentHash.cpp deleted file mode 100644 index 88de93f65d9..00000000000 --- a/src/Functions/sumburConsistentHash.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include "FunctionsConsistentHashing.h" -#include - -#include - - -namespace DB -{ -namespace -{ - -struct SumburConsistentHashImpl -{ - static constexpr auto name = "sumburConsistentHash"; - - using HashType = UInt32; - using ResultType = UInt16; - using BucketsType = ResultType; - static constexpr auto max_buckets = static_cast(std::numeric_limits::max()); - - static inline ResultType apply(HashType hash, BucketsType n) - { - return static_cast(sumburConsistentHash(hash, n)); - } -}; - -using FunctionSumburConsistentHash = FunctionConsistentHashImpl; - -} - -void registerFunctionSumburConsistentHash(FunctionFactory & factory) -{ - factory.registerFunction(); -} - -} - - diff --git a/src/Functions/ya.make.in b/src/Functions/ya.make.in index 9a646afc14b..5a3369c55a1 100644 --- a/src/Functions/ya.make.in +++ b/src/Functions/ya.make.in @@ -35,7 +35,7 @@ PEERDIR( # "Arcadia" build is slightly deficient. It lacks many libraries that we need. SRCS( - + ) END() diff --git a/utils/check-style/check-include b/utils/check-style/check-include index 6c7b3d02e6a..3c0c6103958 100755 --- a/utils/check-style/check-include +++ b/utils/check-style/check-include @@ -50,7 +50,6 @@ inc="-I. \ -I${BUILD_DIR}/base \ -I./base/daemon \ -I./base/consistent-hashing \ --I./base/consistent-hashing-sumbur \ -I./contrib/libhdfs3/include \ -I./contrib/base64/include \ -I./contrib/protobuf/src \ From 4c2295e69cf6adee7a7b361f9f3e3faefdad1cd9 Mon Sep 17 00:00:00 2001 From: feng lv Date: Wed, 30 Dec 2020 16:37:08 +0000 Subject: [PATCH 272/284] add query_id column to system.part_log update --- src/Interpreters/PartLog.cpp | 8 ++++++++ src/Interpreters/PartLog.h | 2 ++ 2 files changed, 10 insertions(+) diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index b8c3920c3e4..bf63b6f49c7 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -12,6 +12,7 @@ #include #include +#include namespace DB { @@ -32,6 +33,7 @@ Block PartLogElement::createBlock() return { + {ColumnString::create(), std::make_shared(), "query_id"}, {ColumnInt8::create(), std::move(event_type_datatype), "event_type"}, {ColumnUInt16::create(), std::make_shared(), "event_date"}, {ColumnUInt32::create(), std::make_shared(), "event_time"}, @@ -63,6 +65,7 @@ void PartLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; + columns[i++]->insert(query_id); columns[i++]->insert(event_type); columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); columns[i++]->insert(event_time); @@ -114,10 +117,15 @@ bool PartLog::addNewParts(Context & current_context, const PartLog::MutableDataP if (!part_log) return false; + auto query_id = CurrentThread::getQueryId(); + for (const auto & part : parts) { PartLogElement elem; + if (query_id.data && query_id.size) + elem.query_id.insert(0, query_id.data, query_id.size); + elem.event_type = PartLogElement::NEW_PART; elem.event_time = time(nullptr); elem.duration_ms = elapsed_ns / 1000000; diff --git a/src/Interpreters/PartLog.h b/src/Interpreters/PartLog.h index 9c8bf414099..a06d28f1f12 100644 --- a/src/Interpreters/PartLog.h +++ b/src/Interpreters/PartLog.h @@ -18,6 +18,8 @@ struct PartLogElement MOVE_PART = 6, }; + String query_id; + Type event_type = NEW_PART; time_t event_time = 0; From 17bfaa3e6c6679becc5aee814105603d09b9546b Mon Sep 17 00:00:00 2001 From: tavplubix Date: Thu, 31 Dec 2020 12:06:55 +0300 Subject: [PATCH 273/284] Update mysql.md --- docs/en/sql-reference/table-functions/mysql.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/table-functions/mysql.md b/docs/en/sql-reference/table-functions/mysql.md index b28afa4f2ef..eec4a1d0c46 100644 --- a/docs/en/sql-reference/table-functions/mysql.md +++ b/docs/en/sql-reference/table-functions/mysql.md @@ -81,8 +81,8 @@ SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123'); Replacing and inserting: ```sql -INSERT INTO FUNCTION mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 1) ('int_id', 'float') VALUES (1, 3); -INSERT INTO TABLE FUNCTION mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 0, 'UPDATE int_id = int_id + 1') ('int_id', 'float') VALUES (1, 4); +INSERT INTO FUNCTION mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 1) (int_id, float) VALUES (1, 3); +INSERT INTO TABLE FUNCTION mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 0, 'UPDATE int_id = int_id + 1') (int_id, float) VALUES (1, 4); SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123'); ``` From 970356f13c94f2458033d739f8c4200f7aba9619 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 31 Dec 2020 15:17:27 +0300 Subject: [PATCH 274/284] Update build.md --- docs/en/development/build.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 080011e683a..f98329e748f 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -42,7 +42,7 @@ $ export CXX=clang++-11 ### Install GCC 10 {#install-gcc-10} -We recommend building ClickHouse with clang-11, GCC-10 also supported, but it produces less optimal code. +We recommend building ClickHouse with clang-11, GCC-10 also supported, but it is not used for production builds. If you want to use GCC-10 there are several ways to install it. From f9d5d3dcd37f74505a0199f695eb5726d1ef1647 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 31 Dec 2020 20:17:51 +0400 Subject: [PATCH 275/284] Remove auto in function parameters --- src/Functions/FunctionBinaryArithmetic.h | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 848de344585..c10ff581b0e 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -416,8 +416,8 @@ private: DivideIntegralImpl, /// substitute divide by intDiv (throw on division by zero) Operation>; - template - static auto unwrap(const auto& elem, size_t i) + template + static auto unwrap(const E& elem, size_t i) { if constexpr (op_case == target) return undec(elem); @@ -744,8 +744,8 @@ class FunctionBinaryArithmetic : public IFunction return function->execute(new_arguments, result_type, input_rows_count); } - template - static auto helperGetOrConvert(const auto & col_const, const auto & col) + template + static auto helperGetOrConvert(const CC & col_const, const C & col) { using ResultType = typename ResultDataType::FieldType; using NativeResultType = typename NativeType::Type; @@ -756,8 +756,9 @@ class FunctionBinaryArithmetic : public IFunction return col_const->template getValue(); } - template - void helperInvokeEither(const auto& left, const auto& right, auto& vec_res, auto scale_a, auto scale_b) const + template + void helperInvokeEither(const L& left, const R& right, VR& vec_res, SA scale_a, SB scale_b) const { if (check_decimal_overflow) OpImplCheck::template process(left, right, vec_res, scale_a, scale_b); @@ -765,11 +766,12 @@ class FunctionBinaryArithmetic : public IFunction OpImpl::template process(left, right, vec_res, scale_a, scale_b); } - template + template ColumnPtr executeNumericWithDecimal( - const auto & left, const auto & right, + const L & left, const R & right, const ColumnConst * const col_left_const, const ColumnConst * const col_right_const, - const auto * const col_left, const auto * const col_right, + const CL * const col_left, const CR * const col_right, size_t col_left_size) const { using T0 = typename LeftDataType::FieldType; From ea4c25e7dd47a339b68225e81f7565e38bd68f1a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 31 Dec 2020 22:10:39 +0300 Subject: [PATCH 276/284] Do not ignore server memory limits during Buffer flush But ignore them during rollback, since it is better to account memory incorrectly them terminating the server. --- src/Storages/StorageBuffer.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 070d2b676d7..e24db51688e 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -384,11 +384,11 @@ static void appendBlock(const Block & from, Block & to) size_t old_rows = to.rows(); - MemoryTracker::BlockerInThread temporarily_disable_memory_tracker; - MutableColumnPtr last_col; try { + MemoryTracker::BlockerInThread temporarily_disable_memory_tracker(VariableContext::User); + for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no) { const IColumn & col_from = *from.getByPosition(column_no).column.get(); @@ -402,6 +402,11 @@ static void appendBlock(const Block & from, Block & to) catch (...) { /// Rollback changes. + + /// In case of rollback, it is better to ignore memory limits instead of abnormal server termination. + /// So ignore any memory limits, even global (since memory tracking has drift). + MemoryTracker::BlockerInThread temporarily_ignore_any_memory_limits(VariableContext::Global); + try { for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no) @@ -774,7 +779,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl } auto destination_metadata_snapshot = table->getInMemoryMetadataPtr(); - MemoryTracker::BlockerInThread temporarily_disable_memory_tracker; + MemoryTracker::BlockerInThread temporarily_disable_memory_tracker(VariableContext::User); auto insert = std::make_shared(); insert->table_id = destination_id; From c0fa2fb6e65bd98a3fc3db7833af10a5741d4dbd Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 31 Dec 2020 10:08:12 +0300 Subject: [PATCH 277/284] compressor: remove extra check for seeking of input --- programs/compressor/Compressor.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index d4b706abf4d..d47372631fe 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -169,11 +169,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) if (offset_in_compressed_file || offset_in_decompressed_block) { - if (!options.count("input")) - { - throw DB::Exception("--offset-in-compressed-file/--offset-in-decompressed-block requires --input", DB::ErrorCodes::BAD_ARGUMENTS); - } - CompressedReadBufferFromFile compressed_file(options["input"].as(), 0, 0, 0); + CompressedReadBufferFromFile compressed_file(std::move(rb)); compressed_file.seek(offset_in_compressed_file, offset_in_decompressed_block); copyData(compressed_file, *wb); } From 6a644b2af10699be5a5f14b8cfb80ef1bc88c31f Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 1 Jan 2021 12:42:22 +0800 Subject: [PATCH 278/284] Fix SimpleAggregateFunction in SummingMergeTree 2 --- .../Merges/Algorithms/SummingSortedAlgorithm.cpp | 4 ++-- ...regate_function_in_summing_merge_tree.reference | 1 + ...le_aggregate_function_in_summing_merge_tree.sql | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp index 61065aad5c3..749b460a934 100644 --- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp @@ -215,8 +215,9 @@ static SummingSortedAlgorithm::ColumnsDefinition defineColumns( { const ColumnWithTypeAndName & column = header.safeGetByPosition(i); + const auto * simple = dynamic_cast(column.type->getCustomName()); /// Discover nested Maps and find columns for summation - if (typeid_cast(column.type.get())) + if (typeid_cast(column.type.get()) && !simple) { const auto map_name = Nested::extractTableName(column.name); /// if nested table name ends with `Map` it is a possible candidate for special handling @@ -231,7 +232,6 @@ static SummingSortedAlgorithm::ColumnsDefinition defineColumns( else { bool is_agg_func = WhichDataType(column.type).isAggregateFunction(); - const auto * simple = dynamic_cast(column.type->getCustomName()); /// There are special const columns for example after prewhere sections. if ((!column.type->isSummable() && !is_agg_func && !simple) || isColumnConst(*column.column)) diff --git a/tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.reference b/tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.reference index dbc5d8aae43..14f997de123 100644 --- a/tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.reference +++ b/tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.reference @@ -1 +1,2 @@ ([25],[1]) ([25],[1]) +[0,1] [0,1] diff --git a/tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.sql b/tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.sql index 764cebbc9a9..498c5d93a89 100644 --- a/tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.sql +++ b/tests/queries/0_stateless/01630_simple_aggregate_function_in_summing_merge_tree.sql @@ -7,3 +7,17 @@ insert into test_smt select id, sumMap(k), sumMapState(k) from (select 2 as id, select sumMap(sMap), sumMapMerge(aMap) from test_smt; drop table if exists test_smt; + +drop table if exists simple_agf_summing_mt; + +create table simple_agf_summing_mt (a Int64, grp_aggreg AggregateFunction(groupUniqArrayArray, Array(UInt64)), grp_simple SimpleAggregateFunction(groupUniqArrayArray, Array(UInt64))) engine = SummingMergeTree() order by a; + +insert into simple_agf_summing_mt select 1 a, groupUniqArrayArrayState([toUInt64(number)]), groupUniqArrayArray([toUInt64(number)]) from numbers(1) group by a; + +insert into simple_agf_summing_mt select 1 a, groupUniqArrayArrayState([toUInt64(number)]), groupUniqArrayArray([toUInt64(number)]) from numbers(2) group by a; + +optimize table simple_agf_summing_mt final; + +SELECT arraySort(groupUniqArrayArrayMerge(grp_aggreg)) gra , arraySort(groupUniqArrayArray(grp_simple)) grs FROM simple_agf_summing_mt group by a; + +drop table if exists simple_agf_summing_mt; From 829e82c27c14beae342de9550f3849a8e26e39c2 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 1 Jan 2021 14:36:42 +0800 Subject: [PATCH 279/284] better linker name matcher --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 92d2aa81db3..556d5d84ad6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -195,13 +195,13 @@ endif () option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker.") if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") - if (LINKER_NAME STREQUAL "lld") + if (LINKER_NAME MATCHES "lld$") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index") set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gdb-index") message (STATUS "Adding .gdb-index via --gdb-index linker option.") # we use another tool for gdb-index, because gold linker removes section .debug_aranges, which used inside clickhouse stacktraces # http://sourceware-org.1504.n7.nabble.com/gold-No-debug-aranges-section-when-linking-with-gdb-index-td540965.html#a556932 - elseif (LINKER_NAME STREQUAL "gold" AND ADD_GDB_INDEX_FOR_GOLD) + elseif (LINKER_NAME MATCHES "gold$" AND ADD_GDB_INDEX_FOR_GOLD) find_program (GDB_ADD_INDEX_EXE NAMES "gdb-add-index" DOC "Path to gdb-add-index executable") if (NOT GDB_ADD_INDEX_EXE) set (USE_GDB_ADD_INDEX 0) From 56bbdcccccb524036f2c46a569dc4e8ead987d84 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 1 Jan 2021 18:49:09 +0300 Subject: [PATCH 280/284] More robust docs release --- docs/tools/release.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/tools/release.sh b/docs/tools/release.sh index 389b63ace7f..035581534f1 100755 --- a/docs/tools/release.sh +++ b/docs/tools/release.sh @@ -31,9 +31,11 @@ then git add * git add ".nojekyll" - # Push to GitHub rewriting the existing contents. git commit --quiet -m "Add new release at $(date)" - git push --force origin master + + # Push to GitHub rewriting the existing contents. + # Sometimes it does not work with error message "! [remote rejected] master -> master (cannot lock ref 'refs/heads/master': is at 42a0f6b6b6c7be56a469441b4bf29685c1cebac3 but expected 520e9b02c0d4678a2a5f41d2f561e6532fb98cc1)" + for _ in {1..10}; do git push --force origin master && break; sleep 5; done if [[ ! -z "${CLOUDFLARE_TOKEN}" ]] then From 17009ccb9837e0772c267b504f60f34d5b40a20b Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 1 Jan 2021 18:52:02 +0300 Subject: [PATCH 281/284] Update CMakeLists.txt --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 556d5d84ad6..7716fe82677 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -195,6 +195,7 @@ endif () option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker.") if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") + # Can be lld or ld-lld. if (LINKER_NAME MATCHES "lld$") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index") set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gdb-index") From aff724ea7d371d322589fba96a1214aacab78e59 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 2 Jan 2021 17:07:54 +0300 Subject: [PATCH 282/284] PODArray: Avoid call to memcpy with (nullptr, 0) arguments (#18526) * Avoid call to memcpy with nullptr, 0 arguments * Add assert to PODArray * Fix build * Fix build * Minor fixes for min/sim hash * Fix build * Fix build * Fix build * Fix error * Fix "not actually an error" * Fix build * Fix build * Fix build --- .../AggregateFunctionGroupArray.h | 4 +- src/Common/PODArray.h | 66 ++++++++++++++++--- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.h b/src/AggregateFunctions/AggregateFunctionGroupArray.h index 3255ea42edb..e8c4d70ea26 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArray.h +++ b/src/AggregateFunctions/AggregateFunctionGroupArray.h @@ -188,13 +188,13 @@ public: if (!limit_num_elems) { if (rhs_elems.value.size()) - cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.end(), arena); + cur_elems.value.insertByOffsets(rhs_elems.value, 0, rhs_elems.value.size(), arena); } else { UInt64 elems_to_insert = std::min(static_cast(max_elems) - cur_elems.value.size(), rhs_elems.value.size()); if (elems_to_insert) - cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.begin() + elems_to_insert, arena); + cur_elems.value.insertByOffsets(rhs_elems.value, 0, elems_to_insert, arena); } } diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 7bd9550500e..344f7bfcbe1 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -89,8 +89,8 @@ protected: static constexpr size_t pad_right = integerRoundUp(pad_right_, ELEMENT_SIZE); /// pad_left is also rounded up to 16 bytes to maintain alignment of allocated memory. static constexpr size_t pad_left = integerRoundUp(integerRoundUp(pad_left_, ELEMENT_SIZE), 16); - /// Empty array will point to this static memory as padding. - static constexpr char * null = pad_left ? const_cast(empty_pod_array) + empty_pod_array_size : nullptr; + /// Empty array will point to this static memory as padding and begin/end. + static constexpr char * null = const_cast(empty_pod_array) + pad_left; static_assert(pad_left <= empty_pod_array_size && "Left Padding exceeds empty_pod_array_size. Is the element size too large?"); @@ -268,8 +268,11 @@ public: reserve(required_capacity, std::forward(allocator_params)...); size_t items_byte_size = byte_size(number_of_items); - memcpy(c_end, ptr, items_byte_size); - c_end += items_byte_size; + if (items_byte_size) + { + memcpy(c_end, ptr, items_byte_size); + c_end += items_byte_size; + } } void protect() @@ -289,6 +292,18 @@ public: #endif } + template + inline void assertNotIntersects(It1 from_begin [[maybe_unused]], It2 from_end [[maybe_unused]]) + { +#if !defined(NDEBUG) + const char * ptr_begin = reinterpret_cast(&*from_begin); + const char * ptr_end = reinterpret_cast(&*from_end); + + /// Also it's safe if the range is empty. + assert(!((ptr_begin >= c_start && ptr_begin <= c_end) || (ptr_end >= c_start && ptr_end <= c_end)) || (ptr_begin == ptr_end)); +#endif + } + ~PODArrayBase() { dealloc(); @@ -444,6 +459,7 @@ public: template void insertPrepare(It1 from_begin, It2 from_end, TAllocatorParams &&... allocator_params) { + this->assertNotIntersects(from_begin, from_end); size_t required_capacity = this->size() + (from_end - from_begin); if (required_capacity > this->capacity()) this->reserve(roundUpToPowerOfTwoOrZero(required_capacity), std::forward(allocator_params)...); @@ -457,6 +473,28 @@ public: insert_assume_reserved(from_begin, from_end); } + /// In contrast to 'insert' this method is Ok even for inserting from itself. + /// Because we obtain iterators after reserving memory. + template + void insertByOffsets(Container && rhs, size_t from_begin, size_t from_end, TAllocatorParams &&... allocator_params) + { + static_assert(memcpy_can_be_used_for_assignment, std::decay_t>); + + assert(from_end >= from_begin); + assert(from_end <= rhs.size()); + + size_t required_capacity = this->size() + (from_end - from_begin); + if (required_capacity > this->capacity()) + this->reserve(roundUpToPowerOfTwoOrZero(required_capacity), std::forward(allocator_params)...); + + size_t bytes_to_copy = this->byte_size(from_end - from_begin); + if (bytes_to_copy) + { + memcpy(this->c_end, reinterpret_cast(rhs.begin() + from_begin), bytes_to_copy); + this->c_end += bytes_to_copy; + } + } + /// Works under assumption, that it's possible to read up to 15 excessive bytes after `from_end` and this PODArray is padded. template void insertSmallAllowReadWriteOverflow15(It1 from_begin, It2 from_end, TAllocatorParams &&... allocator_params) @@ -476,6 +514,9 @@ public: static_assert(memcpy_can_be_used_for_assignment, std::decay_t>); size_t bytes_to_copy = this->byte_size(from_end - from_begin); + if (!bytes_to_copy) + return; + size_t bytes_to_move = this->byte_size(end() - it); insertPrepare(from_begin, from_end); @@ -492,10 +533,14 @@ public: void insert_assume_reserved(It1 from_begin, It2 from_end) { static_assert(memcpy_can_be_used_for_assignment, std::decay_t>); + this->assertNotIntersects(from_begin, from_end); size_t bytes_to_copy = this->byte_size(from_end - from_begin); - memcpy(this->c_end, reinterpret_cast(&*from_begin), bytes_to_copy); - this->c_end += bytes_to_copy; + if (bytes_to_copy) + { + memcpy(this->c_end, reinterpret_cast(&*from_begin), bytes_to_copy); + this->c_end += bytes_to_copy; + } } template @@ -626,15 +671,18 @@ public: void assign(It1 from_begin, It2 from_end, TAllocatorParams &&... allocator_params) { static_assert(memcpy_can_be_used_for_assignment, std::decay_t>); + this->assertNotIntersects(from_begin, from_end); size_t required_capacity = from_end - from_begin; if (required_capacity > this->capacity()) this->reserve_exact(required_capacity, std::forward(allocator_params)...); size_t bytes_to_copy = this->byte_size(required_capacity); - memcpy(this->c_start, reinterpret_cast(&*from_begin), bytes_to_copy); - - this->c_end = this->c_start + bytes_to_copy; + if (bytes_to_copy) + { + memcpy(this->c_start, reinterpret_cast(&*from_begin), bytes_to_copy); + this->c_end = this->c_start + bytes_to_copy; + } } // ISO C++ has strict ambiguity rules, thus we cannot apply TAllocatorParams here. From 469b631ca6af5a998db5cce190f596ab4ad7f37e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 2 Jan 2021 17:09:20 +0300 Subject: [PATCH 283/284] Fix performance comparison In #18113 the top_level_domains list copying was moved into docker/packager/binary/build.sh, this was done to avoid symlinks (since Dockerfile cannot dereference them). But the patch was wrong, since it copied into the root (/), which is not included into performance.tgz and also compare.sh was not modified. This wasn't showed up with CI checks since the docker image wasn't updated and it still included that top_level_domains, once it was modified the image was updated and it became broken. Cc: @akuzm --- docker/packager/binary/build.sh | 2 +- docker/test/performance-comparison/compare.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index c46e3b23994..85e2e8b0f04 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -31,7 +31,7 @@ find . -name '*.so.*' -print -exec mv '{}' /output \; if [ "performance" == "$COMBINED_OUTPUT" ] then cp -r ../tests/performance /output - cp -r ../tests/config/top_level_domains / + cp -r ../tests/config/top_level_domains /output cp -r ../docker/test/performance-comparison/config /output ||: rm /output/unit_tests_dbms ||: rm /output/clickhouse-odbc-bridge ||: diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 59d7cc98063..a5d6a136880 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -55,7 +55,7 @@ function configure # server *config* directives overrides --path db0 --user_files_path db0/user_files - --top_level_domains_path /top_level_domains + --top_level_domains_path right/top_level_domains --tcp_port $LEFT_SERVER_PORT ) left/clickhouse-server "${setup_left_server_opts[@]}" &> setup-server-log.log & @@ -103,7 +103,7 @@ function restart # server *config* directives overrides --path left/db --user_files_path left/db/user_files - --top_level_domains_path /top_level_domains + --top_level_domains_path left/top_level_domains --tcp_port $LEFT_SERVER_PORT ) left/clickhouse-server "${left_server_opts[@]}" &>> left-server-log.log & @@ -118,7 +118,7 @@ function restart # server *config* directives overrides --path right/db --user_files_path right/db/user_files - --top_level_domains_path /top_level_domains + --top_level_domains_path right/top_level_domains --tcp_port $RIGHT_SERVER_PORT ) right/clickhouse-server "${right_server_opts[@]}" &>> right-server-log.log & From de7909211d67d94f35a151e022b87561ceaeb4fb Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 2 Jan 2021 17:27:03 +0300 Subject: [PATCH 284/284] Try using top_level_domains from the patched performance archive If there is top_level_domains list in the upstream/master, use from the patched version (this is required to run all performance tests for upstream/master in the PR). --- docker/test/performance-comparison/compare.sh | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index a5d6a136880..9a0d8093a55 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -36,6 +36,22 @@ function wait_for_server # port, pid fi } +function left_or_right() +{ + local from=$1 && shift + local basename=$1 && shift + + if [ -e "$from/$basename" ]; then + echo "$from/$basename" + return + fi + + case "$from" in + left) echo "right/$basename" ;; + right) echo "left/$basename" ;; + esac +} + function configure { # Use the new config for both servers, so that we can change it in a PR. @@ -55,7 +71,7 @@ function configure # server *config* directives overrides --path db0 --user_files_path db0/user_files - --top_level_domains_path right/top_level_domains + --top_level_domains_path "$(left_or_right right top_level_domains)" --tcp_port $LEFT_SERVER_PORT ) left/clickhouse-server "${setup_left_server_opts[@]}" &> setup-server-log.log & @@ -103,7 +119,7 @@ function restart # server *config* directives overrides --path left/db --user_files_path left/db/user_files - --top_level_domains_path left/top_level_domains + --top_level_domains_path "$(left_or_right left top_level_domains)" --tcp_port $LEFT_SERVER_PORT ) left/clickhouse-server "${left_server_opts[@]}" &>> left-server-log.log & @@ -118,7 +134,7 @@ function restart # server *config* directives overrides --path right/db --user_files_path right/db/user_files - --top_level_domains_path right/top_level_domains + --top_level_domains_path "$(left_or_right right top_level_domains)" --tcp_port $RIGHT_SERVER_PORT ) right/clickhouse-server "${right_server_opts[@]}" &>> right-server-log.log &