diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 064787fb64e..0c7465ec7a5 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -322,8 +322,14 @@ public: if (offset_is_whole_number_of_hours_everytime) return (UInt32(t) / 60) % 60; - UInt32 date = find(t).date; - return (UInt32(t) - date) / 60 % 60; + /// To consider the DST changing situation within this day. + /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account + DayNum index = findIndex(t); + UInt32 res = t - lut[index].date; + if (lut[index].amount_of_offset_change != 0 && t >= lut[index].date + lut[index].time_at_offset_change) + res += lut[index].amount_of_offset_change; + + return res / 60 % 60; } inline time_t toStartOfMinute(time_t t) const { return t / 60 * 60; } diff --git a/base/mysqlxx/Exception.h b/base/mysqlxx/Exception.h index eaeb3565af1..48cd0997b94 100644 --- a/base/mysqlxx/Exception.h +++ b/base/mysqlxx/Exception.h @@ -26,6 +26,15 @@ struct ConnectionFailed : public Exception }; +/// Connection to MySQL server was lost +struct ConnectionLost : public Exception +{ + ConnectionLost(const std::string & msg, int code = 0) : Exception(msg, code) {} + const char * name() const throw() override { return "mysqlxx::ConnectionLost"; } + const char * className() const throw() override { return "mysqlxx::ConnectionLost"; } +}; + + /// Erroneous query. struct BadQuery : public Exception { diff --git a/base/mysqlxx/Pool.cpp b/base/mysqlxx/Pool.cpp index b524087836d..cf8b3cf9267 100644 --- a/base/mysqlxx/Pool.cpp +++ b/base/mysqlxx/Pool.cpp @@ -10,7 +10,6 @@ #include -#include #include @@ -41,7 +40,9 @@ void Pool::Entry::decrementRefCount() Pool::Pool(const Poco::Util::AbstractConfiguration & cfg, const std::string & config_name, unsigned default_connections_, unsigned max_connections_, const char * parent_config_name_) - : default_connections(default_connections_), max_connections(max_connections_) + : logger(Poco::Logger::get("mysqlxx::Pool")) + , default_connections(default_connections_) + , max_connections(max_connections_) { server = cfg.getString(config_name + ".host"); @@ -130,20 +131,30 @@ Pool::Entry Pool::get() initialize(); for (;;) { + logger.trace("(%s): Iterating through existing MySQL connections", getDescription()); + for (auto & connection : connections) { if (connection->ref_count == 0) return Entry(connection, this); } + logger.trace("(%s): Trying to allocate a new connection.", getDescription()); if (connections.size() < static_cast(max_connections)) { Connection * conn = allocConnection(); if (conn) return Entry(conn, this); + + logger.trace("(%s): Unable to create a new connection: Allocation failed.", getDescription()); + } + else + { + logger.trace("(%s): Unable to create a new connection: Max number of connections has been reached.", getDescription()); } lock.unlock(); + logger.trace("(%s): Sleeping for %d seconds.", getDescription(), MYSQLXX_POOL_SLEEP_ON_CONNECT_FAIL); sleepForSeconds(MYSQLXX_POOL_SLEEP_ON_CONNECT_FAIL); lock.lock(); } @@ -167,8 +178,7 @@ Pool::Entry Pool::tryGet() if (res.tryForceConnected()) /// Tries to reestablish connection as well return res; - auto & logger = Poco::Util::Application::instance().logger(); - logger.information("Idle connection to mysql server cannot be recovered, dropping it."); + logger.debug("(%s): Idle connection to MySQL server cannot be recovered, dropping it.", getDescription()); /// This one is disconnected, cannot be reestablished and so needs to be disposed of. connection_it = connections.erase(connection_it); @@ -191,6 +201,8 @@ Pool::Entry Pool::tryGet() void Pool::removeConnection(Connection* connection) { + logger.trace("(%s): Removing connection.", getDescription()); + std::lock_guard lock(mutex); if (connection) { @@ -215,8 +227,6 @@ void Pool::Entry::forceConnected() const if (data == nullptr) throw Poco::RuntimeException("Tried to access NULL database connection."); - Poco::Util::Application & app = Poco::Util::Application::instance(); - bool first = true; while (!tryForceConnected()) { @@ -225,7 +235,7 @@ void Pool::Entry::forceConnected() const else sleepForSeconds(MYSQLXX_POOL_SLEEP_ON_CONNECT_FAIL); - app.logger().information("MYSQL: Reconnecting to " + pool->description); + pool->logger.debug("Entry: Reconnecting to MySQL server %s", pool->description); data->conn.connect( pool->db.c_str(), pool->server.c_str(), @@ -248,18 +258,22 @@ bool Pool::Entry::tryForceConnected() const { auto * const mysql_driver = data->conn.getDriver(); const auto prev_connection_id = mysql_thread_id(mysql_driver); + + pool->logger.trace("Entry(connection %lu): sending PING to check if it is alive.", prev_connection_id); if (data->conn.ping()) /// Attempts to reestablish lost connection { const auto current_connection_id = mysql_thread_id(mysql_driver); if (prev_connection_id != current_connection_id) { - auto & logger = Poco::Util::Application::instance().logger(); - logger.information("Reconnected to mysql server. Connection id changed: %lu -> %lu", - prev_connection_id, current_connection_id); + pool->logger.debug("Entry(connection %lu): Reconnected to MySQL server. Connection id changed: %lu -> %lu", + current_connection_id, prev_connection_id, current_connection_id); } + + pool->logger.trace("Entry(connection %lu): PING ok.", current_connection_id); return true; } + pool->logger.trace("Entry(connection %lu): PING failed.", prev_connection_id); return false; } @@ -280,15 +294,13 @@ void Pool::initialize() Pool::Connection * Pool::allocConnection(bool dont_throw_if_failed_first_time) { - Poco::Util::Application & app = Poco::Util::Application::instance(); - - std::unique_ptr conn(new Connection); + std::unique_ptr conn_ptr{new Connection}; try { - app.logger().information("MYSQL: Connecting to " + description); + logger.debug("Connecting to %s", description); - conn->conn.connect( + conn_ptr->conn.connect( db.c_str(), server.c_str(), user.c_str(), @@ -305,25 +317,24 @@ Pool::Connection * Pool::allocConnection(bool dont_throw_if_failed_first_time) } catch (mysqlxx::ConnectionFailed & e) { + logger.error(e.what()); + if ((!was_successful && !dont_throw_if_failed_first_time) || e.errnum() == ER_ACCESS_DENIED_ERROR || e.errnum() == ER_DBACCESS_DENIED_ERROR || e.errnum() == ER_BAD_DB_ERROR) { - app.logger().error(e.what()); throw; } else { - app.logger().error(e.what()); return nullptr; } } + connections.push_back(conn_ptr.get()); was_successful = true; - auto * connection = conn.release(); - connections.push_back(connection); - return connection; + return conn_ptr.release(); } } diff --git a/base/mysqlxx/Pool.h b/base/mysqlxx/Pool.h index 0f64e58054c..b6189663f55 100644 --- a/base/mysqlxx/Pool.h +++ b/base/mysqlxx/Pool.h @@ -6,6 +6,8 @@ #include #include +#include + #include @@ -167,13 +169,13 @@ public: unsigned max_connections_ = MYSQLXX_POOL_DEFAULT_MAX_CONNECTIONS, unsigned enable_local_infile_ = MYSQLXX_DEFAULT_ENABLE_LOCAL_INFILE, bool opt_reconnect_ = MYSQLXX_DEFAULT_MYSQL_OPT_RECONNECT) - : default_connections(default_connections_), max_connections(max_connections_), - db(db_), server(server_), user(user_), password(password_), port(port_), socket(socket_), + : logger(Poco::Logger::get("mysqlxx::Pool")), default_connections(default_connections_), + max_connections(max_connections_), db(db_), server(server_), user(user_), password(password_), port(port_), socket(socket_), connect_timeout(connect_timeout_), rw_timeout(rw_timeout_), enable_local_infile(enable_local_infile_), opt_reconnect(opt_reconnect_) {} Pool(const Pool & other) - : default_connections{other.default_connections}, + : logger(other.logger), default_connections{other.default_connections}, max_connections{other.max_connections}, db{other.db}, server{other.server}, user{other.user}, password{other.password}, @@ -203,6 +205,8 @@ public: void removeConnection(Connection * connection); protected: + Poco::Logger & logger; + /// Number of MySQL connections which are created at launch. unsigned default_connections; /// Maximum possible number of connections diff --git a/base/mysqlxx/Query.cpp b/base/mysqlxx/Query.cpp index f3485c54edc..d4514c3e988 100644 --- a/base/mysqlxx/Query.cpp +++ b/base/mysqlxx/Query.cpp @@ -1,11 +1,16 @@ #if __has_include() +#include #include #else +#include #include #endif +#include + #include #include +#include namespace mysqlxx @@ -57,8 +62,24 @@ void Query::reset() void Query::executeImpl() { std::string query_string = query_buf.str(); - if (mysql_real_query(conn->getDriver(), query_string.data(), query_string.size())) - throw BadQuery(errorMessage(conn->getDriver()), mysql_errno(conn->getDriver())); + + MYSQL* mysql_driver = conn->getDriver(); + + auto & logger = Poco::Logger::get("mysqlxx::Query"); + logger.trace("Running MySQL query using connection %lu", mysql_thread_id(mysql_driver)); + if (mysql_real_query(mysql_driver, query_string.data(), query_string.size())) + { + const auto err_no = mysql_errno(mysql_driver); + switch (err_no) + { + case CR_SERVER_GONE_ERROR: + [[fallthrough]]; + case CR_SERVER_LOST: + throw ConnectionLost(errorMessage(mysql_driver), err_no); + default: + throw BadQuery(errorMessage(mysql_driver), err_no); + } + } } UseQueryResult Query::use() diff --git a/contrib/brotli b/contrib/brotli index 5805f99a533..63be8a99401 160000 --- a/contrib/brotli +++ b/contrib/brotli @@ -1 +1 @@ -Subproject commit 5805f99a533a8f8118699c0100d8c102f3605f65 +Subproject commit 63be8a99401992075c23e99f7c84de1c653e39e2 diff --git a/contrib/brotli-cmake/CMakeLists.txt b/contrib/brotli-cmake/CMakeLists.txt index e22f4593c02..4c5f584de9d 100644 --- a/contrib/brotli-cmake/CMakeLists.txt +++ b/contrib/brotli-cmake/CMakeLists.txt @@ -2,6 +2,8 @@ set(BROTLI_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/brotli/c) set(BROTLI_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/brotli/c) set(SRCS + ${BROTLI_SOURCE_DIR}/enc/command.c + ${BROTLI_SOURCE_DIR}/enc/fast_log.c ${BROTLI_SOURCE_DIR}/dec/bit_reader.c ${BROTLI_SOURCE_DIR}/dec/state.c ${BROTLI_SOURCE_DIR}/dec/huffman.c @@ -26,6 +28,9 @@ set(SRCS ${BROTLI_SOURCE_DIR}/enc/memory.c ${BROTLI_SOURCE_DIR}/common/dictionary.c ${BROTLI_SOURCE_DIR}/common/transform.c + ${BROTLI_SOURCE_DIR}/common/platform.c + ${BROTLI_SOURCE_DIR}/common/context.c + ${BROTLI_SOURCE_DIR}/common/constants.c ) add_library(brotli ${SRCS}) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 370311b13c5..1bfc91ecd92 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -356,7 +356,6 @@ function run_tests # JSON functions 01666_blns - 01674_htm_xml_coarse_parse ) (time clickhouse-test --hung-check -j 8 --order=random --use-skip-list --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" -- "$FASTTEST_FOCUS" 2>&1 ||:) | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt" diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index 4963ff0094d..938d8d45ffd 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -18,7 +18,8 @@ RUN apt-get update \ curl \ tar \ krb5-user \ - iproute2 + iproute2 \ + lsof RUN rm -rf \ /var/lib/apt/lists/* \ /var/cache/debconf \ diff --git a/docker/test/integration/runner/compose/docker_compose_zookeeper.yml b/docker/test/integration/runner/compose/docker_compose_zookeeper.yml index 49e285b5515..1601d217a25 100644 --- a/docker/test/integration/runner/compose/docker_compose_zookeeper.yml +++ b/docker/test/integration/runner/compose/docker_compose_zookeeper.yml @@ -1,11 +1,11 @@ version: '2.3' services: zoo1: - image: zookeeper:3.4.12 + image: zookeeper:3.6.2 restart: always environment: ZOO_TICK_TIME: 500 - ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888 + ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 ZOO_MY_ID: 1 JVMFLAGS: -Dzookeeper.forceSync=no volumes: @@ -16,11 +16,11 @@ services: source: ${ZK_DATA_LOG1:-} target: /datalog zoo2: - image: zookeeper:3.4.12 + image: zookeeper:3.6.2 restart: always environment: ZOO_TICK_TIME: 500 - ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888 + ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888 ZOO_MY_ID: 2 JVMFLAGS: -Dzookeeper.forceSync=no volumes: @@ -31,11 +31,11 @@ services: source: ${ZK_DATA_LOG2:-} target: /datalog zoo3: - image: zookeeper:3.4.12 + image: zookeeper:3.6.2 restart: always environment: ZOO_TICK_TIME: 500 - ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888 + ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 ZOO_MY_ID: 3 JVMFLAGS: -Dzookeeper.forceSync=no volumes: diff --git a/docs/_description_templates/template-data-type.md b/docs/_description_templates/template-data-type.md index edb6586ee7d..5e560b9325d 100644 --- a/docs/_description_templates/template-data-type.md +++ b/docs/_description_templates/template-data-type.md @@ -26,4 +26,4 @@ The name of an additional section can be any, for example, **Usage**. - [link](#) -[Original article](https://clickhouse.tech/docs/en/data_types//) +[Original article](https://clickhouse.tech/docs/en/data-types//) diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index c519d6bb136..fb1df62bb15 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -38,20 +38,20 @@ SETTINGS Required parameters: -- `kafka_broker_list` – A comma-separated list of brokers (for example, `localhost:9092`). -- `kafka_topic_list` – A list of Kafka topics. -- `kafka_group_name` – A group of Kafka consumers. Reading margins are tracked for each group separately. If you don’t want messages to be duplicated in the cluster, use the same group name everywhere. -- `kafka_format` – Message format. Uses the same notation as the SQL `FORMAT` function, such as `JSONEachRow`. For more information, see the [Formats](../../../interfaces/formats.md) section. +- `kafka_broker_list` — A comma-separated list of brokers (for example, `localhost:9092`). +- `kafka_topic_list` — A list of Kafka topics. +- `kafka_group_name` — A group of Kafka consumers. Reading margins are tracked for each group separately. If you don’t want messages to be duplicated in the cluster, use the same group name everywhere. +- `kafka_format` — Message format. Uses the same notation as the SQL `FORMAT` function, such as `JSONEachRow`. For more information, see the [Formats](../../../interfaces/formats.md) section. Optional parameters: -- `kafka_row_delimiter` – Delimiter character, which ends the message. -- `kafka_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap’n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root `schema.capnp:Message` object. -- `kafka_num_consumers` – The number of consumers per table. Default: `1`. Specify more consumers if the throughput of one consumer is insufficient. The total number of consumers should not exceed the number of partitions in the topic, since only one consumer can be assigned per partition. -- `kafka_max_block_size` - The maximum batch size (in messages) for poll (default: `max_block_size`). -- `kafka_skip_broken_messages` – Kafka message parser tolerance to schema-incompatible messages per block. Default: `0`. If `kafka_skip_broken_messages = N` then the engine skips *N* Kafka messages that cannot be parsed (a message equals a row of data). -- `kafka_commit_every_batch` - Commit every consumed and handled batch instead of a single commit after writing a whole block (default: `0`). -- `kafka_thread_per_consumer` - Provide independent thread for each consumer (default: `0`). When enabled, every consumer flush the data independently, in parallel (otherwise - rows from several consumers squashed to form one block). +- `kafka_row_delimiter` — Delimiter character, which ends the message. +- `kafka_schema` — Parameter that must be used if the format requires a schema definition. For example, [Cap’n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root `schema.capnp:Message` object. +- `kafka_num_consumers` — The number of consumers per table. Default: `1`. Specify more consumers if the throughput of one consumer is insufficient. The total number of consumers should not exceed the number of partitions in the topic, since only one consumer can be assigned per partition. +- `kafka_max_block_size` — The maximum batch size (in messages) for poll (default: `max_block_size`). +- `kafka_skip_broken_messages` — Kafka message parser tolerance to schema-incompatible messages per block. Default: `0`. If `kafka_skip_broken_messages = N` then the engine skips *N* Kafka messages that cannot be parsed (a message equals a row of data). +- `kafka_commit_every_batch` — Commit every consumed and handled batch instead of a single commit after writing a whole block (default: `0`). +- `kafka_thread_per_consumer` — Provide independent thread for each consumer (default: `0`). When enabled, every consumer flush the data independently, in parallel (otherwise — rows from several consumers squashed to form one block). Examples: diff --git a/docs/en/getting-started/example-datasets/cell-towers.md b/docs/en/getting-started/example-datasets/cell-towers.md new file mode 100644 index 00000000000..76effdd4c62 --- /dev/null +++ b/docs/en/getting-started/example-datasets/cell-towers.md @@ -0,0 +1,133 @@ +--- +toc_priority: 21 +toc_title: Cell Towers +--- + +# Cell Towers + +This dataset is from [OpenCellid](https://www.opencellid.org/) - The world's largest Open Database of Cell Towers. + +As of 2021 it contains more than 40 million records about cell towers (GSM, LTE, UMTS, etc.) around the world with their geographical coordinates and metadata (country code, network, etc). + +OpenCelliD Project is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License, and we redistribute a snapshot of this dataset under the terms of the same license. The up to date version of the dataset is available to download after sign in. + + +## Get the Dataset + +Download the snapshot of the dataset from Feb 2021: [https://datasets.clickhouse.tech/cell_towers.csv.xz] (729 MB). + +Optionally validate the integrity: +``` +md5sum cell_towers.csv.xz +8cf986f4a0d9f12c6f384a0e9192c908 cell_towers.csv.xz +``` + +Decompress it with the following command: +``` +xz -d cell_towers.csv.xz +``` + +Create a table: + +``` +CREATE TABLE cell_towers +( + radio Enum8('' = 0, 'CDMA' = 1, 'GSM' = 2, 'LTE' = 3, 'NR' = 4, 'UMTS' = 5), + mcc UInt16, + net UInt16, + area UInt16, + cell UInt64, + unit Int16, + lon Float64, + lat Float64, + range UInt32, + samples UInt32, + changeable UInt8, + created DateTime, + updated DateTime, + averageSignal UInt8 +) +ENGINE = MergeTree ORDER BY (radio, mcc, net, created); +``` + +Insert the dataset: +``` +clickhouse-client --query "INSERT INTO cell_towers FORMAT CSVWithNames" < cell_towers.csv +``` + + +## Run some queries + +Number of cell towers by type: +``` +SELECT radio, count() AS c FROM cell_towers GROUP BY radio ORDER BY c DESC + +┌─radio─┬────────c─┐ +│ UMTS │ 20686487 │ +│ LTE │ 12101148 │ +│ GSM │ 9931312 │ +│ CDMA │ 556344 │ +│ NR │ 867 │ +└───────┴──────────┘ + +5 rows in set. Elapsed: 0.011 sec. Processed 43.28 million rows, 43.28 MB (3.83 billion rows/s., 3.83 GB/s.) +``` + +Cell towers by mobile country code (MCC): +``` +SELECT mcc, count() FROM cell_towers GROUP BY mcc ORDER BY count() DESC LIMIT 10 + +┌─mcc─┬─count()─┐ +│ 310 │ 5024650 │ +│ 262 │ 2622423 │ +│ 250 │ 1953176 │ +│ 208 │ 1891187 │ +│ 724 │ 1836150 │ +│ 404 │ 1729151 │ +│ 234 │ 1618924 │ +│ 510 │ 1353998 │ +│ 440 │ 1343355 │ +│ 311 │ 1332798 │ +└─────┴─────────┘ + +10 rows in set. Elapsed: 0.019 sec. Processed 43.28 million rows, 86.55 MB (2.33 billion rows/s., 4.65 GB/s.) +``` + +See the dictionary here: [https://en.wikipedia.org/wiki/Mobile_country_code](https://en.wikipedia.org/wiki/Mobile_country_code). + +So, the top countries are USA, Germany and Russia. + +You may want to create an [External Dictionary](../../sql-reference/dictionaries/external-dictionaries/external-dicts/) in ClickHouse to decode these values. + + +### Example of using `pointInPolygon` function + +Create a table where we will store polygons: + +``` +CREATE TEMPORARY TABLE moscow (polygon Array(Tuple(Float64, Float64))); +``` + +This is a rough shape of Moscow (without "new Moscow"): + +``` +INSERT INTO moscow VALUES ([(37.84172564285271, 55.78000432402266), (37.8381207618713, 55.775874525970494), (37.83979446823122, 55.775626746008065), (37.84243326983639, 55.77446586811748), (37.84262672750849, 55.771974101091104), (37.84153238623039, 55.77114545193181), (37.841124690460184, 55.76722010265554), (37.84239076983644, 55.76654891107098), (37.842283558197025, 55.76258709833121), (37.8421759312134, 55.758073999993734), (37.84198330422974, 55.75381499999371), (37.8416827275085, 55.749277102484484), (37.84157576190186, 55.74794544108413), (37.83897929098507, 55.74525257875241), (37.83739676451868, 55.74404373042019), (37.838732481460525, 55.74298009816793), (37.841183997352545, 55.743060321833575), (37.84097476190185, 55.73938799999373), (37.84048155819702, 55.73570799999372), (37.840095812164286, 55.73228210777237), (37.83983814285274, 55.73080491981639), (37.83846476321406, 55.729799917464675), (37.83835745269769, 55.72919751082619), (37.838636380279524, 55.72859509486539), (37.8395161005249, 55.727705075632784), (37.83897964285276, 55.722727886185154), (37.83862557539366, 55.72034817326636), (37.83559735744853, 55.71944437307499), (37.835370708803126, 55.71831419154461), (37.83738169402022, 55.71765218986692), (37.83823396494291, 55.71691750159089), (37.838056931213345, 55.71547311301385), (37.836812846557606, 55.71221445615604), (37.83522525396725, 55.709331054395555), (37.83269301586908, 55.70953687463627), (37.829667367706236, 55.70903403789297), (37.83311126588435, 55.70552351822608), (37.83058993121339, 55.70041317726053), (37.82983872750851, 55.69883771404813), (37.82934501586913, 55.69718947487017), (37.828926414016685, 55.69504441658371), (37.82876530422971, 55.69287499999378), (37.82894754100031, 55.690759754047335), (37.827697554878185, 55.68951421135665), (37.82447346292115, 55.68965045405069), (37.83136543914793, 55.68322046195302), (37.833554015869154, 55.67814012759211), (37.83544184655761, 55.67295011628339), (37.837480388885474, 55.6672498719639), (37.838960677246064, 55.66316274139358), (37.83926093121332, 55.66046999999383), (37.839025050262435, 55.65869897264431), (37.83670784390257, 55.65794084879904), (37.835656529083245, 55.65694309303843), (37.83704060449217, 55.65689306460552), (37.83696819873806, 55.65550363526252), (37.83760389616388, 55.65487847246661), (37.83687972750851, 55.65356745541324), (37.83515216004943, 55.65155951234079), (37.83312418518067, 55.64979413590619), (37.82801726983639, 55.64640836412121), (37.820614174591, 55.64164525405531), (37.818908190475426, 55.6421883258084), (37.81717543386075, 55.64112490388471), (37.81690987037274, 55.63916106913107), (37.815099354492155, 55.637925371757085), (37.808769150787356, 55.633798276884455), (37.80100123544311, 55.62873670012244), (37.79598013491824, 55.62554336109055), (37.78634567724606, 55.62033499605651), (37.78334147619623, 55.618768681480326), (37.77746201055901, 55.619855533402706), (37.77527329626457, 55.61909966711279), (37.77801986242668, 55.618770300976294), (37.778212973541216, 55.617257701952106), (37.77784818518065, 55.61574504433011), (37.77016867724609, 55.61148576294007), (37.760191219573976, 55.60599579539028), (37.75338926983641, 55.60227892751446), (37.746329965606634, 55.59920577639331), (37.73939925396728, 55.59631430313617), (37.73273665739439, 55.5935318803559), (37.7299954450912, 55.59350760316188), (37.7268679946899, 55.59469840523759), (37.72626726983634, 55.59229549697373), (37.7262673598022, 55.59081598950582), (37.71897193121335, 55.5877595845419), (37.70871550793456, 55.58393177431724), (37.700497489410374, 55.580917323756644), (37.69204305026244, 55.57778089778455), (37.68544477378839, 55.57815154690915), (37.68391050793454, 55.57472945079756), (37.678803592590306, 55.57328235936491), (37.6743402539673, 55.57255251445782), (37.66813862698363, 55.57216388774464), (37.617927457672096, 55.57505691895805), (37.60443099999999, 55.5757737568051), (37.599683515869145, 55.57749105910326), (37.59754177842709, 55.57796291823627), (37.59625834786988, 55.57906686095235), (37.59501783265684, 55.57746616444403), (37.593090671936025, 55.57671634534502), (37.587018007904, 55.577944600233785), (37.578692203704804, 55.57982895000019), (37.57327546607398, 55.58116294118248), (37.57385012109279, 55.581550362779), (37.57399562266922, 55.5820107079112), (37.5735356072979, 55.58226289171689), (37.57290393054962, 55.582393529795155), (37.57037722355653, 55.581919415056234), (37.5592298306885, 55.584471614867844), (37.54189249206543, 55.58867650795186), (37.5297256269836, 55.59158133551745), (37.517837865081766, 55.59443656218868), (37.51200186508174, 55.59635625174229), (37.506808949737554, 55.59907823904434), (37.49820432275389, 55.6062944994944), (37.494406071441674, 55.60967103463367), (37.494760001358024, 55.61066689753365), (37.49397137107085, 55.61220931698269), (37.49016528606031, 55.613417718449064), (37.48773249206542, 55.61530616333343), (37.47921386508177, 55.622640129112334), (37.470652153442394, 55.62993723476164), (37.46273446298218, 55.6368075123157), (37.46350692265317, 55.64068225239439), (37.46050283203121, 55.640794546982576), (37.457627470916734, 55.64118904154646), (37.450718034393326, 55.64690488145138), (37.44239252645875, 55.65397824729769), (37.434587576721185, 55.66053543155961), (37.43582144975277, 55.661693766520735), (37.43576786245721, 55.662755031737014), (37.430982915344174, 55.664610641628116), (37.428547447097685, 55.66778515273695), (37.42945134592044, 55.668633314343566), (37.42859571562949, 55.66948145750025), (37.4262836402282, 55.670813882451405), (37.418709037048295, 55.6811141674414), (37.41922139651101, 55.68235377885389), (37.419218771842885, 55.68359335082235), (37.417196501327446, 55.684375235224735), (37.41607020370478, 55.68540557585352), (37.415640857147146, 55.68686637150793), (37.414632153442334, 55.68903015131686), (37.413344899475064, 55.690896881757396), (37.41171432275391, 55.69264232162232), (37.40948282275393, 55.69455101638112), (37.40703674603271, 55.69638690385348), (37.39607169577025, 55.70451821283731), (37.38952706878662, 55.70942491932811), (37.387778313491815, 55.71149057784176), (37.39049275399779, 55.71419814298992), (37.385557272491454, 55.7155489617061), (37.38388335714726, 55.71849856042102), (37.378368238098155, 55.7292763261685), (37.37763597123337, 55.730845879211614), (37.37890062088197, 55.73167906388319), (37.37750451918789, 55.734703664681774), (37.375610832015965, 55.734851959522246), (37.3723813571472, 55.74105626086403), (37.37014935714723, 55.746115620904355), (37.36944173016362, 55.750883999993725), (37.36975304365541, 55.76335905525834), (37.37244070571134, 55.76432079697595), (37.3724259757175, 55.76636979670426), (37.369922155757884, 55.76735417953104), (37.369892695770275, 55.76823419316575), (37.370214730163575, 55.782312184391266), (37.370493611114505, 55.78436801120489), (37.37120164550783, 55.78596427165359), (37.37284851456452, 55.7874378183096), (37.37608325135799, 55.7886695054807), (37.3764587460632, 55.78947647305964), (37.37530000265506, 55.79146512926804), (37.38235915344241, 55.79899647809345), (37.384344043655396, 55.80113596939471), (37.38594269577028, 55.80322699999366), (37.38711208598329, 55.804919036911976), (37.3880239841309, 55.806610999993666), (37.38928977249147, 55.81001864976979), (37.39038389947512, 55.81348641242801), (37.39235781481933, 55.81983538336746), (37.393709457672124, 55.82417822811877), (37.394685720901464, 55.82792275755836), (37.39557615344238, 55.830447148154136), (37.39844478226658, 55.83167107969975), (37.40019761214057, 55.83151823557964), (37.400398790382326, 55.83264967594742), (37.39659544313046, 55.83322180909622), (37.39667059524539, 55.83402792148566), (37.39682089947515, 55.83638877400216), (37.39643489154053, 55.83861656112751), (37.3955338994751, 55.84072348043264), (37.392680272491454, 55.84502158126453), (37.39241188227847, 55.84659117913199), (37.392529730163616, 55.84816071336481), (37.39486835714723, 55.85288092980303), (37.39873052645878, 55.859893456073635), (37.40272161111449, 55.86441833633205), (37.40697072750854, 55.867579567544375), (37.410007082016016, 55.868369880337), (37.4120992989502, 55.86920843741314), (37.412668021163924, 55.87055369615854), (37.41482461111453, 55.87170587948249), (37.41862266137694, 55.873183961039565), (37.42413732540892, 55.874879126654704), (37.4312182698669, 55.875614937236705), (37.43111093783558, 55.8762723478417), (37.43332105622856, 55.87706546369396), (37.43385747619623, 55.87790681284802), (37.441303050262405, 55.88027084462084), (37.44747234260555, 55.87942070143253), (37.44716141796871, 55.88072960917233), (37.44769797085568, 55.88121221323979), (37.45204320500181, 55.882080694420715), (37.45673176190186, 55.882346110794586), (37.463383999999984, 55.88252729504517), (37.46682797486874, 55.88294937719063), (37.470014457672086, 55.88361266759345), (37.47751410450743, 55.88546991372396), (37.47860317658232, 55.88534929207307), (37.48165826025772, 55.882563306475106), (37.48316434442331, 55.8815803226785), (37.483831555817645, 55.882427612793315), (37.483182967125686, 55.88372791409729), (37.483092277908824, 55.88495581062434), (37.4855716508179, 55.8875561994203), (37.486440636245746, 55.887827444039566), (37.49014203439328, 55.88897899871799), (37.493210285705544, 55.890208937135604), (37.497512451065035, 55.891342397444696), (37.49780744510645, 55.89174030252967), (37.49940333499519, 55.89239745507079), (37.50018383334346, 55.89339220941865), (37.52421672750851, 55.903869074155224), (37.52977457672118, 55.90564076517974), (37.53503220370484, 55.90661661218259), (37.54042858064267, 55.90714113744566), (37.54320461007303, 55.905645048442985), (37.545686966066306, 55.906608607018505), (37.54743976120755, 55.90788552162358), (37.55796999999999, 55.90901557907218), (37.572711542327866, 55.91059395704873), (37.57942799999998, 55.91073854155573), (37.58502865872187, 55.91009969268444), (37.58739968913264, 55.90794809960554), (37.59131567193598, 55.908713267595054), (37.612687423278814, 55.902866854295375), (37.62348079629517, 55.90041967242986), (37.635797880950896, 55.898141151686396), (37.649487626983664, 55.89639275532968), (37.65619302513125, 55.89572360207488), (37.66294133862307, 55.895295577183965), (37.66874564418033, 55.89505457604897), (37.67375601586915, 55.89254677027454), (37.67744661901856, 55.8947775867987), (37.688347, 55.89450045676125), (37.69480554232789, 55.89422926332761), (37.70107096560668, 55.89322256101114), (37.705962965606716, 55.891763491662616), (37.711885134918205, 55.889110234998974), (37.71682005026245, 55.886577568759876), (37.7199315476074, 55.88458159806678), (37.72234560316464, 55.882281005794134), (37.72364385977171, 55.8809452036196), (37.725371142837474, 55.8809722706006), (37.727870902099546, 55.88037213862385), (37.73394330422971, 55.877941504088696), (37.745339592590376, 55.87208120378722), (37.75525267724611, 55.86703807949492), (37.76919976190188, 55.859821640197474), (37.827835219574, 55.82962968399116), (37.83341438888553, 55.82575289922351), (37.83652584655761, 55.82188784027888), (37.83809213491821, 55.81612575504693), (37.83605359521481, 55.81460347077685), (37.83632178569025, 55.81276696067908), (37.838623105812026, 55.811486181656385), (37.83912198147584, 55.807329380532785), (37.839079078033414, 55.80510270463816), (37.83965844708251, 55.79940712529036), (37.840581150787344, 55.79131399999368), (37.84172564285271, 55.78000432402266)]); +``` + +Check how many cell towers are in Moscow: + +``` +SELECT count() FROM cell_towers WHERE pointInPolygon((lon, lat), (SELECT * FROM moscow)) + +┌─count()─┐ +│ 310463 │ +└─────────┘ + +1 rows in set. Elapsed: 0.067 sec. Processed 43.28 million rows, 692.42 MB (645.83 million rows/s., 10.33 GB/s.) +``` + +The data is also available for interactive queries in the [Playground](https://gh-api.clickhouse.tech/play?user=play), [example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIG1jYywgY291bnQoKSBGUk9NIGNlbGxfdG93ZXJzIEdST1VQIEJZIG1jYyBPUkRFUiBCWSBjb3VudCgpIERFU0M=). + +Although you cannot create temporary tables there. + +[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets/cell-towers/) diff --git a/docs/en/getting-started/example-datasets/index.md b/docs/en/getting-started/example-datasets/index.md index 72f44d8caf1..53007c33306 100644 --- a/docs/en/getting-started/example-datasets/index.md +++ b/docs/en/getting-started/example-datasets/index.md @@ -20,5 +20,6 @@ The list of documented datasets: - [Terabyte of Click Logs from Criteo](../../getting-started/example-datasets/criteo.md) - [AMPLab Big Data Benchmark](../../getting-started/example-datasets/amplab-benchmark.md) - [Brown University Benchmark](../../getting-started/example-datasets/brown-benchmark.md) +- [Cell Towers](../../getting-started/example-datasets/cell-towers.md) [Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets) diff --git a/docs/en/getting-started/example-datasets/ontime.md b/docs/en/getting-started/example-datasets/ontime.md index 6e46cddba52..83673cdceb6 100644 --- a/docs/en/getting-started/example-datasets/ontime.md +++ b/docs/en/getting-started/example-datasets/ontime.md @@ -15,17 +15,9 @@ This dataset can be obtained in two ways: Downloading data: ``` bash -for s in `seq 1987 2018` -do -for m in `seq 1 12` -do -wget https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_${s}_${m}.zip -done -done +echo https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_{1987..2021}_{1..12}.zip | xargs -P10 wget --no-check-certificate --continue ``` -(from https://github.com/Percona-Lab/ontime-airline-performance/blob/master/download.sh ) - Creating a table: ``` sql @@ -145,12 +137,14 @@ ORDER BY (Carrier, FlightDate) SETTINGS index_granularity = 8192; ``` -Loading data: +Loading data with multiple threads: ``` bash -$ for i in *.zip; do echo $i; unzip -cq $i '*.csv' | sed 's/\.00//g' | clickhouse-client --input_format_with_names_use_header=0 --host=example-perftest01j --query="INSERT INTO ontime FORMAT CSVWithNames"; done +ls -1 *.zip | xargs -I{} -P $(nproc) bash -c "echo {}; unzip -cq {} '*.csv' | sed 's/\.00//g' | clickhouse-client --input_format_with_names_use_header=0 --query='INSERT INTO ontime FORMAT CSVWithNames'" ``` +(if you will have memory shortage or other issues on your server, remove the `-P $(nproc)` part) + ## Download of Prepared Partitions {#download-of-prepared-partitions} ``` bash diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index d94b69508f3..3c343e09fd3 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1956,8 +1956,8 @@ Default value: 16. **See Also** -- [Kafka](../../engines/table-engines/integrations/kafka.md#kafka) engine -- [RabbitMQ](../../engines/table-engines/integrations/rabbitmq.md#rabbitmq-engine) engine +- [Kafka](../../engines/table-engines/integrations/kafka.md#kafka) engine. +- [RabbitMQ](../../engines/table-engines/integrations/rabbitmq.md#rabbitmq-engine) engine. ## validate_polygons {#validate_polygons} @@ -2658,8 +2658,6 @@ Result: Note that this setting influences [Materialized view](../../sql-reference/statements/create/view.md#materialized) and [MaterializeMySQL](../../engines/database-engines/materialize-mysql.md) behaviour. -[Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) - ## engine_file_empty_if_not_exists {#engine-file-empty_if-not-exists} Allows to select data from a file engine table without file. @@ -2679,3 +2677,16 @@ Possible values: - 1 — Enabled. Default value: `0`. + +## allow_experimental_geo_types {#allow-experimental-geo-types} + +Allows working with experimental [geo data types](../../sql-reference/data-types/geo.md). + +Possible values: + +- 0 — Working with geo data types is disabled. +- 1 — Working with geo data types is enabled. + +Default value: `0`. + +[Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/en/operations/system-tables/index.md b/docs/en/operations/system-tables/index.md index 5dc23aee686..e66f082167e 100644 --- a/docs/en/operations/system-tables/index.md +++ b/docs/en/operations/system-tables/index.md @@ -20,7 +20,7 @@ System tables: Most of system tables store their data in RAM. A ClickHouse server creates such system tables at the start. -Unlike other system tables, the system log tables [metric_log](../../operations/system-tables/metric_log.md), [query_log](../../operations/system-tables/query_log.md), [query_thread_log](../../operations/system-tables/query_thread_log.md), [trace_log](../../operations/system-tables/trace_log.md), [part_log](../../operations/system-tables/part_log.md), crash_log and [text_log](../../operations/system-tables/text_log.md) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem by default. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one. +Unlike other system tables, the system log tables [metric_log](../../operations/system-tables/metric_log.md), [query_log](../../operations/system-tables/query_log.md), [query_thread_log](../../operations/system-tables/query_thread_log.md), [trace_log](../../operations/system-tables/trace_log.md), [part_log](../../operations/system-tables/part_log.md), [crash_log](../../operations/system-tables/crash-log.md) and [text_log](../../operations/system-tables/text_log.md) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a filesystem by default. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one. System log tables can be customized by creating a config file with the same name as the table under `/etc/clickhouse-server/config.d/`, or setting corresponding elements in `/etc/clickhouse-server/config.xml`. Elements can be customized are: @@ -33,7 +33,7 @@ System log tables can be customized by creating a config file with the same name An example: -``` +```xml system diff --git a/docs/en/operations/utilities/clickhouse-local.md b/docs/en/operations/utilities/clickhouse-local.md index 04f9f3660b5..cfabf42bff1 100644 --- a/docs/en/operations/utilities/clickhouse-local.md +++ b/docs/en/operations/utilities/clickhouse-local.md @@ -91,6 +91,8 @@ $ clickhouse-local --query " Now let’s output memory user for each Unix user: +Query: + ``` bash $ ps aux | tail -n +2 | awk '{ printf("%s\t%s\n", $1, $4) }' \ | clickhouse-local --structure "user String, mem Float64" \ @@ -98,6 +100,8 @@ $ ps aux | tail -n +2 | awk '{ printf("%s\t%s\n", $1, $4) }' \ FROM table GROUP BY user ORDER BY memTotal DESC FORMAT Pretty" ``` +Result: + ``` text Read 186 rows, 4.15 KiB in 0.035 sec., 5302 rows/sec., 118.34 KiB/sec. ┏━━━━━━━━━━┳━━━━━━━━━━┓ diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 035bc91b9ed..c6c97b5428b 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -253,8 +253,8 @@ windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) **Parameters** -- `window` — Length of the sliding window. The unit of `window` depends on the timestamp itself and varies. Determined using the expression `timestamp of cond2 <= timestamp of cond1 + window`. -- `mode` - It is an optional parameter. +- `window` — Length of the sliding window. The unit of `window` depends on the `timestamp` itself and varies. Determined using the expression `timestamp of cond2 <= timestamp of cond1 + window`. +- `mode` - It is an optional argument. - `'strict'` - When the `'strict'` is set, the windowFunnel() applies conditions only for the unique values. **Returned value** diff --git a/docs/en/sql-reference/data-types/geo.md b/docs/en/sql-reference/data-types/geo.md new file mode 100644 index 00000000000..9ed328e0de6 --- /dev/null +++ b/docs/en/sql-reference/data-types/geo.md @@ -0,0 +1,106 @@ +--- +toc_priority: 62 +toc_title: Geo +--- + +# Geo Data Types {#geo-data-types} + +Clickhouse supports data types for representing geographical objects — locations, lands, etc. + +!!! warning "Warning" + Currently geo data types are an experimental feature. To work with them you must set `allow_experimental_geo_types = 1`. + +**See Also** +- [Representing simple geographical features](https://en.wikipedia.org/wiki/GeoJSON). +- [allow_experimental_geo_types](../../operations/settings/settings.md#allow-experimental-geo-types) setting. + +## Point {#point-data-type} + +`Point` is represented by its X and Y coordinates, stored as a [Tuple](tuple.md)([Float64](float.md), [Float64](float.md)). + +**Example** + +Query: + +```sql +SET allow_experimental_geo_types = 1; +CREATE TABLE geo_point (p Point) ENGINE = Memory(); +INSERT INTO geo_point VALUES((10, 10)); +SELECT p, toTypeName(p) FROM geo_point; +``` +Result: + +``` text +┌─p─────┬─toTypeName(p)─┐ +│ (10,10) │ Point │ +└───────┴───────────────┘ +``` + +## Ring {#ring-data-type} + +`Ring` is a simple polygon without holes stored as an array of points: [Array](array.md)([Point](#point-data-type)). + +**Example** + +Query: + +```sql +SET allow_experimental_geo_types = 1; +CREATE TABLE geo_ring (r Ring) ENGINE = Memory(); +INSERT INTO geo_ring VALUES([(0, 0), (10, 0), (10, 10), (0, 10)]); +SELECT r, toTypeName(r) FROM geo_ring; +``` +Result: + +``` text +┌─r─────────────────────────────┬─toTypeName(r)─┐ +│ [(0,0),(10,0),(10,10),(0,10)] │ Ring │ +└───────────────────────────────┴───────────────┘ +``` + +## Polygon {#polygon-data-type} + +`Polygon` is a polygon with holes stored as an array of rings: [Array](array.md)([Ring](#ring-data-type)). First element of outer array is the outer shape of polygon and all the following elements are holes. + +**Example** + +This is a polygon with one hole: + +```sql +SET allow_experimental_geo_types = 1; +CREATE TABLE geo_polygon (pg Polygon) ENGINE = Memory(); +INSERT INTO geo_polygon VALUES([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]]); +SELECT pg, toTypeName(pg) FROM geo_polygon; +``` + +Result: + +``` text +┌─pg────────────────────────────────────────────────────────────┬─toTypeName(pg)─┐ +│ [[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]] │ Polygon │ +└───────────────────────────────────────────────────────────────┴────────────────┘ +``` + +## MultiPolygon {#multipolygon-data-type} + +`MultiPolygon` consists of multiple polygons and is stored as an array of polygons: [Array](array.md)([Polygon](#polygon-data-type)). + +**Example** + +This multipolygon consists of two separate polygons — the first one without holes, and the second with one hole: + +```sql +SET allow_experimental_geo_types = 1; +CREATE TABLE geo_multipolygon (mpg MultiPolygon) ENGINE = Memory(); +INSERT INTO geo_multipolygon VALUES([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]]); +SELECT mpg, toTypeName(mpg) FROM geo_multipolygon; +``` +Result: + +``` text +┌─mpg─────────────────────────────────────────────────────────────────────────────────────────────┬─toTypeName(mpg)─┐ +│ [[[(0,0),(10,0),(10,10),(0,10)]],[[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]]] │ MultiPolygon │ +└─────────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────┘ +``` + +[Original article](https://clickhouse.tech/docs/en/data-types/geo/) diff --git a/docs/en/sql-reference/data-types/simpleaggregatefunction.md b/docs/en/sql-reference/data-types/simpleaggregatefunction.md index 2d2746f85d3..244779c5ca8 100644 --- a/docs/en/sql-reference/data-types/simpleaggregatefunction.md +++ b/docs/en/sql-reference/data-types/simpleaggregatefunction.md @@ -21,7 +21,11 @@ The following aggregate functions are supported: - [`argMin`](../../sql-reference/aggregate-functions/reference/argmin.md) - [`argMax`](../../sql-reference/aggregate-functions/reference/argmax.md) -Values of the `SimpleAggregateFunction(func, Type)` look and stored the same way as `Type`, so you do not need to apply functions with `-Merge`/`-State` suffixes. `SimpleAggregateFunction` has better performance than `AggregateFunction` with same aggregation function. + +!!! note "Note" + Values of the `SimpleAggregateFunction(func, Type)` look and stored the same way as `Type`, so you do not need to apply functions with `-Merge`/`-State` suffixes. + + `SimpleAggregateFunction` has better performance than `AggregateFunction` with same aggregation function. **Parameters** @@ -31,11 +35,7 @@ Values of the `SimpleAggregateFunction(func, Type)` look and stored the same way **Example** ``` sql -CREATE TABLE t -( - column1 SimpleAggregateFunction(sum, UInt64), - column2 SimpleAggregateFunction(any, String) -) ENGINE = ... +CREATE TABLE simple (id UInt64, val SimpleAggregateFunction(sum, Double)) ENGINE=AggregatingMergeTree ORDER BY id; ``` [Original article](https://clickhouse.tech/docs/en/data_types/simpleaggregatefunction/) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 688511450e2..c557d8f369c 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -771,7 +771,7 @@ This is necessary for searching for pageviews in the corresponding session. ## formatDateTime {#formatdatetime} -Function formats a Time according given Format string. N.B.: Format is a constant expression, e.g. you can not have multiple formats for single result column. +Formats a Time according to the given Format string. Format is a constant expression, so you cannot have multiple formats for a single result column. **Syntax** @@ -830,31 +830,32 @@ Result: └────────────────────────────────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/date_time_functions/) - ## FROM\_UNIXTIME {#fromunixfime} -When there is only single argument of integer type, it act in the same way as `toDateTime` and return [DateTime](../../sql-reference/data-types/datetime.md). -type. +Function converts Unix timestamp to a calendar date and a time of a day. When there is only a single argument of [Integer](../../sql-reference/data-types/int-uint.md) type, it acts in the same way as [toDateTime](../../sql-reference/functions/type-conversion-functions.md#todatetime) and return [DateTime](../../sql-reference/data-types/datetime.md) type. -For example: +**Example:** + +Query: ```sql -SELECT FROM_UNIXTIME(423543535) +SELECT FROM_UNIXTIME(423543535); ``` +Result: + ```text ┌─FROM_UNIXTIME(423543535)─┐ │ 1983-06-04 10:58:55 │ └──────────────────────────┘ ``` -When there are two arguments, first is integer or DateTime, second is constant format string, it act in the same way as `formatDateTime` and return `String` type. +When there are two arguments: first is an [Integer](../../sql-reference/data-types/int-uint.md) or [DateTime](../../sql-reference/data-types/datetime.md), second is a constant format string — it acts in the same way as [formatDateTime](#formatdatetime) and return [String](../../sql-reference/data-types/string.md#string) type. For example: ```sql -SELECT FROM_UNIXTIME(1234334543, '%Y-%m-%d %R:%S') AS DateTime +SELECT FROM_UNIXTIME(1234334543, '%Y-%m-%d %R:%S') AS DateTime; ``` ```text @@ -1006,3 +1007,5 @@ Result: │ 2020-01-01 │ └────────────────────────────────────┘ ``` + +[Original article](https://clickhouse.tech/docs/en/query_language/functions/date_time_functions/) diff --git a/docs/en/sql-reference/functions/encryption-functions.md b/docs/en/sql-reference/functions/encryption-functions.md index c7b26b1f288..df27685dcb3 100644 --- a/docs/en/sql-reference/functions/encryption-functions.md +++ b/docs/en/sql-reference/functions/encryption-functions.md @@ -55,7 +55,7 @@ CREATE TABLE encryption_test `comment` String, `secret` String ) -ENGINE = Memory +ENGINE = Memory; ``` Insert some data (please avoid storing the keys/ivs in the database as this undermines the whole concept of encryption), also storing 'hints' is unsafe too and used only for illustrative purposes: @@ -110,7 +110,7 @@ Result: Compatible with mysql encryption and resulting ciphertext can be decrypted with [AES_DECRYPT](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_aes-decrypt) function. -Will produce same ciphertext as `encrypt` on equal inputs. But when `key` or `iv` are longer than they should normally be, `aes_encrypt_mysql` will stick to what MySQL's `aes_encrypt` does: 'fold' `key` and ignore excess bits of `IV`. +Will produce the same ciphertext as `encrypt` on equal inputs. But when `key` or `iv` are longer than they should normally be, `aes_encrypt_mysql` will stick to what MySQL's `aes_encrypt` does: 'fold' `key` and ignore excess bits of `iv`. Supported encryption modes: @@ -138,7 +138,6 @@ aes_encrypt_mysql('mode', 'plaintext', 'key' [, iv]) - Ciphertext binary string. [String](../../sql-reference/data-types/string.md#string). - **Examples** Given equal input `encrypt` and `aes_encrypt_mysql` produce the same ciphertext: @@ -157,7 +156,6 @@ Result: └───────────────────┘ ``` - But `encrypt` fails when `key` or `iv` is longer than expected: Query: @@ -252,7 +250,7 @@ decrypt('mode', 'ciphertext', 'key' [, iv, aad]) **Examples** -Re-using table from [encrypt](./encryption-functions.md#encrypt). +Re-using table from [encrypt](#encrypt). Query: @@ -284,6 +282,7 @@ SELECT comment, decrypt('aes-256-cfb128', secret, '12345678910121314151617181920 ``` Result: + ``` text ┌─comment─────────────────────────────┬─plaintext─┐ │ aes-256-cfb128 no IV │ Secret │ @@ -294,7 +293,7 @@ Result: └─────────────────────────────────────┴───────────┘ ``` -Notice how only portion of the data was properly decrypted, and the rest is gibberish since either `mode`, `key`, or `iv` were different upon encryption. +Notice how only a portion of the data was properly decrypted, and the rest is gibberish since either `mode`, `key`, or `iv` were different upon encryption. ## aes_decrypt_mysql {#aes_decrypt_mysql} @@ -331,6 +330,7 @@ aes_decrypt_mysql('mode', 'ciphertext', 'key' [, iv]) **Examples** Let's decrypt data we've previously encrypted with MySQL: + ``` sql mysql> SET block_encryption_mode='aes-256-cfb128'; Query OK, 0 rows affected (0.00 sec) @@ -345,11 +345,13 @@ mysql> SELECT aes_encrypt('Secret', '123456789101213141516171819202122', 'iviviv ``` Query: + ``` sql SELECT aes_decrypt_mysql('aes-256-cfb128', unhex('24E9E4966469'), '123456789101213141516171819202122', 'iviviviviviviviv123456') AS plaintext ``` Result: + ``` text ┌─plaintext─┐ │ Secret │ diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index 6bf1bebabaa..465ad01527f 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -9,7 +9,7 @@ Hash functions can be used for the deterministic pseudo-random shuffling of elem ## halfMD5 {#hash-functions-halfmd5} -[Interprets](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-reinterpretAsString) all the input parameters as strings and calculates the [MD5](https://en.wikipedia.org/wiki/MD5) hash value for each of them. Then combines hashes, takes the first 8 bytes of the hash of the resulting string, and interprets them as `UInt64` in big-endian byte order. +[Interprets](../../sql-reference/functions/type-conversion-functions.md#type_conversion_functions-reinterpretAsString) all the input parameters as strings and calculates the [MD5](https://en.wikipedia.org/wiki/MD5) hash value for each of them. Then combines hashes, takes the first 8 bytes of the hash of the resulting string, and interprets them as `UInt64` in big-endian byte order. ``` sql halfMD5(par1, ...) @@ -54,7 +54,7 @@ sipHash64(par1,...) This is a cryptographic hash function. It works at least three times faster than the [MD5](#hash_functions-md5) function. -Function [interprets](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-reinterpretAsString) all the input parameters as strings and calculates the hash value for each of them. Then combines hashes by the following algorithm: +Function [interprets](../../sql-reference/functions/type-conversion-functions.md#type_conversion_functions-reinterpretAsString) all the input parameters as strings and calculates the hash value for each of them. Then combines hashes by the following algorithm: 1. After hashing all the input parameters, the function gets the array of hashes. 2. Function takes the first and the second elements and calculates a hash for the array of them. diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index 2b3a9d9103f..1d4839cbbf9 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -174,4 +174,129 @@ Result: └──────────────────────────────┴───────────────────────────────────┘ ``` +## mapContains {#mapcontains} + +Determines whether the `map` contains the `key` parameter. + +**Syntax** + +``` sql +mapContains(map, key) +``` + +**Parameters** + +- `map` — Map. [Map](../../sql-reference/data-types/map.md). +- `key` — Key. Type matches the type of keys of `map` parameter. + +**Returned value** + +- `1` if `map` contains `key`, `0` if not. + +Type: [UInt8](../../sql-reference/data-types/int-uint.md). + +**Example** + +Query: + +```sql +CREATE TABLE test (a Map(String,String)) ENGINE = Memory; + +INSERT INTO test VALUES ({'name':'eleven','age':'11'}), ({'number':'twelve','position':'6.0'}); + +SELECT mapContains(a, 'name') FROM test; + +``` + +Result: + +```text +┌─mapContains(a, 'name')─┐ +│ 1 │ +│ 0 │ +└────────────────────────┘ +``` + +## mapKeys {#mapkeys} + +Returns all keys from the `map` parameter. + +**Syntax** + +```sql +mapKeys(map) +``` + +**Parameters** + +- `map` — Map. [Map](../../sql-reference/data-types/map.md). + +**Returned value** + +- Array containing all keys from the `map`. + +Type: [Array](../../sql-reference/data-types/array.md). + +**Example** + +Query: + +```sql +CREATE TABLE test (a Map(String,String)) ENGINE = Memory; + +INSERT INTO test VALUES ({'name':'eleven','age':'11'}), ({'number':'twelve','position':'6.0'}); + +SELECT mapKeys(a) FROM test; +``` + +Result: + +```text +┌─mapKeys(a)────────────┐ +│ ['name','age'] │ +│ ['number','position'] │ +└───────────────────────┘ +``` + +## mapValues {#mapvalues} + +Returns all values from the `map` parameter. + +**Syntax** + +```sql +mapKeys(map) +``` + +**Parameters** + +- `map` — Map. [Map](../../sql-reference/data-types/map.md). + +**Returned value** + +- Array containing all the values from `map`. + +Type: [Array](../../sql-reference/data-types/array.md). + +**Example** + +Query: + +```sql +CREATE TABLE test (a Map(String,String)) ENGINE = Memory; + +INSERT INTO test VALUES ({'name':'eleven','age':'11'}), ({'number':'twelve','position':'6.0'}); + +SELECT mapValues(a) FROM test; +``` + +Result: + +```text +┌─mapValues(a)─────┐ +│ ['eleven','11'] │ +│ ['twelve','6.0'] │ +└──────────────────┘ +``` + [Original article](https://clickhouse.tech/docs/en/sql-reference/functions/tuple-map-functions/) diff --git a/docs/en/sql-reference/statements/alter/ttl.md b/docs/en/sql-reference/statements/alter/ttl.md index 5331afdb2f8..e8bfb78ec68 100644 --- a/docs/en/sql-reference/statements/alter/ttl.md +++ b/docs/en/sql-reference/statements/alter/ttl.md @@ -81,5 +81,5 @@ The `TTL` is no longer there, so the second row is not deleted: ### See Also -- More about the [TTL-expression](../../../sql-reference/statements/create/table#ttl-expression). -- Modify column [with TTL](../../../sql-reference/statements/alter/column#alter_modify-column). +- More about the [TTL-expression](../../../../sql-reference/statements/create/table#ttl-expression). +- Modify column [with TTL](../../../../sql-reference/statements/alter/column#alter_modify-column). diff --git a/docs/en/sql-reference/statements/select/all.md b/docs/en/sql-reference/statements/select/all.md index 5e0de4c142b..891b82c4319 100644 --- a/docs/en/sql-reference/statements/select/all.md +++ b/docs/en/sql-reference/statements/select/all.md @@ -4,10 +4,8 @@ toc_title: ALL # ALL Clause {#select-all} -`SELECT ALL` is identical to `SELECT` without `DISTINCT`. +If there are multiple matching rows in the table, then `ALL` returns all of them. `SELECT ALL` is identical to `SELECT` without `DISTINCT`. If both `ALL` and `DISTINCT` specified, exception will be thrown. -- If `ALL` specified, ignore it. -- If both `ALL` and `DISTINCT` specified, exception will be thrown. `ALL` can also be specified inside aggregate function with the same effect(noop), for instance: @@ -19,3 +17,5 @@ equals to ```sql SELECT sum(number) FROM numbers(10); ``` + +[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/all) diff --git a/docs/ru/engines/table-engines/integrations/kafka.md b/docs/ru/engines/table-engines/integrations/kafka.md index 940fee2452b..5a6971b1ae6 100644 --- a/docs/ru/engines/table-engines/integrations/kafka.md +++ b/docs/ru/engines/table-engines/integrations/kafka.md @@ -31,21 +31,26 @@ SETTINGS [kafka_schema = '',] [kafka_num_consumers = N,] [kafka_skip_broken_messages = N] + [kafka_commit_every_batch = 0,] + [kafka_thread_per_consumer = 0] ``` Обязательные параметры: -- `kafka_broker_list` – перечень брокеров, разделенный запятыми (`localhost:9092`). -- `kafka_topic_list` – перечень необходимых топиков Kafka. -- `kafka_group_name` – группа потребителя Kafka. Отступы для чтения отслеживаются для каждой группы отдельно. Если необходимо, чтобы сообщения не повторялись на кластере, используйте везде одно имя группы. -- `kafka_format` – формат сообщений. Названия форматов должны быть теми же, что можно использовать в секции `FORMAT`, например, `JSONEachRow`. Подробнее читайте в разделе [Форматы](../../../interfaces/formats.md). +- `kafka_broker_list` — перечень брокеров, разделенный запятыми (`localhost:9092`). +- `kafka_topic_list` — перечень необходимых топиков Kafka. +- `kafka_group_name` — группа потребителя Kafka. Отступы для чтения отслеживаются для каждой группы отдельно. Если необходимо, чтобы сообщения не повторялись на кластере, используйте везде одно имя группы. +- `kafka_format` — формат сообщений. Названия форматов должны быть теми же, что можно использовать в секции `FORMAT`, например, `JSONEachRow`. Подробнее читайте в разделе [Форматы](../../../interfaces/formats.md). Опциональные параметры: -- `kafka_row_delimiter` – символ-разделитель записей (строк), которым завершается сообщение. -- `kafka_schema` – опциональный параметр, необходимый, если используется формат, требующий определения схемы. Например, [Cap’n Proto](https://capnproto.org/) требует путь к файлу со схемой и название корневого объекта `schema.capnp:Message`. -- `kafka_num_consumers` – количество потребителей (consumer) на таблицу. По умолчанию: `1`. Укажите больше потребителей, если пропускная способность одного потребителя недостаточна. Общее число потребителей не должно превышать количество партиций в топике, так как на одну партицию может быть назначено не более одного потребителя. -- `kafka_skip_broken_messages` – максимальное количество некорректных сообщений в блоке. Если `kafka_skip_broken_messages = N`, то движок отбрасывает `N` сообщений Кафки, которые не получилось обработать. Одно сообщение в точности соответствует одной записи (строке). Значение по умолчанию – 0. +- `kafka_row_delimiter` — символ-разделитель записей (строк), которым завершается сообщение. +- `kafka_schema` — опциональный параметр, необходимый, если используется формат, требующий определения схемы. Например, [Cap’n Proto](https://capnproto.org/) требует путь к файлу со схемой и название корневого объекта `schema.capnp:Message`. +- `kafka_num_consumers` — количество потребителей (consumer) на таблицу. По умолчанию: `1`. Укажите больше потребителей, если пропускная способность одного потребителя недостаточна. Общее число потребителей не должно превышать количество партиций в топике, так как на одну партицию может быть назначено не более одного потребителя. +- `kafka_max_block_size` — максимальный размер пачек (в сообщениях) для poll (по умолчанию `max_block_size`). +- `kafka_skip_broken_messages` — максимальное количество некорректных сообщений в блоке. Если `kafka_skip_broken_messages = N`, то движок отбрасывает `N` сообщений Кафки, которые не получилось обработать. Одно сообщение в точности соответствует одной записи (строке). Значение по умолчанию – 0. +- `kafka_commit_every_batch` — включает или отключает режим записи каждой принятой и обработанной пачки по отдельности вместо единой записи целого блока (по умолчанию `0`). +- `kafka_thread_per_consumer` — включает или отключает предоставление отдельного потока каждому потребителю (по умолчанию `0`). При включенном режиме каждый потребитель сбрасывает данные независимо и параллельно, при отключённом — строки с данными от нескольких потребителей собираются в один блок. Примеры diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index 26e05b02509..97fa382fdd9 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -22,6 +22,7 @@ toc_title: "\u041a\u043b\u0438\u0435\u043d\u0442\u0441\u043a\u0438\u0435\u0020\u - [seva-code/php-click-house-client](https://packagist.org/packages/seva-code/php-click-house-client) - [SeasClick C++ client](https://github.com/SeasX/SeasClick) - [glushkovds/phpclickhouse-laravel](https://packagist.org/packages/glushkovds/phpclickhouse-laravel) + - [kolya7k ClickHouse PHP extension](https://github.com//kolya7k/clickhouse-php) - Go - [clickhouse](https://github.com/kshvakov/clickhouse/) - [go-clickhouse](https://github.com/roistat/go-clickhouse) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 7322b6c9184..f8f587c8a36 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1937,6 +1937,21 @@ SELECT idx, i FROM null_in WHERE i IN (1, NULL) SETTINGS transform_null_in = 1; Значение по умолчанию: 16. +## background_message_broker_schedule_pool_size {#background_message_broker_schedule_pool_size} + +Задает количество потоков для фонового потокового вывода сообщений. Настройка применяется при запуске сервера ClickHouse и не может быть изменена в пользовательском сеансе. + +Допустимые значения: + +- Положительное целое число. + +Значение по умолчанию: 16. + +**Смотрите также** + +- Движок [Kafka](../../engines/table-engines/integrations/kafka.md#kafka). +- Движок [RabbitMQ](../../engines/table-engines/integrations/rabbitmq.md#rabbitmq-engine). + ## format_avro_schema_registry_url {#format_avro_schema_registry_url} Задает URL реестра схем [Confluent](https://docs.confluent.io/current/schema-registry/index.html) для использования с форматом [AvroConfluent](../../interfaces/formats.md#data-format-avro-confluent). @@ -2537,4 +2552,15 @@ SELECT * FROM test2; Обратите внимание на то, что эта настройка влияет на поведение [материализованных представлений](../../sql-reference/statements/create/view.md#materialized) и БД [MaterializeMySQL](../../engines/database-engines/materialize-mysql.md). +## allow_experimental_geo_types {#allow-experimental-geo-types} + +Разрешает использование экспериментальных типов данных для работы с [географическими структурами](../../sql-reference/data-types/geo.md). + +Возможные значения: + +- 0 — Использование типов данных для работы с географическими структурами не поддерживается. +- 1 — Использование типов данных для работы с географическими структурами поддерживается. + +Значение по умолчанию: `0`. + [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) diff --git a/docs/ru/operations/system-tables/index.md b/docs/ru/operations/system-tables/index.md index 93ea1c92068..cdea6102a81 100644 --- a/docs/ru/operations/system-tables/index.md +++ b/docs/ru/operations/system-tables/index.md @@ -9,25 +9,54 @@ toc_title: "\u0421\u0438\u0441\u0442\u0435\u043c\u043d\u044b\u0435\u0020\u0442\u Системные таблицы содержат информацию о: -- Состоянии сервера, процессов и окружении. -- Внутренних процессах сервера. +- состоянии сервера, процессов и окружении. +- внутренних процессах сервера. Системные таблицы: -- Находятся в базе данных `system`. -- Доступны только для чтения данных. -- Не могут быть удалены или изменены, но их можно отсоединить. +- находятся в базе данных `system`. +- доступны только для чтения данных. +- не могут быть удалены или изменены, но их можно отсоединить. -Системные таблицы `metric_log`, `query_log`, `query_thread_log`, `trace_log` системные таблицы хранят данные в файловой системе. Остальные системные таблицы хранят свои данные в оперативной памяти. Сервер ClickHouse создает такие системные таблицы при запуске. +Большинство системных таблиц хранят свои данные в оперативной памяти. Сервер ClickHouse создает эти системные таблицы при старте. + +В отличие от других системных таблиц, таблицы с системными логами [metric_log](../../operations/system-tables/metric_log.md), [query_log](../../operations/system-tables/query_log.md), [query_thread_log](../../operations/system-tables/query_thread_log.md), [trace_log](../../operations/system-tables/trace_log.md), [part_log](../../operations/system-tables/part_log.md), [crash_log](../../operations/system-tables/crash-log.md) и [text_log](../../operations/system-tables/text_log.md) используют движок таблиц [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) и по умолчанию хранят свои данные в файловой системе. Если удалить таблицу из файловой системы, сервер ClickHouse снова создаст пустую таблицу во время следующей записи данных. Если схема системной таблицы изменилась в новом релизе, то ClickHouse переименует текущую таблицу и создаст новую. + +Таблицы с системными логами `log` можно настроить, создав конфигурационный файл с тем же именем, что и таблица в разделе `/etc/clickhouse-server/config.d/`, или указав соответствующие элементы в `/etc/clickhouse-server/config.xml`. Настраиваться могут следующие элементы: + +- `database` — база данных, к которой принадлежит системная таблица. Эта опция на текущий момент устарела. Все системные таблицы находятся в базе данных `system`. +- `table` — таблица для добавления данных. +- `partition_by` — [ключ партиционирования](../../engines/table-engines/mergetree-family/custom-partitioning-key.md). +- `ttl` — [время жизни](../../sql-reference/statements/alter/ttl.md) таблицы. +- `flush_interval_milliseconds` — интервал сброса данных на диск, в миллисекундах. +- `engine` — полное имя движка (начиная с `ENGINE =` ) с параметрами. Эта опция противоречит `partition_by` и `ttl`. Если указать оба параметра вместе, сервер вернет ошибку и завершит работу. + +Пример: + +```xml + + + system + query_log
+ toYYYYMM(event_date) + event_date + INTERVAL 30 DAY DELETE + + 7500 + + +``` + +По умолчанию размер таблицы не ограничен. Управлять размером таблицы можно используя [TTL](../../sql-reference/statements/alter/ttl.md#manipuliatsii-s-ttl-tablitsy) для удаления устаревших записей журнала. Также вы можете использовать функцию партиционирования для таблиц `MergeTree`. ### Источники системных показателей Для сбора системных показателей сервер ClickHouse использует: -- Возможности `CAP_NET_ADMIN`. +- возможности `CAP_NET_ADMIN`. - [procfs](https://ru.wikipedia.org/wiki/Procfs) (только Linux). -**procfs** Если для сервера ClickHouse не включено `CAP_NET_ADMIN`, он пытается обратиться к `ProcfsMetricsProvider`. `ProcfsMetricsProvider` позволяет собирать системные показатели для каждого запроса (для CPU и I/O). diff --git a/docs/ru/operations/utilities/clickhouse-local.md b/docs/ru/operations/utilities/clickhouse-local.md index 2b5c9b119e2..15d069c9acf 100644 --- a/docs/ru/operations/utilities/clickhouse-local.md +++ b/docs/ru/operations/utilities/clickhouse-local.md @@ -21,7 +21,8 @@ toc_title: clickhouse-local Основной формат вызова: ``` bash -$ clickhouse-local --structure "table_structure" --input-format "format_of_incoming_data" -q "query" +$ clickhouse-local --structure "table_structure" --input-format "format_of_incoming_data" \ + --query "query" ``` Ключи команды: @@ -76,7 +77,9 @@ $ clickhouse-local --query " 1 2 ``` -А теперь давайте выведем на экран объём оперативной памяти, занимаемой пользователями (Unix): +Объём оперативной памяти, занимаемой процессами, которые запустил пользователь (Unix): + +Запрос: ``` bash $ ps aux | tail -n +2 | awk '{ printf("%s\t%s\n", $1, $4) }' \ @@ -85,6 +88,8 @@ $ ps aux | tail -n +2 | awk '{ printf("%s\t%s\n", $1, $4) }' \ FROM table GROUP BY user ORDER BY memTotal DESC FORMAT Pretty" ``` +Результат: + ``` text Read 186 rows, 4.15 KiB in 0.035 sec., 5302 rows/sec., 118.34 KiB/sec. ┏━━━━━━━━━━┳━━━━━━━━━━┓ diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index f20acaa45c3..d96f7a13bcc 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -239,7 +239,7 @@ windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) **Параметры** -- `window` — ширина скользящего окна по времени в секундах. [UInt](../../sql-reference/aggregate-functions/parametric-functions.md). +- `window` — ширина скользящего окна по времени. Единица измерения зависит от `timestamp` и может варьироваться. Должно соблюдаться условие `timestamp события cond2 <= timestamp события cond1 + window`. - `mode` - необязательный параметр. Если установлено значение `'strict'`, то функция `windowFunnel()` применяет условия только для уникальных значений. - `timestamp` — имя столбца, содержащего временные отметки. [Date](../../sql-reference/aggregate-functions/parametric-functions.md), [DateTime](../../sql-reference/aggregate-functions/parametric-functions.md#data_type-datetime) и другие параметры с типом `Integer`. В случае хранения меток времени в столбцах с типом `UInt64`, максимально допустимое значение соответствует ограничению для типа `Int64`, т.е. равно `2^63-1`. - `cond` — условия или данные, описывающие цепочку событий. [UInt8](../../sql-reference/aggregate-functions/parametric-functions.md). diff --git a/docs/ru/sql-reference/data-types/geo.md b/docs/ru/sql-reference/data-types/geo.md new file mode 100644 index 00000000000..23293b30927 --- /dev/null +++ b/docs/ru/sql-reference/data-types/geo.md @@ -0,0 +1,106 @@ +--- +toc_priority: 62 +toc_title: Географические структуры +--- + +# Типы данных для работы с географическими структурами {#geo-data-types} + +ClickHouse поддерживает типы данных для отображения географических объектов — точек (местоположений), территорий и т.п. + +!!! warning "Предупреждение" + Сейчас использование типов данных для работы с географическими структурами является экспериментальной возможностью. Чтобы использовать эти типы данных, включите настройку `allow_experimental_geo_types = 1`. + +**См. также** +- [Хранение географических структур данных](https://ru.wikipedia.org/wiki/GeoJSON). +- Настройка [allow_experimental_geo_types](../../operations/settings/settings.md#allow-experimental-geo-types). + +## Point {#point-data-type} + +Тип `Point` (точка) определяется парой координат X и Y и хранится в виде кортежа [Tuple](tuple.md)([Float64](float.md), [Float64](float.md)). + +**Пример** + +Запрос: + +```sql +SET allow_experimental_geo_types = 1; +CREATE TABLE geo_point (p Point) ENGINE = Memory(); +INSERT INTO geo_point VALUES((10, 10)); +SELECT p, toTypeName(p) FROM geo_point; +``` +Результат: + +``` text +┌─p─────┬─toTypeName(p)─┐ +│ (10,10) │ Point │ +└───────┴───────────────┘ +``` + +## Ring {#ring-data-type} + +Тип `Ring` описывает простой многоугольник без внутренних областей (дыр) и хранится в виде массива точек: [Array](array.md)([Point](#point-data-type)). + +**Пример** + +Запрос: + +```sql +SET allow_experimental_geo_types = 1; +CREATE TABLE geo_ring (r Ring) ENGINE = Memory(); +INSERT INTO geo_ring VALUES([(0, 0), (10, 0), (10, 10), (0, 10)]); +SELECT r, toTypeName(r) FROM geo_ring; +``` +Результат: + +``` text +┌─r─────────────────────────────┬─toTypeName(r)─┐ +│ [(0,0),(10,0),(10,10),(0,10)] │ Ring │ +└───────────────────────────────┴───────────────┘ +``` + +## Polygon {#polygon-data-type} + +Тип `Polygon` описывает многоугольник с внутренними областями (дырами) и хранится в виде массива: [Array](array.md)([Ring](#ring-data-type)). Первый элемент массива описывает внешний многоугольник (контур), а остальные элементы описывают дыры. + +**Пример** + +Запись в этой таблице описывает многоугольник с одной дырой: + +```sql +SET allow_experimental_geo_types = 1; +CREATE TABLE geo_polygon (pg Polygon) ENGINE = Memory(); +INSERT INTO geo_polygon VALUES([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]]); +SELECT pg, toTypeName(pg) FROM geo_polygon; +``` + +Результат: + +``` text +┌─pg────────────────────────────────────────────────────────────┬─toTypeName(pg)─┐ +│ [[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]] │ Polygon │ +└───────────────────────────────────────────────────────────────┴────────────────┘ +``` + +## MultiPolygon {#multipolygon-data-type} + +Тип `MultiPolygon` описывает элемент, состоящий из нескольких простых многоугольников (полигональную сетку). Он хранится в виде массива многоугольников: [Array](array.md)([Polygon](#polygon-data-type)). + +**Пример** + +Запись в этой таблице описывает элемент, состоящий из двух многоугольников — первый без дыр, а второй с одной дырой: + +```sql +SET allow_experimental_geo_types = 1; +CREATE TABLE geo_multipolygon (mpg MultiPolygon) ENGINE = Memory(); +INSERT INTO geo_multipolygon VALUES([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]]); +SELECT mpg, toTypeName(mpg) FROM geo_multipolygon; +``` +Result: + +``` text +┌─mpg─────────────────────────────────────────────────────────────────────────────────────────────┬─toTypeName(mpg)─┐ +│ [[[(0,0),(10,0),(10,10),(0,10)]],[[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]]] │ MultiPolygon │ +└─────────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────┘ +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/data-types/geo/) diff --git a/docs/ru/sql-reference/data-types/simpleaggregatefunction.md b/docs/ru/sql-reference/data-types/simpleaggregatefunction.md index 52f0412a177..668b579ff78 100644 --- a/docs/ru/sql-reference/data-types/simpleaggregatefunction.md +++ b/docs/ru/sql-reference/data-types/simpleaggregatefunction.md @@ -1,8 +1,9 @@ -# SimpleAggregateFunction {#data-type-simpleaggregatefunction} +# SimpleAggregateFunction(func, type) {#data-type-simpleaggregatefunction} -`SimpleAggregateFunction(name, types_of_arguments…)` data type stores current value of the aggregate function, and does not store its full state as [`AggregateFunction`](../../sql-reference/data-types/aggregatefunction.md) does. This optimization can be applied to functions for which the following property holds: the result of applying a function `f` to a row set `S1 UNION ALL S2` can be obtained by applying `f` to parts of the row set separately, and then again applying `f` to the results: `f(S1 UNION ALL S2) = f(f(S1) UNION ALL f(S2))`. This property guarantees that partial aggregation results are enough to compute the combined one, so we don’t have to store and process any extra data. +Хранит только текущее значение агрегатной функции и не сохраняет ее полное состояние, как это делает [`AggregateFunction`](../../sql-reference/data-types/aggregatefunction.md). Такая оптимизация может быть применена к функциям, которые обладают следующим свойством: результат выполнения функции `f` к набору строк `S1 UNION ALL S2` может быть получен путем выполнения `f` к отдельным частям набора строк, +а затем повторного выполнения `f` к результатам: `f(S1 UNION ALL S2) = f(f(S1) UNION ALL f(S2))`. Это свойство гарантирует, что результатов частичной агрегации достаточно для вычисления комбинированной, поэтому хранить и обрабатывать какие-либо дополнительные данные не требуется. -The following aggregate functions are supported: +Поддерживаются следующие агрегатные функции: - [`any`](../../sql-reference/aggregate-functions/reference/any.md#agg_function-any) - [`anyLast`](../../sql-reference/aggregate-functions/reference/anylast.md#anylastx) @@ -15,22 +16,24 @@ The following aggregate functions are supported: - [`groupBitXor`](../../sql-reference/aggregate-functions/reference/groupbitxor.md#groupbitxor) - [`groupArrayArray`](../../sql-reference/aggregate-functions/reference/grouparray.md#agg_function-grouparray) - [`groupUniqArrayArray`](../../sql-reference/aggregate-functions/reference/groupuniqarray.md#groupuniqarray) +- [`sumMap`](../../sql-reference/aggregate-functions/reference/summap.md#agg_functions-summap) +- [`minMap`](../../sql-reference/aggregate-functions/reference/minmap.md#agg_functions-minmap) +- [`maxMap`](../../sql-reference/aggregate-functions/reference/maxmap.md#agg_functions-maxmap) -Values of the `SimpleAggregateFunction(func, Type)` look and stored the same way as `Type`, so you do not need to apply functions with `-Merge`/`-State` suffixes. `SimpleAggregateFunction` has better performance than `AggregateFunction` with same aggregation function. +!!! note "Примечание" + Значения `SimpleAggregateFunction(func, Type)` отображаются и хранятся так же, как и `Type`, поэтому комбинаторы [-Merge](../../sql-reference/aggregate-functions/combinators.md#aggregate_functions_combinators-merge) и [-State]((../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-state) не требуются. + + `SimpleAggregateFunction` имеет лучшую производительность, чем `AggregateFunction` с той же агрегатной функцией. -**Parameters** +**Параметры** -- Name of the aggregate function. -- Types of the aggregate function arguments. +- `func` — имя агрегатной функции. +- `type` — типы аргументов агрегатной функции. -**Example** +**Пример** ``` sql -CREATE TABLE t -( - column1 SimpleAggregateFunction(sum, UInt64), - column2 SimpleAggregateFunction(any, String) -) ENGINE = ... +CREATE TABLE simple (id UInt64, val SimpleAggregateFunction(sum, Double)) ENGINE=AggregatingMergeTree ORDER BY id; ``` -[Original article](https://clickhouse.tech/docs/en/data_types/simpleaggregatefunction/) +[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/data-types/simpleaggregatefunction/) diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 3bb11b638b2..77275b65a05 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -572,7 +572,7 @@ SOURCE(CLICKHOUSE( или ``` sql -SOURCE(MONGO( +SOURCE(MONGODB( host 'localhost' port 27017 user '' diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index 85d7c275f27..4fab11e39a8 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -323,7 +323,9 @@ WITH toDateTime64('2020-01-01 10:20:30.999', 3) AS dt64 SELECT toStartOfSecond(d Переводит дату-с-временем или дату в число типа UInt16, содержащее номер ISO года. ISO год отличается от обычного года, потому что в соответствии с [ISO 8601:1988](https://en.wikipedia.org/wiki/ISO_8601) ISO год начинается необязательно первого января. -Пример: +**Пример** + +Запрос: ```sql SELECT @@ -331,6 +333,9 @@ SELECT toYear(date), toISOYear(date) ``` + +Результат: + ```text ┌───────date─┬─toYear(toDate('2017-01-01'))─┬─toISOYear(toDate('2017-01-01'))─┐ │ 2017-01-01 │ 2017 │ 2016 │ @@ -344,12 +349,18 @@ SELECT 1 Января 2017 г. - воскресение, т.е. первая ISO неделя 2017 года началась в понедельник 2 января, поэтому 1 января 2017 это последняя неделя 2016 года. +**Пример** + +Запрос: + ```sql SELECT toISOWeek(toDate('2017-01-01')) AS ISOWeek20170101, toISOWeek(toDate('2017-01-02')) AS ISOWeek20170102 ``` +Результат: + ```text ┌─ISOWeek20170101─┬─ISOWeek20170102─┐ │ 52 │ 1 │ @@ -386,10 +397,14 @@ SELECT **Пример** +Запрос: + ```sql SELECT toDate('2016-12-27') AS date, toWeek(date) AS week0, toWeek(date,1) AS week1, toWeek(date,9) AS week9; ``` +Результат: + ```text ┌───────date─┬─week0─┬─week1─┬─week9─┐ │ 2016-12-27 │ 52 │ 52 │ 1 │ @@ -405,10 +420,14 @@ SELECT toDate('2016-12-27') AS date, toWeek(date) AS week0, toWeek(date,1) AS we **Пример** +Запрос: + ```sql SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(date,1) AS yearWeek1, toYearWeek(date,9) AS yearWeek9; ``` +Результат: + ```text ┌───────date─┬─yearWeek0─┬─yearWeek1─┬─yearWeek9─┐ │ 2016-12-27 │ 201652 │ 201652 │ 201701 │ @@ -591,7 +610,7 @@ dateDiff('unit', startdate, enddate, [timezone]) SELECT dateDiff('hour', toDateTime('2018-01-01 22:00:00'), toDateTime('2018-01-02 23:00:00')); ``` -Ответ: +Результат: ``` text ┌─dateDiff('hour', toDateTime('2018-01-01 22:00:00'), toDateTime('2018-01-02 23:00:00'))─┐ @@ -672,10 +691,10 @@ formatDateTime(Time, Format\[, Timezone\]) Запрос: ``` sql -SELECT formatDateTime(toDate('2010-01-04'), '%g') +SELECT formatDateTime(toDate('2010-01-04'), '%g'); ``` -Ответ: +Результат: ``` ┌─formatDateTime(toDate('2010-01-04'), '%g')─┐ @@ -683,4 +702,43 @@ SELECT formatDateTime(toDate('2010-01-04'), '%g') └────────────────────────────────────────────┘ ``` +## FROM\_UNIXTIME {#fromunixtime} + +Функция преобразует Unix timestamp в календарную дату и время. + +**Примеры** + +Если указан только один аргумент типа [Integer](../../sql-reference/data-types/int-uint.md), то функция действует так же, как [toDateTime](../../sql-reference/functions/type-conversion-functions.md#todatetime), и возвращает тип [DateTime](../../sql-reference/data-types/datetime.md). + +Запрос: + +```sql +SELECT FROM_UNIXTIME(423543535); +``` + +Результат: + +```text +┌─FROM_UNIXTIME(423543535)─┐ +│ 1983-06-04 10:58:55 │ +└──────────────────────────┘ +``` + +В случае, когда есть два аргумента: первый типа [Integer](../../sql-reference/data-types/int-uint.md) или [DateTime](../../sql-reference/data-types/datetime.md), а второй является строкой постоянного формата — функция работает также, как [formatDateTime](#formatdatetime), и возвращает значение типа [String](../../sql-reference/data-types/string.md#string). + + +Запрос: + +```sql +SELECT FROM_UNIXTIME(1234334543, '%Y-%m-%d %R:%S') AS DateTime; +``` + +Результат: + +```text +┌─DateTime────────────┐ +│ 2009-02-11 14:42:23 │ +└─────────────────────┘ +``` + [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/date_time_functions/) diff --git a/docs/ru/sql-reference/functions/encryption-functions.md b/docs/ru/sql-reference/functions/encryption-functions.md index f1f6516d453..0216a6b2356 100644 --- a/docs/ru/sql-reference/functions/encryption-functions.md +++ b/docs/ru/sql-reference/functions/encryption-functions.md @@ -11,7 +11,7 @@ toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438 \u0434\u043b\u044f \u0448 Длина инициализирующего вектора всегда 16 байт (лишнии байты игнорируются). -Обратите внимание, что эти функции работают медленно. +Обратите внимание, что до версии Clickhouse 21.1 эти функции работали медленно. ## encrypt {#encrypt} @@ -41,7 +41,7 @@ encrypt('mode', 'plaintext', 'key' [, iv, aad]) **Возвращаемое значение** -- Зашифрованная строка. [String](../../sql-reference/data-types/string.md#string). +- Бинарная зашифрованная строка. [String](../../sql-reference/data-types/string.md#string). **Примеры** @@ -52,57 +52,38 @@ encrypt('mode', 'plaintext', 'key' [, iv, aad]) ``` sql CREATE TABLE encryption_test ( - input String, - key String DEFAULT unhex('fb9958e2e897ef3fdb49067b51a24af645b3626eed2f9ea1dc7fd4dd71b7e38f9a68db2a3184f952382c783785f9d77bf923577108a88adaacae5c141b1576b0'), - iv String DEFAULT unhex('8CA3554377DFF8A369BC50A89780DD85'), - key32 String DEFAULT substring(key, 1, 32), - key24 String DEFAULT substring(key, 1, 24), - key16 String DEFAULT substring(key, 1, 16) -) Engine = Memory; + `comment` String, + `secret` String +) +ENGINE = Memory; ``` -Вставим эти данные: +Вставим некоторые данные (замечание: не храните ключи или инициализирующие векторы в базе данных, так как это компрометирует всю концепцию шифрования), также хранение "подсказок" небезопасно и используется только для наглядности: Запрос: ``` sql -INSERT INTO encryption_test (input) VALUES (''), ('text'), ('What Is ClickHouse?'); +INSERT INTO encryption_test VALUES('aes-256-cfb128 no IV', encrypt('aes-256-cfb128', 'Secret', '12345678910121314151617181920212')),\ +('aes-256-cfb128 no IV, different key', encrypt('aes-256-cfb128', 'Secret', 'keykeykeykeykeykeykeykeykeykeyke')),\ +('aes-256-cfb128 with IV', encrypt('aes-256-cfb128', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv')),\ +('aes-256-cbc no IV', encrypt('aes-256-cbc', 'Secret', '12345678910121314151617181920212')); ``` -Пример без `iv`: - Запрос: ``` sql -SELECT 'aes-128-ecb' AS mode, hex(encrypt(mode, input, key16)) FROM encryption_test; +SELECT comment, hex(secret) FROM encryption_test; ``` Результат: ``` text -┌─mode────────┬─hex(encrypt('aes-128-ecb', input, key16))────────────────────────┐ -│ aes-128-ecb │ 4603E6862B0D94BBEC68E0B0DF51D60F │ -│ aes-128-ecb │ 3004851B86D3F3950672DE7085D27C03 │ -│ aes-128-ecb │ E807F8C8D40A11F65076361AFC7D8B68D8658C5FAA6457985CAA380F16B3F7E4 │ -└─────────────┴──────────────────────────────────────────────────────────────────┘ -``` - -Пример с `iv`: - -Запрос: - -``` sql -SELECT 'aes-256-ctr' AS mode, hex(encrypt(mode, input, key32, iv)) FROM encryption_test; -``` - -Результат: - -``` text -┌─mode────────┬─hex(encrypt('aes-256-ctr', input, key32, iv))─┐ -│ aes-256-ctr │ │ -│ aes-256-ctr │ 7FB039F7 │ -│ aes-256-ctr │ 5CBD20F7ABD3AC41FCAA1A5C0E119E2B325949 │ -└─────────────┴───────────────────────────────────────────────┘ +┌─comment─────────────────────────────┬─hex(secret)──────────────────────┐ +│ aes-256-cfb128 no IV │ B4972BDC4459 │ +│ aes-256-cfb128 no IV, different key │ 2FF57C092DC9 │ +│ aes-256-cfb128 with IV │ 5E6CB398F653 │ +│ aes-256-cbc no IV │ 1BC0629A92450D9E73A00E7D02CF4142 │ +└─────────────────────────────────────┴──────────────────────────────────┘ ``` Пример в режиме `-gcm`: @@ -110,41 +91,27 @@ SELECT 'aes-256-ctr' AS mode, hex(encrypt(mode, input, key32, iv)) FROM encrypti Запрос: ``` sql -SELECT 'aes-256-gcm' AS mode, hex(encrypt(mode, input, key32, iv)) FROM encryption_test; +INSERT INTO encryption_test VALUES('aes-256-gcm', encrypt('aes-256-gcm', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv')), \ +('aes-256-gcm with AAD', encrypt('aes-256-gcm', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv', 'aad')); + +SELECT comment, hex(secret) FROM encryption_test WHERE comment LIKE '%gcm%'; ``` Результат: ``` text -┌─mode────────┬─hex(encrypt('aes-256-gcm', input, key32, iv))──────────────────────────┐ -│ aes-256-gcm │ E99DBEBC01F021758352D7FBD9039EFA │ -│ aes-256-gcm │ 8742CE3A7B0595B281C712600D274CA881F47414 │ -│ aes-256-gcm │ A44FD73ACEB1A64BDE2D03808A2576EDBB60764CC6982DB9AF2C33C893D91B00C60DC5 │ -└─────────────┴────────────────────────────────────────────────────────────────────────┘ -``` - -Пример в режиме `-gcm` и с `aad`: - -Запрос: - -``` sql -SELECT 'aes-192-gcm' AS mode, hex(encrypt(mode, input, key24, iv, 'AAD')) FROM encryption_test; -``` - -Результат: - -``` text -┌─mode────────┬─hex(encrypt('aes-192-gcm', input, key24, iv, 'AAD'))───────────────────┐ -│ aes-192-gcm │ 04C13E4B1D62481ED22B3644595CB5DB │ -│ aes-192-gcm │ 9A6CF0FD2B329B04EAD18301818F016DF8F77447 │ -│ aes-192-gcm │ B961E9FD9B940EBAD7ADDA75C9F198A40797A5EA1722D542890CC976E21113BBB8A7AA │ -└─────────────┴────────────────────────────────────────────────────────────────────────┘ +┌─comment──────────────┬─hex(secret)──────────────────────────────────┐ +│ aes-256-gcm │ A8A3CCBC6426CFEEB60E4EAE03D3E94204C1B09E0254 │ +│ aes-256-gcm with AAD │ A8A3CCBC6426D9A1017A0A932322F1852260A4AD6837 │ +└──────────────────────┴──────────────────────────────────────────────┘ ``` ## aes_encrypt_mysql {#aes_encrypt_mysql} Совместима с шифрованием myqsl, результат может быть расшифрован функцией [AES_DECRYPT](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_aes-decrypt). +При одинаковых входящих значениях зашифрованный текст будет совпадать с результатом, возвращаемым функцией `encrypt`. Однако если `key` или `iv` длиннее, чем должны быть, `aes_encrypt_mysql` будет работать аналогично функции `aes_encrypt` в MySQL: свернет ключ и проигнорирует лишнюю часть `iv`. + Функция поддерживает шифрофание данных следующими режимами: - aes-128-ecb, aes-192-ecb, aes-256-ecb @@ -156,7 +123,7 @@ SELECT 'aes-192-gcm' AS mode, hex(encrypt(mode, input, key24, iv, 'AAD')) FROM e **Синтаксис** -```sql +``` sql aes_encrypt_mysql('mode', 'plaintext', 'key' [, iv]) ``` @@ -164,78 +131,96 @@ aes_encrypt_mysql('mode', 'plaintext', 'key' [, iv]) - `mode` — режим шифрования. [String](../../sql-reference/data-types/string.md#string). - `plaintext` — текст, который будет зашифрован. [String](../../sql-reference/data-types/string.md#string). -- `key` — ключ шифрования. [String](../../sql-reference/data-types/string.md#string). -- `iv` — инициализирующий вектор. Необязателен. [String](../../sql-reference/data-types/string.md#string). +- `key` — ключ шифрования. Если ключ длиннее, чем требует режим шифрования, производится специфичная для MySQL свертка ключа. [String](../../sql-reference/data-types/string.md#string). +- `iv` — инициализирующий вектор. Необязателен, учитываются только первые 16 байтов. [String](../../sql-reference/data-types/string.md#string). **Возвращаемое значение** -- Зашифрованная строка. [String](../../sql-reference/data-types/string.md#string). +- Бинарная зашифрованная строка. [String](../../sql-reference/data-types/string.md#string). **Примеры** -Создадим такую таблицу: +При одинаковых входящих значениях результаты шифрования у функций `encrypt` и `aes_encrypt_mysql` совпадают. Запрос: ``` sql -CREATE TABLE encryption_test -( - input String, - key String DEFAULT unhex('fb9958e2e897ef3fdb49067b51a24af645b3626eed2f9ea1dc7fd4dd71b7e38f9a68db2a3184f952382c783785f9d77bf923577108a88adaacae5c141b1576b0'), - iv String DEFAULT unhex('8CA3554377DFF8A369BC50A89780DD85'), - key32 String DEFAULT substring(key, 1, 32), - key24 String DEFAULT substring(key, 1, 24), - key16 String DEFAULT substring(key, 1, 16) -) Engine = Memory; -``` - -Вставим эти данные: - -Запрос: - -``` sql -INSERT INTO encryption_test (input) VALUES (''), ('text'), ('What Is ClickHouse?'); -``` - -Пример без `iv`: - -Запрос: - -``` sql -SELECT 'aes-128-cbc' AS mode, hex(aes_encrypt_mysql(mode, input, key32)) FROM encryption_test; +SELECT encrypt('aes-256-cfb128', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv') = aes_encrypt_mysql('aes-256-cfb128', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv') AS ciphertexts_equal; ``` Результат: ``` text -┌─mode────────┬─hex(aes_encrypt_mysql('aes-128-cbc', input, key32))──────────────┐ -│ aes-128-cbc │ FEA8CFDE6EE2C6E7A2CC6ADDC9F62C83 │ -│ aes-128-cbc │ 78B16CD4BE107660156124C5FEE6454A │ -│ aes-128-cbc │ 67C0B119D96F18E2823968D42871B3D179221B1E7EE642D628341C2B29BA2E18 │ -└─────────────┴──────────────────────────────────────────────────────────────────┘ +┌─ciphertexts_equal─┐ +│ 1 │ +└───────────────────┘ ``` -Пример с `iv`: +Функция `encrypt` генерирует исключение, если `key` или `iv` длиннее чем нужно: Запрос: ``` sql -SELECT 'aes-256-cfb128' AS mode, hex(aes_encrypt_mysql(mode, input, key32, iv)) FROM encryption_test; +SELECT encrypt('aes-256-cfb128', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123'); ``` Результат: ``` text -┌─mode───────────┬─hex(aes_encrypt_mysql('aes-256-cfb128', input, key32, iv))─┐ -│ aes-256-cfb128 │ │ -│ aes-256-cfb128 │ 7FB039F7 │ -│ aes-256-cfb128 │ 5CBD20F7ABD3AC41FCAA1A5C0E119E2BB5174F │ -└────────────────┴────────────────────────────────────────────────────────────┘ +Received exception from server (version 21.1.2): +Code: 36. DB::Exception: Received from localhost:9000. DB::Exception: Invalid key size: 33 expected 32: While processing encrypt('aes-256-cfb128', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123'). +``` + +Однако функция `aes_encrypt_mysql` в аналогичном случае возвращает результат, который может быть обработан MySQL: + +Запрос: + +``` sql +SELECT hex(aes_encrypt_mysql('aes-256-cfb128', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123')) AS ciphertext; +``` + +Результат: + +```text +┌─ciphertext───┐ +│ 24E9E4966469 │ +└──────────────┘ +``` + +Если передать `iv` еще длиннее, результат останется таким же: + +Запрос: + +``` sql +SELECT hex(aes_encrypt_mysql('aes-256-cfb128', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123456')) AS ciphertext +``` + +Результат: + +``` text +┌─ciphertext───┐ +│ 24E9E4966469 │ +└──────────────┘ +``` + +Это совпадает с результатом, возвращаемым MySQL при таких же входящих значениях: + +``` sql +mysql> SET block_encryption_mode='aes-256-cfb128'; +Query OK, 0 rows affected (0.00 sec) + +mysql> SELECT aes_encrypt('Secret', '123456789101213141516171819202122', 'iviviviviviviviv123456') as ciphertext; ++------------------------+ +| ciphertext | ++------------------------+ +| 0x24E9E4966469 | ++------------------------+ +1 row in set (0.00 sec) ``` ## decrypt {#decrypt} -Функция поддерживает расшифровку данных следующими режимами: +Функция расшифровывает зашифрованный текст и может работать в следующих режимах: - aes-128-ecb, aes-192-ecb, aes-256-ecb - aes-128-cbc, aes-192-cbc, aes-256-cbc @@ -247,7 +232,7 @@ SELECT 'aes-256-cfb128' AS mode, hex(aes_encrypt_mysql(mode, input, key32, iv)) **Синтаксис** -```sql +``` sql decrypt('mode', 'ciphertext', 'key' [, iv, aad]) ``` @@ -265,52 +250,58 @@ decrypt('mode', 'ciphertext', 'key' [, iv, aad]) **Примеры** -Создадим такую таблицу: +Рассмотрим таблицу из примера для функции [encrypt](#encrypt). Запрос: ``` sql -CREATE TABLE encryption_test -( - input String, - key String DEFAULT unhex('fb9958e2e897ef3fdb49067b51a24af645b3626eed2f9ea1dc7fd4dd71b7e38f9a68db2a3184f952382c783785f9d77bf923577108a88adaacae5c141b1576b0'), - iv String DEFAULT unhex('8CA3554377DFF8A369BC50A89780DD85'), - key32 String DEFAULT substring(key, 1, 32), - key24 String DEFAULT substring(key, 1, 24), - key16 String DEFAULT substring(key, 1, 16) -) Engine = Memory; -``` - -Вставим эти данные: - -Запрос: - -``` sql -INSERT INTO encryption_test (input) VALUES (''), ('text'), ('What Is ClickHouse?'); -``` - -Запрос: - -``` sql - -SELECT 'aes-128-ecb' AS mode, decrypt(mode, encrypt(mode, input, key16), key16) FROM encryption_test; +SELECT comment, hex(secret) FROM encryption_test; ``` Результат: -```text -┌─mode────────┬─decrypt('aes-128-ecb', encrypt('aes-128-ecb', input, key16), key16)─┐ -│ aes-128-ecb │ │ -│ aes-128-ecb │ text │ -│ aes-128-ecb │ What Is ClickHouse? │ -└─────────────┴─────────────────────────────────────────────────────────────────────┘ +``` text +┌─comment──────────────┬─hex(secret)──────────────────────────────────┐ +│ aes-256-gcm │ A8A3CCBC6426CFEEB60E4EAE03D3E94204C1B09E0254 │ +│ aes-256-gcm with AAD │ A8A3CCBC6426D9A1017A0A932322F1852260A4AD6837 │ +└──────────────────────┴──────────────────────────────────────────────┘ +┌─comment─────────────────────────────┬─hex(secret)──────────────────────┐ +│ aes-256-cfb128 no IV │ B4972BDC4459 │ +│ aes-256-cfb128 no IV, different key │ 2FF57C092DC9 │ +│ aes-256-cfb128 with IV │ 5E6CB398F653 │ +│ aes-256-cbc no IV │ 1BC0629A92450D9E73A00E7D02CF4142 │ +└─────────────────────────────────────┴──────────────────────────────────┘ ``` +Теперь попытаемся расшифровать эти данные: + +Запрос: + +``` sql +SELECT comment, decrypt('aes-256-cfb128', secret, '12345678910121314151617181920212') as plaintext FROM encryption_test; +``` + +Результат: + +``` text +┌─comment─────────────────────────────┬─plaintext─┐ +│ aes-256-cfb128 no IV │ Secret │ +│ aes-256-cfb128 no IV, different key │ �4� + � │ +│ aes-256-cfb128 with IV │ ���6�~ │ + │aes-256-cbc no IV │ �2*4�h3c�4w��@ +└─────────────────────────────────────┴───────────┘ +``` + +Обратите внимание, что только часть данных была расшифрована верно. Оставшаяся часть расшифрована некорректно, так как при шифровании использовались другие значения `mode`, `key`, или `iv`. + ## aes_decrypt_mysql {#aes_decrypt_mysql} Совместима с шифрованием myqsl и может расшифровать данные, зашифрованные функцией [AES_ENCRYPT](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_aes-encrypt). -Функция поддерживает расшифровку данных следующими режимами: +При одинаковых входящих значениях расшифрованный текст будет совпадать с результатом, возвращаемым функцией `decrypt`. Однако если `key` или `iv` длиннее, чем должны быть, `aes_decrypt_mysql` будет работать аналогично функции `aes_decrypt` в MySQL: свернет ключ и проигнорирует лишнюю часть `iv`. + +Функция поддерживает расшифровку данных в следующих режимах: - aes-128-ecb, aes-192-ecb, aes-256-ecb - aes-128-cbc, aes-192-cbc, aes-256-cbc @@ -321,7 +312,7 @@ SELECT 'aes-128-ecb' AS mode, decrypt(mode, encrypt(mode, input, key16), key16) **Синтаксис** -```sql +``` sql aes_decrypt_mysql('mode', 'ciphertext', 'key' [, iv]) ``` @@ -332,51 +323,39 @@ aes_decrypt_mysql('mode', 'ciphertext', 'key' [, iv]) - `key` — ключ шифрования. [String](../../sql-reference/data-types/string.md#string). - `iv` — инициализирующий вектор. Необязателен. [String](../../sql-reference/data-types/string.md#string). - **Возвращаемое значение** - Расшифрованная строка. [String](../../sql-reference/data-types/string.md#string). **Примеры** -Создадим такую таблицу: +Расшифруем данные, которые до этого были зашифрованы в MySQL: -Запрос: ``` sql -CREATE TABLE encryption_test -( - input String, - key String DEFAULT unhex('fb9958e2e897ef3fdb49067b51a24af645b3626eed2f9ea1dc7fd4dd71b7e38f9a68db2a3184f952382c783785f9d77bf923577108a88adaacae5c141b1576b0'), - iv String DEFAULT unhex('8CA3554377DFF8A369BC50A89780DD85'), - key32 String DEFAULT substring(key, 1, 32), - key24 String DEFAULT substring(key, 1, 24), - key16 String DEFAULT substring(key, 1, 16) -) Engine = Memory; -``` +mysql> SET block_encryption_mode='aes-256-cfb128'; +Query OK, 0 rows affected (0.00 sec) -Вставим эти данные: - -Запрос: - -``` sql -INSERT INTO encryption_test (input) VALUES (''), ('text'), ('What Is ClickHouse?'); +mysql> SELECT aes_encrypt('Secret', '123456789101213141516171819202122', 'iviviviviviviviv123456') as ciphertext; ++------------------------+ +| ciphertext | ++------------------------+ +| 0x24E9E4966469 | ++------------------------+ +1 row in set (0.00 sec) ``` Запрос: ``` sql -SELECT 'aes-128-cbc' AS mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key), key) FROM encryption_test; +SELECT aes_decrypt_mysql('aes-256-cfb128', unhex('24E9E4966469'), '123456789101213141516171819202122', 'iviviviviviviviv123456') AS plaintext; ``` Результат: ``` text -┌─mode────────┬─aes_decrypt_mysql('aes-128-cbc', aes_encrypt_mysql('aes-128-cbc', input, key), key)─┐ -│ aes-128-cbc │ │ -│ aes-128-cbc │ text │ -│ aes-128-cbc │ What Is ClickHouse? │ -└─────────────┴─────────────────────────────────────────────────────────────────────────────────────┘ +┌─plaintext─┐ +│ Secret │ +└───────────┘ ``` - [Original article](https://clickhouse.tech/docs/ru/sql-reference/functions/encryption_functions/) diff --git a/docs/ru/sql-reference/functions/tuple-map-functions.md b/docs/ru/sql-reference/functions/tuple-map-functions.md index a36613280a1..696fdb9e5ae 100644 --- a/docs/ru/sql-reference/functions/tuple-map-functions.md +++ b/docs/ru/sql-reference/functions/tuple-map-functions.md @@ -176,4 +176,129 @@ select mapPopulateSeries([1,2,4], [11,22,44], 5) as res, toTypeName(res) as type └──────────────────────────────┴───────────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/query_language/functions/tuple-map-functions/) +## mapContains {#mapcontains} + +Определяет, содержит ли контейнер `map` ключ `key`. + +**Синтаксис** + +``` sql +mapContains(map, key) +``` + +**Параметры** + +- `map` — контейнер Map. [Map](../../sql-reference/data-types/map.md). +- `key` — ключ. Тип соответстует типу ключей параметра `map`. + +**Возвращаемое значение** + +- `1` если `map` включает `key`, иначе `0`. + +Тип: [UInt8](../../sql-reference/data-types/int-uint.md). + +**Пример** + +Запрос: + +```sql +CREATE TABLE test (a Map(String,String)) ENGINE = Memory; + +INSERT INTO test VALUES ({'name':'eleven','age':'11'}), ({'number':'twelve','position':'6.0'}); + +SELECT mapContains(a, 'name') FROM test; + +``` + +Результат: + +```text +┌─mapContains(a, 'name')─┐ +│ 1 │ +│ 0 │ +└────────────────────────┘ +``` + +## mapKeys {#mapkeys} + +Возвращает все ключи контейнера `map`. + +**Синтаксис** + +```sql +mapKeys(map) +``` + +**Параметры** + +- `map` — контейнер Map. [Map](../../sql-reference/data-types/map.md). + +**Возвращаемое значение** + +- Массив со всеми ключами контейнера `map`. + +Тип: [Array](../../sql-reference/data-types/array.md). + +**Пример** + +Запрос: + +```sql +CREATE TABLE test (a Map(String,String)) ENGINE = Memory; + +INSERT INTO test VALUES ({'name':'eleven','age':'11'}), ({'number':'twelve','position':'6.0'}); + +SELECT mapKeys(a) FROM test; +``` + +Результат: + +```text +┌─mapKeys(a)────────────┐ +│ ['name','age'] │ +│ ['number','position'] │ +└───────────────────────┘ +``` + +## mapValues {#mapvalues} + +Возвращает все значения контейнера `map`. + +**Синтаксис** + +```sql +mapKeys(map) +``` + +**Параметры** + +- `map` — контейнер Map. [Map](../../sql-reference/data-types/map.md). + +**Возвращаемое значение** + +- Массив со всеми значениями контейнера `map`. + +Тип: [Array](../../sql-reference/data-types/array.md). + +**Примеры** + +Запрос: + +```sql +CREATE TABLE test (a Map(String,String)) ENGINE = Memory; + +INSERT INTO test VALUES ({'name':'eleven','age':'11'}), ({'number':'twelve','position':'6.0'}); + +SELECT mapValues(a) FROM test; +``` + +Результат: + +```text +┌─mapValues(a)─────┐ +│ ['eleven','11'] │ +│ ['twelve','6.0'] │ +└──────────────────┘ +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/functions/tuple-map-functions/) diff --git a/docs/ru/sql-reference/statements/select/all.md b/docs/ru/sql-reference/statements/select/all.md new file mode 100644 index 00000000000..4049d77a173 --- /dev/null +++ b/docs/ru/sql-reference/statements/select/all.md @@ -0,0 +1,22 @@ +--- +toc_title: ALL +--- + +# Секция ALL {#select-all} + +Если в таблице несколько совпадающих строк, то `ALL` возвращает все из них. Поведение запроса `SELECT ALL` точно такое же, как и `SELECT` без аргумента `DISTINCT`. Если указаны оба аргумента: `ALL` и `DISTINCT`, функция вернет исключение. + + +`ALL` может быть указан внутри агрегатной функции, например, результат выполнения запроса: + +```sql +SELECT sum(ALL number) FROM numbers(10); +``` + +равен результату выполнения запроса: + +```sql +SELECT sum(number) FROM numbers(10); +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/select/all) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index f501e182cb7..8fcdfc4beac 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -82,6 +82,7 @@ #if defined(OS_LINUX) # include +# include # include #endif @@ -480,16 +481,26 @@ int Server::main(const std::vector & /*args*/) } else { - throw Exception(ErrorCodes::CORRUPTED_DATA, - "Calculated checksum of the ClickHouse binary ({0}) does not correspond" - " to the reference checksum stored in the binary ({1})." - " It may indicate one of the following:" - " - the file {2} was changed just after startup;" - " - the file {2} is damaged on disk due to faulty hardware;" - " - the loaded executable is damaged in memory due to faulty hardware;" - " - the file {2} was intentionally modified;" - " - logical error in code." - , calculated_binary_hash, stored_binary_hash, executable_path); + /// If program is run under debugger, ptrace will fail. + if (ptrace(PTRACE_TRACEME, 0, nullptr, nullptr) == -1) + { + /// Program is run under debugger. Modification of it's binary image is ok for breakpoints. + LOG_WARNING(log, "Server is run under debugger and its binary image is modified (most likely with breakpoints).", + calculated_binary_hash); + } + else + { + throw Exception(ErrorCodes::CORRUPTED_DATA, + "Calculated checksum of the ClickHouse binary ({0}) does not correspond" + " to the reference checksum stored in the binary ({1})." + " It may indicate one of the following:" + " - the file {2} was changed just after startup;" + " - the file {2} is damaged on disk due to faulty hardware;" + " - the loaded executable is damaged in memory due to faulty hardware;" + " - the file {2} was intentionally modified;" + " - logical error in code." + , calculated_binary_hash, stored_binary_hash, executable_path); + } } } else diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.h b/src/AggregateFunctions/AggregateFunctionSumMap.h index f6a473546f9..3233199f01e 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.h +++ b/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -118,6 +118,8 @@ public: WhichDataType value_type_to_check(value_type); /// Do not promote decimal because of implementation issues of this function design + /// Currently we cannot get result column type in case of decimal we cannot get decimal scale + /// in method void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override /// If we decide to make this function more efficient we should promote decimal type during summ if (value_type_to_check.isDecimal()) result_type = value_type_without_nullable; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 215a13cce1a..b80bcfdf4d4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -101,8 +101,8 @@ endif() list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD}) list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON}) -list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/replicate.cpp) -list (APPEND dbms_headers Functions/IFunctionImpl.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/replicate.h) +list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/replicate.cpp Functions/FunctionsLogical.cpp) +list (APPEND dbms_headers Functions/IFunctionImpl.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/replicate.h Functions/FunctionsLogical.h) list (APPEND dbms_sources AggregateFunctions/AggregateFunctionFactory.cpp diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index a9d094fc2c7..f81e377da2b 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -538,12 +538,14 @@ M(569, MULTIPLE_COLUMNS_SERIALIZED_TO_SAME_PROTOBUF_FIELD) \ M(570, DATA_TYPE_INCOMPATIBLE_WITH_PROTOBUF_FIELD) \ M(571, DATABASE_REPLICATION_FAILED) \ - M(572, EPOLL_ERROR) \ + M(572, TOO_MANY_QUERY_PLAN_OPTIMIZATIONS) \ + M(573, EPOLL_ERROR) \ + \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ M(1001, STD_EXCEPTION) \ M(1002, UNKNOWN_EXCEPTION) \ - M(1003, INVALID_SHARD_ID) + M(1003, INVALID_SHARD_ID) \ /* See END */ diff --git a/src/Common/ZooKeeper/IKeeper.h b/src/Common/ZooKeeper/IKeeper.h index c53ea60ec7c..9ff37a7045d 100644 --- a/src/Common/ZooKeeper/IKeeper.h +++ b/src/Common/ZooKeeper/IKeeper.h @@ -391,6 +391,9 @@ public: virtual void multi( const Requests & requests, MultiCallback callback) = 0; + + /// Expire session and finish all pending requests + virtual void finalize() = 0; }; } diff --git a/src/Common/ZooKeeper/TestKeeper.h b/src/Common/ZooKeeper/TestKeeper.h index ca9f584304f..b46f98c0074 100644 --- a/src/Common/ZooKeeper/TestKeeper.h +++ b/src/Common/ZooKeeper/TestKeeper.h @@ -30,7 +30,7 @@ using TestKeeperRequestPtr = std::shared_ptr; * * NOTE: You can add various failure modes for better testing. */ -class TestKeeper : public IKeeper +class TestKeeper final : public IKeeper { public: TestKeeper(const String & root_path_, Poco::Timespan operation_timeout_); @@ -83,6 +83,7 @@ public: const Requests & requests, MultiCallback callback) override; + void finalize() override; struct Node { @@ -130,7 +131,6 @@ private: void pushRequest(RequestInfo && request); - void finalize(); ThreadFromGlobalPool processing_thread; diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index a1c6eb9b481..330985e1599 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -44,7 +44,7 @@ static void check(Coordination::Error code, const std::string & path) } -void ZooKeeper::init(const std::string & implementation_, const std::string & hosts_, const std::string & identity_, +void ZooKeeper::init(const std::string & implementation_, const Strings & hosts_, const std::string & identity_, int32_t session_timeout_ms_, int32_t operation_timeout_ms_, const std::string & chroot_) { log = &Poco::Logger::get("ZooKeeper"); @@ -60,13 +60,16 @@ void ZooKeeper::init(const std::string & implementation_, const std::string & ho if (hosts.empty()) throw KeeperException("No hosts passed to ZooKeeper constructor.", Coordination::Error::ZBADARGUMENTS); - std::vector hosts_strings; - splitInto<','>(hosts_strings, hosts); Coordination::ZooKeeper::Nodes nodes; - nodes.reserve(hosts_strings.size()); + nodes.reserve(hosts.size()); + + Strings shuffled_hosts = hosts; + /// Shuffle the hosts to distribute the load among ZooKeeper nodes. + pcg64 generator(randomSeed()); + std::shuffle(shuffled_hosts.begin(), shuffled_hosts.end(), generator); bool dns_error = false; - for (auto & host_string : hosts_strings) + for (auto & host_string : shuffled_hosts) { try { @@ -109,9 +112,9 @@ void ZooKeeper::init(const std::string & implementation_, const std::string & ho Poco::Timespan(0, operation_timeout_ms_ * 1000)); if (chroot.empty()) - LOG_TRACE(log, "Initialized, hosts: {}", hosts); + LOG_TRACE(log, "Initialized, hosts: {}", fmt::join(hosts, ",")); else - LOG_TRACE(log, "Initialized, hosts: {}, chroot: {}", hosts, chroot); + LOG_TRACE(log, "Initialized, hosts: {}, chroot: {}", fmt::join(hosts, ","), chroot); } else if (implementation == "testkeeper") { @@ -128,7 +131,16 @@ void ZooKeeper::init(const std::string & implementation_, const std::string & ho throw KeeperException("Zookeeper root doesn't exist. You should create root node " + chroot + " before start.", Coordination::Error::ZNONODE); } -ZooKeeper::ZooKeeper(const std::string & hosts_, const std::string & identity_, int32_t session_timeout_ms_, +ZooKeeper::ZooKeeper(const std::string & hosts_string, const std::string & identity_, int32_t session_timeout_ms_, + int32_t operation_timeout_ms_, const std::string & chroot_, const std::string & implementation_) +{ + Strings hosts_strings; + splitInto<','>(hosts_strings, hosts_string); + + init(implementation_, hosts_strings, identity_, session_timeout_ms_, operation_timeout_ms_, chroot_); +} + +ZooKeeper::ZooKeeper(const Strings & hosts_, const std::string & identity_, int32_t session_timeout_ms_, int32_t operation_timeout_ms_, const std::string & chroot_, const std::string & implementation_) { init(implementation_, hosts_, identity_, session_timeout_ms_, operation_timeout_ms_, chroot_); @@ -141,8 +153,6 @@ struct ZooKeeperArgs Poco::Util::AbstractConfiguration::Keys keys; config.keys(config_name, keys); - std::vector hosts_strings; - session_timeout_ms = Coordination::DEFAULT_SESSION_TIMEOUT_MS; operation_timeout_ms = Coordination::DEFAULT_OPERATION_TIMEOUT_MS; implementation = "zookeeper"; @@ -150,7 +160,7 @@ struct ZooKeeperArgs { if (startsWith(key, "node")) { - hosts_strings.push_back( + hosts.push_back( (config.getBool(config_name + "." + key + ".secure", false) ? "secure://" : "") + config.getString(config_name + "." + key + ".host") + ":" + config.getString(config_name + "." + key + ".port", "2181") @@ -180,17 +190,6 @@ struct ZooKeeperArgs throw KeeperException(std::string("Unknown key ") + key + " in config file", Coordination::Error::ZBADARGUMENTS); } - /// Shuffle the hosts to distribute the load among ZooKeeper nodes. - pcg64 generator(randomSeed()); - std::shuffle(hosts_strings.begin(), hosts_strings.end(), generator); - - for (auto & host : hosts_strings) - { - if (!hosts.empty()) - hosts += ','; - hosts += host; - } - if (!chroot.empty()) { if (chroot.front() != '/') @@ -200,7 +199,7 @@ struct ZooKeeperArgs } } - std::string hosts; + Strings hosts; std::string identity; int session_timeout_ms; int operation_timeout_ms; @@ -922,6 +921,10 @@ Coordination::Error ZooKeeper::tryMultiNoThrow(const Coordination::Requests & re } } +void ZooKeeper::finalize() +{ + impl->finalize(); +} size_t KeeperMultiException::getFailedOpIndex(Coordination::Error exception_code, const Coordination::Responses & responses) { @@ -1000,4 +1003,5 @@ Coordination::RequestPtr makeCheckRequest(const std::string & path, int version) request->version = version; return request; } + } diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 5b37e4d6024..4a65ff070f7 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -50,7 +50,14 @@ class ZooKeeper public: using Ptr = std::shared_ptr; - ZooKeeper(const std::string & hosts_, const std::string & identity_ = "", + /// hosts_string -- comma separated [secure://]host:port list + ZooKeeper(const std::string & hosts_string, const std::string & identity_ = "", + int32_t session_timeout_ms_ = Coordination::DEFAULT_SESSION_TIMEOUT_MS, + int32_t operation_timeout_ms_ = Coordination::DEFAULT_OPERATION_TIMEOUT_MS, + const std::string & chroot_ = "", + const std::string & implementation_ = "zookeeper"); + + ZooKeeper(const Strings & hosts_, const std::string & identity_ = "", int32_t session_timeout_ms_ = Coordination::DEFAULT_SESSION_TIMEOUT_MS, int32_t operation_timeout_ms_ = Coordination::DEFAULT_OPERATION_TIMEOUT_MS, const std::string & chroot_ = "", @@ -247,10 +254,12 @@ public: /// Like the previous one but don't throw any exceptions on future.get() FutureMulti tryAsyncMulti(const Coordination::Requests & ops); + void finalize(); + private: friend class EphemeralNodeHolder; - void init(const std::string & implementation_, const std::string & hosts_, const std::string & identity_, + void init(const std::string & implementation_, const Strings & hosts_, const std::string & identity_, int32_t session_timeout_ms_, int32_t operation_timeout_ms_, const std::string & chroot_); /// The following methods don't throw exceptions but return error codes. @@ -266,7 +275,7 @@ private: std::unique_ptr impl; - std::string hosts; + Strings hosts; std::string identity; int32_t session_timeout_ms; int32_t operation_timeout_ms; diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h index 71b7cd56149..afd2e89538f 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -88,7 +88,7 @@ using namespace DB; /** Usage scenario: look at the documentation for IKeeper class. */ -class ZooKeeper : public IKeeper +class ZooKeeper final : public IKeeper { public: struct Node @@ -167,6 +167,20 @@ public: const Requests & requests, MultiCallback callback) override; + /// Without forcefully invalidating (finalizing) ZooKeeper session before + /// establishing a new one, there was a possibility that server is using + /// two ZooKeeper sessions simultaneously in different parts of code. + /// This is strong antipattern and we always prevented it. + + /// ZooKeeper is linearizeable for writes, but not linearizeable for + /// reads, it only maintains "sequential consistency": in every session + /// you observe all events in order but possibly with some delay. If you + /// perform write in one session, then notify different part of code and + /// it will do read in another session, that read may not see the + /// already performed write. + + void finalize() override { finalize(false, false); } + private: String root_path; ACLs default_acls; diff --git a/src/Coordination/tests/gtest_for_build.cpp b/src/Coordination/tests/gtest_for_build.cpp index c064fcdbef4..37517808ef0 100644 --- a/src/Coordination/tests/gtest_for_build.cpp +++ b/src/Coordination/tests/gtest_for_build.cpp @@ -195,124 +195,6 @@ TEST(CoordinationTest, TestSummingRaft1) s1.launcher.shutdown(5); } -TEST(CoordinationTest, TestSummingRaft3) -{ - ChangelogDirTest test1("./logs1"); - SummingRaftServer s1(1, "localhost", 44444, "./logs1"); - ChangelogDirTest test2("./logs2"); - SummingRaftServer s2(2, "localhost", 44445, "./logs2"); - ChangelogDirTest test3("./logs3"); - SummingRaftServer s3(3, "localhost", 44446, "./logs3"); - - nuraft::srv_config first_config(1, 0, "localhost:44444", "", false, 0); - auto ret1 = s2.raft_instance->add_srv(first_config); - while (!ret1->get_accepted()) - { - - std::cout << "failed to add server: " - << ret1->get_result_str() << std::endl; - - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - ret1 = s2.raft_instance->add_srv(first_config); - } - - while (s1.raft_instance->get_leader() != 2) - { - std::cout << "Waiting s1 to join to s2 quorum\n"; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - nuraft::srv_config third_config(3, 0, "localhost:44446", "", false, 0); - auto ret3 = s2.raft_instance->add_srv(third_config); - if (!ret3->get_accepted()) - { - std::cout << "failed to add server: " - << ret3->get_result_str() << std::endl; - - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - ret3 = s2.raft_instance->add_srv(third_config); - } - - while (s3.raft_instance->get_leader() != 2) - { - std::cout << "Waiting s3 to join to s2 quorum\n"; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - /// S2 is leader - EXPECT_EQ(s1.raft_instance->get_leader(), 2); - EXPECT_EQ(s2.raft_instance->get_leader(), 2); - EXPECT_EQ(s3.raft_instance->get_leader(), 2); - - std::cerr << "Starting to add entries\n"; - auto entry = getBuffer(1); - auto ret = s2.raft_instance->append_entries({entry}); - while (!ret->get_accepted() || ret->get_result_code() != nuraft::cmd_result_code::OK) - { - std::cerr << ret->get_accepted() << "failed to replicate: entry 1" << ret->get_result_code() << std::endl; - ret = s2.raft_instance->append_entries({entry}); - } - - while (s1.state_machine->getValue() != 1) - { - std::cout << "Waiting s1 to apply entry\n"; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - while (s2.state_machine->getValue() != 1) - { - std::cout << "Waiting s2 to apply entry\n"; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - while (s3.state_machine->getValue() != 1) - { - std::cout << "Waiting s3 to apply entry\n"; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - EXPECT_EQ(s1.state_machine->getValue(), 1); - EXPECT_EQ(s2.state_machine->getValue(), 1); - EXPECT_EQ(s3.state_machine->getValue(), 1); - - auto non_leader_entry = getBuffer(3); - auto ret_non_leader1 = s1.raft_instance->append_entries({non_leader_entry}); - - EXPECT_FALSE(ret_non_leader1->get_accepted()); - - auto ret_non_leader3 = s3.raft_instance->append_entries({non_leader_entry}); - - EXPECT_FALSE(ret_non_leader3->get_accepted()); - - auto leader_entry = getBuffer(77); - auto ret_leader = s2.raft_instance->append_entries({leader_entry}); - while (!ret_leader->get_accepted() || ret_leader->get_result_code() != nuraft::cmd_result_code::OK) - { - std::cerr << "failed to replicate: entry 78" << ret_leader->get_result_code() << std::endl; - ret_leader = s2.raft_instance->append_entries({leader_entry}); - } - - while (s1.state_machine->getValue() != 78) - { - std::cout << "Waiting s1 to apply entry\n"; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - while (s3.state_machine->getValue() != 78) - { - std::cout << "Waiting s3 to apply entry\n"; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - EXPECT_EQ(s1.state_machine->getValue(), 78); - EXPECT_EQ(s2.state_machine->getValue(), 78); - EXPECT_EQ(s3.state_machine->getValue(), 78); - - s1.launcher.shutdown(5); - s2.launcher.shutdown(5); - s3.launcher.shutdown(5); -} - nuraft::ptr getZooKeeperLogEntry(int64_t session_id, const Coordination::ZooKeeperRequestPtr & request) { DB::WriteBufferFromNuraftBuffer buf; diff --git a/src/Core/Settings.h b/src/Core/Settings.h index e4fefbb9fec..27fed7c2f8f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -444,6 +444,7 @@ class IColumn; M(UnionMode, union_default_mode, UnionMode::Unspecified, "Set default Union Mode in SelectWithUnion query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without Union Mode will throw exception.", 0) \ M(Bool, optimize_aggregators_of_group_by_keys, true, "Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section", 0) \ M(Bool, optimize_group_by_function_keys, true, "Eliminates functions of other keys in GROUP BY section", 0) \ + M(UInt64, query_plan_max_optimizations_to_apply, 10000, "Limit the total number of optimizations applied to query plan. If zero, ignored. If limit reached, throw exception", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS below. diff --git a/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index a967ee28502..3b0b4db72f9 100644 --- a/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -1,12 +1,15 @@ -#include -#include -#include -#include -#include -#include #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace DB @@ -15,7 +18,7 @@ namespace DB namespace ErrorCodes { extern const int VIOLATED_CONSTRAINT; - extern const int LOGICAL_ERROR; + extern const int UNSUPPORTED_METHOD; } @@ -48,62 +51,75 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) ColumnWithTypeAndName res_column = block_to_calculate.getByName(constraint_ptr->expr->getColumnName()); - if (!isUInt8(res_column.type)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Constraint {} does not return a value of type UInt8", + auto result_type = removeNullable(removeLowCardinality(res_column.type)); + + if (!isUInt8(result_type)) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Constraint {} does not return a value of type UInt8", backQuote(constraint_ptr->name)); - if (const ColumnConst * res_const = typeid_cast(res_column.column.get())) - { - UInt8 value = res_const->getValue(); + auto result_column = res_column.column->convertToFullColumnIfConst()->convertToFullColumnIfLowCardinality(); - /// Is violated. - if (!value) - { - throw Exception(ErrorCodes::VIOLATED_CONSTRAINT, - "Constraint {} for table {} is violated, because it is a constant expression returning 0. " - "It is most likely an error in table definition.", - backQuote(constraint_ptr->name), table_id.getNameForLogs()); - } + if (const auto * column_nullable = checkAndGetColumn(*result_column)) + { + const auto & nested_column = column_nullable->getNestedColumnPtr(); + + /// Check if constraint value is nullable + const auto & null_map = column_nullable->getNullMapColumn(); + const PaddedPODArray & data = null_map.getData(); + bool null_map_contains_null = !memoryIsZero(data.raw_data(), data.size() * sizeof(UInt8)); + + if (null_map_contains_null) + throw Exception( + ErrorCodes::VIOLATED_CONSTRAINT, + "Constraint {} for table {} is violated. Expression: ({})."\ + "Constraint expression returns nullable column that contains null value", + backQuote(constraint_ptr->name), + table_id.getNameForLogs(), + serializeAST(*(constraint_ptr->expr), true)); + + result_column = nested_column; } - else + + const ColumnUInt8 & res_column_uint8 = assert_cast(*result_column); + + const UInt8 * data = res_column_uint8.getData().data(); + size_t size = res_column_uint8.size(); + + /// Is violated. + if (!memoryIsByte(data, size, 1)) { - const ColumnUInt8 & res_column_uint8 = assert_cast(*res_column.column); + size_t row_idx = 0; + for (; row_idx < size; ++row_idx) + if (data[row_idx] != 1) + break; - const UInt8 * data = res_column_uint8.getData().data(); - size_t size = res_column_uint8.size(); + Names related_columns = constraint_expr->getRequiredColumns(); - /// Is violated. - if (!memoryIsByte(data, size, 1)) + bool first = true; + String column_values_msg; + constexpr size_t approx_bytes_for_col = 32; + column_values_msg.reserve(approx_bytes_for_col * related_columns.size()); + for (const auto & name : related_columns) { - size_t row_idx = 0; - for (; row_idx < size; ++row_idx) - if (data[row_idx] != 1) - break; + const IColumn & column = *block.getByName(name).column; + assert(row_idx < column.size()); - Names related_columns = constraint_expr->getRequiredColumns(); - - bool first = true; - String column_values_msg; - constexpr size_t approx_bytes_for_col = 32; - column_values_msg.reserve(approx_bytes_for_col * related_columns.size()); - for (const auto & name : related_columns) - { - const IColumn & column = *block.getByName(name).column; - assert(row_idx < column.size()); - - if (!first) - column_values_msg.append(", "); - column_values_msg.append(backQuoteIfNeed(name)); - column_values_msg.append(" = "); - column_values_msg.append(applyVisitor(FieldVisitorToString(), column[row_idx])); - first = false; - } - - throw Exception(ErrorCodes::VIOLATED_CONSTRAINT, - "Constraint {} for table {} is violated at row {}. Expression: ({}). Column values: {}", - backQuote(constraint_ptr->name), table_id.getNameForLogs(), rows_written + row_idx + 1, - serializeAST(*(constraint_ptr->expr), true), column_values_msg); + if (!first) + column_values_msg.append(", "); + column_values_msg.append(backQuoteIfNeed(name)); + column_values_msg.append(" = "); + column_values_msg.append(applyVisitor(FieldVisitorToString(), column[row_idx])); + first = false; } + + throw Exception( + ErrorCodes::VIOLATED_CONSTRAINT, + "Constraint {} for table {} is violated at row {}. Expression: ({}). Column values: {}", + backQuote(constraint_ptr->name), + table_id.getNameForLogs(), + rows_written + row_idx + 1, + serializeAST(*(constraint_ptr->expr), true), + column_values_msg); } } } diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index a21b1bd50fc..572080aee1e 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -47,11 +47,13 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory) # include # include # include "readInvalidateQuery.h" +# include # include namespace DB { static const UInt64 max_block_size = 8192; +static const size_t default_num_tries_on_connection_loss = 3; MySQLDictionarySource::MySQLDictionarySource( @@ -72,7 +74,10 @@ MySQLDictionarySource::MySQLDictionarySource( , query_builder{dict_struct, db, "", table, where, IdentifierQuotingStyle::Backticks} , load_all_query{query_builder.composeLoadAllQuery()} , invalidate_query{config.getString(config_prefix + ".invalidate_query", "")} - , close_connection{config.getBool(config_prefix + ".close_connection", false) || config.getBool(config_prefix + ".share_connection", false)} + , close_connection( + config.getBool(config_prefix + ".close_connection", false) || config.getBool(config_prefix + ".share_connection", false)) + , max_tries_for_mysql_block_input_stream( + config.getBool(config_prefix + ".fail_on_connection_loss", false) ? 1 : default_num_tries_on_connection_loss) { } @@ -94,6 +99,7 @@ MySQLDictionarySource::MySQLDictionarySource(const MySQLDictionarySource & other , invalidate_query{other.invalidate_query} , invalidate_query_response{other.invalidate_query_response} , close_connection{other.close_connection} + , max_tries_for_mysql_block_input_stream{other.max_tries_for_mysql_block_input_stream} { } @@ -114,13 +120,41 @@ std::string MySQLDictionarySource::getUpdateFieldAndDate() } } +BlockInputStreamPtr MySQLDictionarySource::retriedCreateMySqlBIStream(const std::string & data_fetch_query_str, const size_t max_tries) +{ + size_t count_connection_lost = 0; + + while (true) + { + auto connection = pool.get(); + + try + { + return std::make_shared( + connection, data_fetch_query_str, sample_block, max_block_size, close_connection); + } + catch (const mysqlxx::ConnectionLost & ecl) /// There are two retriable failures: CR_SERVER_GONE_ERROR, CR_SERVER_LOST + { + if (++count_connection_lost < max_tries) + { + LOG_WARNING(log, ecl.displayText()); + LOG_WARNING(log, "Lost connection ({}/{}). Trying to reconnect...", count_connection_lost, max_tries); + continue; + } + + LOG_ERROR(log, "Failed ({}/{}) to create BlockInputStream for MySQL dictionary source.", count_connection_lost, max_tries); + throw; + } + } +} + BlockInputStreamPtr MySQLDictionarySource::loadAll() { auto connection = pool.get(); last_modification = getLastModification(connection, false); LOG_TRACE(log, load_all_query); - return std::make_shared(connection, load_all_query, sample_block, max_block_size, close_connection); + return retriedCreateMySqlBIStream(load_all_query, max_tries_for_mysql_block_input_stream); } BlockInputStreamPtr MySQLDictionarySource::loadUpdatedAll() @@ -130,7 +164,7 @@ BlockInputStreamPtr MySQLDictionarySource::loadUpdatedAll() std::string load_update_query = getUpdateFieldAndDate(); LOG_TRACE(log, load_update_query); - return std::make_shared(connection, load_update_query, sample_block, max_block_size, close_connection); + return retriedCreateMySqlBIStream(load_update_query, max_tries_for_mysql_block_input_stream); } BlockInputStreamPtr MySQLDictionarySource::loadIds(const std::vector & ids) @@ -138,7 +172,7 @@ BlockInputStreamPtr MySQLDictionarySource::loadIds(const std::vector & i /// We do not log in here and do not update the modification time, as the request can be large, and often called. const auto query = query_builder.composeLoadIdsQuery(ids); - return std::make_shared(pool.get(), query, sample_block, max_block_size, close_connection); + return retriedCreateMySqlBIStream(query, max_tries_for_mysql_block_input_stream); } BlockInputStreamPtr MySQLDictionarySource::loadKeys(const Columns & key_columns, const std::vector & requested_rows) @@ -146,7 +180,7 @@ BlockInputStreamPtr MySQLDictionarySource::loadKeys(const Columns & key_columns, /// We do not log in here and do not update the modification time, as the request can be large, and often called. const auto query = query_builder.composeLoadKeysQuery(key_columns, requested_rows, ExternalQueryBuilder::AND_OR_CHAIN); - return std::make_shared(pool.get(), query, sample_block, max_block_size, close_connection); + return retriedCreateMySqlBIStream(query, max_tries_for_mysql_block_input_stream); } bool MySQLDictionarySource::isModified() const diff --git a/src/Dictionaries/MySQLDictionarySource.h b/src/Dictionaries/MySQLDictionarySource.h index 34f784cdfeb..54bc2da6a56 100644 --- a/src/Dictionaries/MySQLDictionarySource.h +++ b/src/Dictionaries/MySQLDictionarySource.h @@ -69,6 +69,9 @@ private: // execute invalidate_query. expects single cell in result std::string doInvalidateQuery(const std::string & request) const; + /// A helper method for recovering from "Lost connection to MySQL server during query" errors + BlockInputStreamPtr retriedCreateMySqlBIStream(const std::string & query_str, const size_t max_tries); + Poco::Logger * log; std::chrono::time_point update_time; @@ -86,6 +89,7 @@ private: std::string invalidate_query; mutable std::string invalidate_query_response; const bool close_connection; + const size_t max_tries_for_mysql_block_input_stream; }; } diff --git a/src/Functions/extractTextFromHTML.cpp b/src/Functions/extractTextFromHTML.cpp new file mode 100644 index 00000000000..528bd0c311f --- /dev/null +++ b/src/Functions/extractTextFromHTML.cpp @@ -0,0 +1,358 @@ +#include +#include +#include +#include +#include +#include + + +/** A function to extract text from HTML or XHTML. + * It does not necessarily 100% conforms to any of the HTML, XML or XHTML standards, + * but the implementation is reasonably accurate and it is fast. + * + * The rules are the following: + * + * 1. Comments are skipped. Example: + * Comment must end with -->. Nested comments are not possible. + * Note: constructions like are not valid comments in HTML but will be skipped by other rules. + * + * 2. CDATA is pasted verbatim. + * Note: CDATA is XML/XHTML specific. But we still process it for "best-effort" approach. + * + * 3. 'script' and 'style' elements are removed with all their content. + * Note: it's assumed that closing tag cannot appear inside content. + * For example, in JS string literal is has to be escaped as "<\/script>". + * Note: comments and CDATA is possible inside script or style - then closing tags are not searched inside CDATA. + * Example: ]]> + * But still searched inside comments. Sometimes it becomes complicated: + * var y = "-->"; alert(x + y); + * Note: script and style can be the names of XML namespaces - then they are not treat like usual script or style. + * Example: Hello. + * Note: whitespaces are possible after closing tag name: but not before: < / script>. + * + * 4. Other tags or tag-like elements are skipped without inner content. + * Example: . + * Note: it's expected that this HTML is illegal: + * Note: it will also skip something like tags: <>, , etc. + * Note: tag without end will be skipped to the end of input: + * 5. HTML and XML entities are not decoded. + * It should be processed by separate function. + * + * 6. Whitespaces in text are collapsed or inserted by specific rules. + * Whitespaces at beginning and at the end are removed. + * Consecutive whitespaces are collapsed. + * But if text is separated by other elements and there is no whitespace, it is inserted. + * It may be unnatural, examples: Helloworld, Helloworld + * - in HTML there will be no whitespace, but the function will insert it. + * But also consider: Hello

world

, Hello
world. + * This behaviour is reasonable for data analysis, e.g. convert HTML to a bag of words. + * + * 7. Also note that correct handling of whitespaces would require + * support of 
 and CSS display and white-space properties.
+  *
+  * Usage example:
+  *
+  * SELECT extractTextFromHTML(html) FROM url('https://yandex.ru/', RawBLOB, 'html String')
+  *
+  * - ClickHouse has embedded web browser.
+  */
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_COLUMN;
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+}
+
+namespace
+{
+
+inline bool startsWith(const char * s, const char * end, const char * prefix)
+{
+    return s + strlen(prefix) < end && 0 == memcmp(s, prefix, strlen(prefix));
+}
+
+inline bool checkAndSkip(const char * __restrict & s, const char * end, const char * prefix)
+{
+    if (startsWith(s, end, prefix))
+    {
+        s += strlen(prefix);
+        return true;
+    }
+    return false;
+}
+
+bool processComment(const char * __restrict & src, const char * end)
+{
+    if (!checkAndSkip(src, end, "world');
+Hello world
+SELECT extractTextFromHTML('Helloworld');
+Hello world
+SELECT extractTextFromHTML('Helloworld');
+Hello world
+SELECT extractTextFromHTML('Hello  World');
+Hello World
+SELECT extractTextFromHTML('Hello World');
+Hello World
+SELECT extractTextFromHTML('HelloWorld');
+Hello World
+SELECT extractTextFromHTML('Hello World');
+Hello World
+SELECT extractTextFromHTML('Hello  World');
+Hello World
+SELECT extractTextFromHTML('Hello World');
+Hello World
+SELECT extractTextFromHTML('HelloWorld');
+Hello World
+SELECT extractTextFromHTML('Hello World');
+Hello World
+SELECT extractTextFromHTML('');
+ \t Hello,\rworld \n 
+SELECT extractTextFromHTML('Hello  world!');
+HelloHello\tworld world!
+SELECT extractTextFromHTML('Helloworld!');
+HelloHello\tworld world!
+SELECT extractTextFromHTML('Hello world]]> world!');
+HelloHello world world!
+SELECT extractTextFromHTML('John Smith]]>');
+John Smith
+SELECT extractTextFromHTML('John ]]>');
+John 
+SELECT extractTextFromHTML('John Smith]]>');
+John 
+SELECT extractTextFromHTML('John ]]>]]>');
+John Smith
+SELECT extractTextFromHTML('John ]]> ]]>');
+John Smith
+SELECT extractTextFromHTML('John]]> ]]>');
+JohnSmith
+SELECT extractTextFromHTML('John ]]>]]>]]>');
+John ]]>Smith
+SELECT extractTextFromHTML('Hello goodbye');
+Hello goodbye
+SELECT extractTextFromHTML('Hello goodbye');
+Hello goodbye
+SELECT extractTextFromHTML('HelloWorld goodbye');
+Hello World goodbye
+SELECT extractTextFromHTML('Hello goodbye');
+Hello goodbye
+SELECT extractTextFromHTML('Hello goodbye');
+Hello goodbye
+SELECT extractTextFromHTML('HelloWorld goodbye');
+Hello World goodbye
+SELECT extractTextFromHTML('HelloWorld goodbye');
+Hello World goodbye
+SELECT extractTextFromHTML('Hello goodbye');
+Hello goodbye
+SELECT extractTextFromHTML('Hello goodbye');
+Hello goodbye
+SELECT extractTextFromHTML('Hello]]>  goodbye');
+Hello
+SELECT extractTextFromHTML('Hello]]>  goodbye');
+Hello goodbye
+SELECT extractTextFromHTML('Hello]]>  goodbye');
+Hello ]]> goodbye
+SELECT extractTextFromHTML('Hello goodbye');
+Hello goodbye
+SELECT extractTextFromHTML('Hello goodbye');
+Hello goodbye
+SELECT extractTextFromHTML(']]>');
+]]>
+SELECT extractTextFromHTML('
+
+
xkcd.com
+
+');
+xkcd.com
diff --git a/tests/queries/0_stateless/01746_extract_text_from_html.sql b/tests/queries/0_stateless/01746_extract_text_from_html.sql
new file mode 100644
index 00000000000..b4ccc775bef
--- /dev/null
+++ b/tests/queries/0_stateless/01746_extract_text_from_html.sql
@@ -0,0 +1,72 @@
+-- { echo }
+
+SELECT extractTextFromHTML('');
+SELECT extractTextFromHTML(' ');
+SELECT extractTextFromHTML('  ');
+SELECT extractTextFromHTML('Hello');
+SELECT extractTextFromHTML('Hello, world');
+SELECT extractTextFromHTML('Hello,  world');
+SELECT extractTextFromHTML(' Hello,  world');
+SELECT extractTextFromHTML(' Hello,  world ');
+SELECT extractTextFromHTML(' \t Hello,\rworld \n ');
+
+SELECT extractTextFromHTML('Hello world');
+SELECT extractTextFromHTML('Hello');
+SELECT extractTextFromHTML('Hello<>world');
+SELECT extractTextFromHTML('Helloworld');
+SELECT extractTextFromHTML('Helloworld');
+SELECT extractTextFromHTML('Helloworld');
+SELECT extractTextFromHTML('Helloworld');
+SELECT extractTextFromHTML('Helloworld');
+
+SELECT extractTextFromHTML('Hello  World');
+SELECT extractTextFromHTML('Hello World');
+SELECT extractTextFromHTML('HelloWorld');
+SELECT extractTextFromHTML('Hello World');
+SELECT extractTextFromHTML('Hello  World');
+SELECT extractTextFromHTML('Hello World');
+SELECT extractTextFromHTML('HelloWorld');
+SELECT extractTextFromHTML('Hello World');
+
+SELECT extractTextFromHTML('');
+SELECT extractTextFromHTML('Hello  world!');
+SELECT extractTextFromHTML('Helloworld!');
+
+SELECT extractTextFromHTML('Hello world]]> world!');
+SELECT extractTextFromHTML('John Smith]]>');
+SELECT extractTextFromHTML('John ]]>');
+SELECT extractTextFromHTML('John Smith]]>');
+SELECT extractTextFromHTML('John ]]>]]>');
+SELECT extractTextFromHTML('John ]]> ]]>');
+SELECT extractTextFromHTML('John]]> ]]>');
+SELECT extractTextFromHTML('John ]]>]]>]]>');
+
+SELECT extractTextFromHTML('Hello goodbye');
+SELECT extractTextFromHTML('Hello goodbye');
+SELECT extractTextFromHTML('HelloWorld goodbye');
+SELECT extractTextFromHTML('Hello goodbye');
+SELECT extractTextFromHTML('Hello goodbye');
+SELECT extractTextFromHTML('HelloWorld goodbye');
+SELECT extractTextFromHTML('HelloWorld goodbye');
+
+SELECT extractTextFromHTML('Hello goodbye');
+SELECT extractTextFromHTML('Hello goodbye');
+SELECT extractTextFromHTML('Hello]]>  goodbye');
+SELECT extractTextFromHTML('Hello]]>  goodbye');
+SELECT extractTextFromHTML('Hello]]>  goodbye');
+SELECT extractTextFromHTML('Hello goodbye');
+SELECT extractTextFromHTML('Hello goodbye');
+
+SELECT extractTextFromHTML(']]>');
+
+SELECT extractTextFromHTML('
+
+
xkcd.com
+
+');
diff --git a/tests/queries/0_stateless/01746_long_zlib_http_compression_json_format.reference b/tests/queries/0_stateless/01746_long_zlib_http_compression_json_format.reference
new file mode 100644
index 00000000000..7c089a2fd05
--- /dev/null
+++ b/tests/queries/0_stateless/01746_long_zlib_http_compression_json_format.reference
@@ -0,0 +1,23 @@
+		},
+		{
+			"datetime": "2020-12-12",
+			"pipeline": "test-pipeline",
+			"host": "clickhouse-test-host-001.clickhouse.com",
+			"home": "clickhouse",
+			"detail": "clickhouse",
+			"row_number": "999998"
+		},
+		{
+			"datetime": "2020-12-12",
+			"pipeline": "test-pipeline",
+			"host": "clickhouse-test-host-001.clickhouse.com",
+			"home": "clickhouse",
+			"detail": "clickhouse",
+			"row_number": "999999"
+		}
+	],
+
+	"rows": 1000000,
+
+	"rows_before_limit_at_least": 1048080,
+
diff --git a/tests/queries/0_stateless/01746_long_zlib_http_compression_json_format.sh b/tests/queries/0_stateless/01746_long_zlib_http_compression_json_format.sh
new file mode 100755
index 00000000000..e663b329660
--- /dev/null
+++ b/tests/queries/0_stateless/01746_long_zlib_http_compression_json_format.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CURL} -sS -H 'Accept-Encoding: gzip'              "${CLICKHOUSE_URL}&enable_http_compression=1&http_zlib_compression_level=1" -d "SELECT toDate('2020-12-12') as datetime, 'test-pipeline' as pipeline, 'clickhouse-test-host-001.clickhouse.com' as host, 'clickhouse' as home, 'clickhouse' as detail, number as row_number FROM numbers(1000000) FORMAT JSON" | gzip -d | tail -n30 | head -n23
diff --git a/tests/queries/0_stateless/01746_long_zstd_http_compression_json_format.reference b/tests/queries/0_stateless/01746_long_zstd_http_compression_json_format.reference
new file mode 100644
index 00000000000..7c089a2fd05
--- /dev/null
+++ b/tests/queries/0_stateless/01746_long_zstd_http_compression_json_format.reference
@@ -0,0 +1,23 @@
+		},
+		{
+			"datetime": "2020-12-12",
+			"pipeline": "test-pipeline",
+			"host": "clickhouse-test-host-001.clickhouse.com",
+			"home": "clickhouse",
+			"detail": "clickhouse",
+			"row_number": "999998"
+		},
+		{
+			"datetime": "2020-12-12",
+			"pipeline": "test-pipeline",
+			"host": "clickhouse-test-host-001.clickhouse.com",
+			"home": "clickhouse",
+			"detail": "clickhouse",
+			"row_number": "999999"
+		}
+	],
+
+	"rows": 1000000,
+
+	"rows_before_limit_at_least": 1048080,
+
diff --git a/tests/queries/0_stateless/01746_long_zstd_http_compression_json_format.sh b/tests/queries/0_stateless/01746_long_zstd_http_compression_json_format.sh
new file mode 100755
index 00000000000..f520a21e9b3
--- /dev/null
+++ b/tests/queries/0_stateless/01746_long_zstd_http_compression_json_format.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CURL} -sS -H 'Accept-Encoding: zstd'              "${CLICKHOUSE_URL}&enable_http_compression=1" -d "SELECT toDate('2020-12-12') as datetime, 'test-pipeline' as pipeline, 'clickhouse-test-host-001.clickhouse.com' as host, 'clickhouse' as home, 'clickhouse' as detail, number as row_number FROM numbers(1000000) FORMAT JSON" | zstd -d | tail -n30 | head -n23
diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json
index 475b87ae36a..36cca55779d 100644
--- a/tests/queries/skip_list.json
+++ b/tests/queries/skip_list.json
@@ -260,7 +260,8 @@
         "00121_drop_column_zookeeper",
         "00116_storage_set",
         "00083_create_merge_tree_zookeeper",
-        "00062_replicated_merge_tree_alter_zookeeper"
+        "00062_replicated_merge_tree_alter_zookeeper",
+        "01720_constraints_complex_types"
     ],
     "polymorphic-parts": [
         "01508_partition_pruning_long", /// bug, shoud be fixed
diff --git a/tests/testflows/aes_encryption/docker-compose/zookeeper-service.yml b/tests/testflows/aes_encryption/docker-compose/zookeeper-service.yml
index f3df33358be..f27405b97a2 100644
--- a/tests/testflows/aes_encryption/docker-compose/zookeeper-service.yml
+++ b/tests/testflows/aes_encryption/docker-compose/zookeeper-service.yml
@@ -2,7 +2,7 @@ version: '2.3'
 
 services:
   zookeeper:
-    image: zookeeper:3.4.12
+    image: zookeeper:3.6.2
     expose:
       - "2181"
     environment:
diff --git a/tests/testflows/example/docker-compose/zookeeper-service.yml b/tests/testflows/example/docker-compose/zookeeper-service.yml
index 6691a2df31c..ca732a48dbd 100644
--- a/tests/testflows/example/docker-compose/zookeeper-service.yml
+++ b/tests/testflows/example/docker-compose/zookeeper-service.yml
@@ -2,7 +2,7 @@ version: '2.3'
 
 services:
   zookeeper:
-    image: zookeeper:3.4.12
+    image: zookeeper:3.6.2
     expose:
       - "2181"
     environment:
diff --git a/tests/testflows/ldap/authentication/docker-compose/zookeeper-service.yml b/tests/testflows/ldap/authentication/docker-compose/zookeeper-service.yml
index 6691a2df31c..ca732a48dbd 100644
--- a/tests/testflows/ldap/authentication/docker-compose/zookeeper-service.yml
+++ b/tests/testflows/ldap/authentication/docker-compose/zookeeper-service.yml
@@ -2,7 +2,7 @@ version: '2.3'
 
 services:
   zookeeper:
-    image: zookeeper:3.4.12
+    image: zookeeper:3.6.2
     expose:
       - "2181"
     environment:
diff --git a/tests/testflows/ldap/external_user_directory/docker-compose/zookeeper-service.yml b/tests/testflows/ldap/external_user_directory/docker-compose/zookeeper-service.yml
index 6691a2df31c..ca732a48dbd 100644
--- a/tests/testflows/ldap/external_user_directory/docker-compose/zookeeper-service.yml
+++ b/tests/testflows/ldap/external_user_directory/docker-compose/zookeeper-service.yml
@@ -2,7 +2,7 @@ version: '2.3'
 
 services:
   zookeeper:
-    image: zookeeper:3.4.12
+    image: zookeeper:3.6.2
     expose:
       - "2181"
     environment:
diff --git a/tests/testflows/ldap/role_mapping/docker-compose/zookeeper-service.yml b/tests/testflows/ldap/role_mapping/docker-compose/zookeeper-service.yml
index 6691a2df31c..ca732a48dbd 100644
--- a/tests/testflows/ldap/role_mapping/docker-compose/zookeeper-service.yml
+++ b/tests/testflows/ldap/role_mapping/docker-compose/zookeeper-service.yml
@@ -2,7 +2,7 @@ version: '2.3'
 
 services:
   zookeeper:
-    image: zookeeper:3.4.12
+    image: zookeeper:3.6.2
     expose:
       - "2181"
     environment:
diff --git a/tests/testflows/rbac/docker-compose/zookeeper-service.yml b/tests/testflows/rbac/docker-compose/zookeeper-service.yml
index f3df33358be..f27405b97a2 100755
--- a/tests/testflows/rbac/docker-compose/zookeeper-service.yml
+++ b/tests/testflows/rbac/docker-compose/zookeeper-service.yml
@@ -2,7 +2,7 @@ version: '2.3'
 
 services:
   zookeeper:
-    image: zookeeper:3.4.12
+    image: zookeeper:3.6.2
     expose:
       - "2181"
     environment:
diff --git a/website/css/highlight.css b/website/css/highlight.css
index 55a0054b07f..7cc8a4865dd 100644
--- a/website/css/highlight.css
+++ b/website/css/highlight.css
@@ -11,7 +11,7 @@
 .syntax .hll { background-color: #b9b6b0 }
 .syntax  { background: #f8f9fa; color: #2f1e2e }
 .syntax .c { color: #8d8687 } /* Comment */
-.syntax .err { color: #ef6155 } /* Error */
+.syntax .err {} /* Error */
 .syntax .k { color: #000000; font-weight: bold } /* Keyword */
 .syntax .l { color: #0088ff } /* Literal */
 .syntax .n { color: #2f1e2e } /* Name */