From ddb02282f195401c68c9e81f339b3e2893bbd330 Mon Sep 17 00:00:00 2001 From: taiyang-li Date: Sun, 24 Jan 2021 11:52:17 +0800 Subject: [PATCH 001/198] enable local disk config reload --- src/Disks/DiskLocal.cpp | 114 +++++++++++++++++++++++-------------- src/Disks/DiskLocal.h | 9 ++- src/Disks/DiskSelector.cpp | 5 +- src/Disks/DiskSelector.h | 2 +- 4 files changed, 83 insertions(+), 47 deletions(-) diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index ed6b82fa73d..ea902bae6dc 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -34,6 +34,60 @@ std::mutex DiskLocal::reservation_mutex; using DiskLocalPtr = std::shared_ptr; +static void loadDiskLocalConfig(const String & name, + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + const Context & context, + String & path, + UInt64 & keep_free_space_bytes) +{ + path = config.getString(config_prefix + ".path", ""); + if (name == "default") + { + if (!path.empty()) + throw Exception( + "\"default\" disk path should be provided in not in ", + ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + path = context.getPath(); + } + else + { + if (path.empty()) + throw Exception("Disk path can not be empty. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + if (path.back() != '/') + throw Exception("Disk path must end with /. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + } + + if (Poco::File disk{path}; !disk.canRead() || !disk.canWrite()) + { + throw Exception("There is no RW access to disk " + name + " (" + path + ")", ErrorCodes::PATH_ACCESS_DENIED); + } + + bool has_space_ratio = config.has(config_prefix + ".keep_free_space_ratio"); + + if (config.has(config_prefix + ".keep_free_space_bytes") && has_space_ratio) + throw Exception( + "Only one of 'keep_free_space_bytes' and 'keep_free_space_ratio' can be specified", + ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); + + keep_free_space_bytes = config.getUInt64(config_prefix + ".keep_free_space_bytes", 0); + + if (has_space_ratio) + { + auto ratio = config.getDouble(config_prefix + ".keep_free_space_ratio"); + if (ratio < 0 || ratio > 1) + throw Exception("'keep_free_space_ratio' have to be between 0 and 1", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); + String tmp_path = path; + if (tmp_path.empty()) + tmp_path = context.getPath(); + + // Create tmp disk for getting total disk space. + keep_free_space_bytes = static_cast(DiskLocal("tmp", tmp_path, 0).getTotalSpace() * ratio); + } +} + + + class DiskLocalReservation : public IReservation { public: @@ -337,6 +391,19 @@ void DiskLocal::sync(int fd) const throw Exception("Cannot fsync", ErrorCodes::CANNOT_FSYNC); } +void DiskLocal::updateFromConfig(const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + const Context & context) +{ + String new_disk_path; + UInt64 new_keep_free_space_bytes; + loadDiskLocalConfig(name, config, config_prefix, context, new_disk_path, new_keep_free_space_bytes); + if (disk_path != new_disk_path) + throw Exception("Disk path can't update from config " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + keep_free_space_bytes = new_keep_free_space_bytes; +} + + DiskPtr DiskLocalReservation::getDisk(size_t i) const { if (i != 0) @@ -388,50 +455,9 @@ void registerDiskLocal(DiskFactory & factory) const Poco::Util::AbstractConfiguration & config, const String & config_prefix, const Context & context) -> DiskPtr { - String path = config.getString(config_prefix + ".path", ""); - if (name == "default") - { - if (!path.empty()) - throw Exception( - "\"default\" disk path should be provided in not it ", - ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); - path = context.getPath(); - } - else - { - if (path.empty()) - throw Exception("Disk path can not be empty. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); - if (path.back() != '/') - throw Exception("Disk path must end with /. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); - } - - if (Poco::File disk{path}; !disk.canRead() || !disk.canWrite()) - { - throw Exception("There is no RW access to disk " + name + " (" + path + ")", ErrorCodes::PATH_ACCESS_DENIED); - } - - bool has_space_ratio = config.has(config_prefix + ".keep_free_space_ratio"); - - if (config.has(config_prefix + ".keep_free_space_bytes") && has_space_ratio) - throw Exception( - "Only one of 'keep_free_space_bytes' and 'keep_free_space_ratio' can be specified", - ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); - - UInt64 keep_free_space_bytes = config.getUInt64(config_prefix + ".keep_free_space_bytes", 0); - - if (has_space_ratio) - { - auto ratio = config.getDouble(config_prefix + ".keep_free_space_ratio"); - if (ratio < 0 || ratio > 1) - throw Exception("'keep_free_space_ratio' have to be between 0 and 1", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); - String tmp_path = path; - if (tmp_path.empty()) - tmp_path = context.getPath(); - - // Create tmp disk for getting total disk space. - keep_free_space_bytes = static_cast(DiskLocal("tmp", tmp_path, 0).getTotalSpace() * ratio); - } - + String path; + UInt64 keep_free_space_bytes; + loadDiskLocalConfig(name, config, config_prefix, context, path, keep_free_space_bytes); return std::make_shared(name, path, keep_free_space_bytes); }; factory.registerDiskType("local", creator); diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index 09f08a01d27..ee87abe2359 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -7,6 +7,7 @@ #include #include +#include namespace DB { @@ -17,6 +18,7 @@ namespace ErrorCodes class DiskLocalReservation; +class Context; class DiskLocal : public IDisk { public: @@ -106,13 +108,17 @@ public: const String getType() const override { return "local"; } + void updateFromConfig(const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + const Context & context); + private: bool tryReserve(UInt64 bytes); private: const String name; const String disk_path; - const UInt64 keep_free_space_bytes; + UInt64 keep_free_space_bytes; UInt64 reserved_bytes = 0; UInt64 reservation_count = 0; @@ -120,4 +126,5 @@ private: static std::mutex reservation_mutex; }; + } diff --git a/src/Disks/DiskSelector.cpp b/src/Disks/DiskSelector.cpp index 0fb728a4f02..8d17515262d 100644 --- a/src/Disks/DiskSelector.cpp +++ b/src/Disks/DiskSelector.cpp @@ -66,9 +66,9 @@ DiskSelectorPtr DiskSelector::updateFromConfig( if (!std::all_of(disk_name.begin(), disk_name.end(), isWordCharASCII)) throw Exception("Disk name can contain only alphanumeric and '_' (" + disk_name + ")", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); + auto disk_config_prefix = config_prefix + "." + disk_name; if (result->getDisksMap().count(disk_name) == 0) { - auto disk_config_prefix = config_prefix + "." + disk_name; result->addToDiskMap(disk_name, factory.create(disk_name, config, disk_config_prefix, context)); } else @@ -77,6 +77,9 @@ DiskSelectorPtr DiskSelector::updateFromConfig( /// TODO: Ideally ClickHouse shall complain if disk has changed, but /// implementing that may appear as not trivial task. + auto disk = std::static_pointer_cast(result->getDisksMap().find(disk_name)->second); + if (disk) + disk->updateFromConfig(config, disk_config_prefix, context); } } diff --git a/src/Disks/DiskSelector.h b/src/Disks/DiskSelector.h index 5d023fe1fbc..a6027616511 100644 --- a/src/Disks/DiskSelector.h +++ b/src/Disks/DiskSelector.h @@ -34,7 +34,7 @@ public: /// Get all disks with names const DisksMap & getDisksMap() const { return disks; } - void addToDiskMap(String name, DiskPtr disk) + void addToDiskMap(const String & name, DiskPtr disk) { disks.emplace(name, disk); } From 901de8968f9df6fd9e5dc3e763b1719b141fb555 Mon Sep 17 00:00:00 2001 From: taiyang-li Date: Tue, 26 Jan 2021 15:07:17 +0800 Subject: [PATCH 002/198] update disk local if config changed --- src/Disks/DiskLocal.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index ea902bae6dc..e0599740651 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -400,7 +400,10 @@ void DiskLocal::updateFromConfig(const Poco::Util::AbstractConfiguration & confi loadDiskLocalConfig(name, config, config_prefix, context, new_disk_path, new_keep_free_space_bytes); if (disk_path != new_disk_path) throw Exception("Disk path can't update from config " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); - keep_free_space_bytes = new_keep_free_space_bytes; + + // Config of DiskLocal has been updated + if (keep_free_space_bytes != new_keep_free_space_bytes) + keep_free_space_bytes = new_keep_free_space_bytes; } From 44200abfb4b8869032d4a542ba555cf0f1badd99 Mon Sep 17 00:00:00 2001 From: taiyang-li Date: Fri, 29 Jan 2021 19:13:33 +0800 Subject: [PATCH 003/198] add interface updateFromConfigIfChanged --- src/Disks/DiskLocal.cpp | 6 +++--- src/Disks/DiskLocal.h | 4 ++-- src/Disks/DiskSelector.cpp | 6 +++--- src/Disks/IDisk.h | 7 +++++++ 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index e0599740651..86bf358b11f 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -391,17 +391,17 @@ void DiskLocal::sync(int fd) const throw Exception("Cannot fsync", ErrorCodes::CANNOT_FSYNC); } -void DiskLocal::updateFromConfig(const Poco::Util::AbstractConfiguration & config, +void DiskLocal::updateFromConfigIfChanged(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, const Context & context) { String new_disk_path; UInt64 new_keep_free_space_bytes; loadDiskLocalConfig(name, config, config_prefix, context, new_disk_path, new_keep_free_space_bytes); + if (disk_path != new_disk_path) throw Exception("Disk path can't update from config " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); - - // Config of DiskLocal has been updated + if (keep_free_space_bytes != new_keep_free_space_bytes) keep_free_space_bytes = new_keep_free_space_bytes; } diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index ee87abe2359..d9f3434cf5e 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -108,9 +108,9 @@ public: const String getType() const override { return "local"; } - void updateFromConfig(const Poco::Util::AbstractConfiguration & config, + void updateFromConfigIfChanged(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, - const Context & context); + const Context & context) override; private: bool tryReserve(UInt64 bytes); diff --git a/src/Disks/DiskSelector.cpp b/src/Disks/DiskSelector.cpp index 8d17515262d..f8dc5cd22ff 100644 --- a/src/Disks/DiskSelector.cpp +++ b/src/Disks/DiskSelector.cpp @@ -77,9 +77,9 @@ DiskSelectorPtr DiskSelector::updateFromConfig( /// TODO: Ideally ClickHouse shall complain if disk has changed, but /// implementing that may appear as not trivial task. - auto disk = std::static_pointer_cast(result->getDisksMap().find(disk_name)->second); - if (disk) - disk->updateFromConfig(config, disk_config_prefix, context); + + /// TODO: Implete updateFromConfigIfChanged for DiskS3 + result->getDisksMap().find(disk_name)->second->updateFromConfigIfChanged(config, disk_config_prefix, context); } } diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index b68bca07de1..ede057d8ef1 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -12,6 +12,7 @@ #include #include #include +#include namespace CurrentMetrics @@ -30,6 +31,7 @@ using Reservations = std::vector; class ReadBufferFromFileBase; class WriteBufferFromFileBase; +class Context; /** * Mode of opening a file for write. @@ -195,6 +197,11 @@ public: /// Returns executor to perform asynchronous operations. virtual Executor & getExecutor() { return *executor; } + /// Reload config if config changed + virtual void updateFromConfigIfChanged(const Poco::Util::AbstractConfiguration & /* config */, + const String & /* config_prefix */, + const Context & /* context */) { } + private: std::unique_ptr executor; }; From 3c61cecab5093812641843ab2298325db49799d0 Mon Sep 17 00:00:00 2001 From: taiyang-li Date: Wed, 10 Feb 2021 12:08:08 +0800 Subject: [PATCH 004/198] add atomic for keep_free_space_bytes --- src/Disks/DiskLocal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index d9f3434cf5e..59fb6196ab3 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -118,7 +118,7 @@ private: private: const String name; const String disk_path; - UInt64 keep_free_space_bytes; + std::atomic keep_free_space_bytes; UInt64 reserved_bytes = 0; UInt64 reservation_count = 0; From 89d8c1264c95857d883c6e508ef03ebc32733fe5 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Fri, 19 Feb 2021 18:11:21 +0800 Subject: [PATCH 005/198] add test of upload disk --- src/Disks/DiskSelector.cpp | 5 +- .../test.py | 65 +++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/src/Disks/DiskSelector.cpp b/src/Disks/DiskSelector.cpp index f8dc5cd22ff..dc84a0a8af0 100644 --- a/src/Disks/DiskSelector.cpp +++ b/src/Disks/DiskSelector.cpp @@ -75,10 +75,7 @@ DiskSelectorPtr DiskSelector::updateFromConfig( { old_disks_minus_new_disks.erase(disk_name); - /// TODO: Ideally ClickHouse shall complain if disk has changed, but - /// implementing that may appear as not trivial task. - - /// TODO: Implete updateFromConfigIfChanged for DiskS3 + // TODO: Implement updateFromConfigIfChanged for DiskS3 (DiskLocal already done) result->getDisksMap().find(disk_name)->second->updateFromConfigIfChanged(config, disk_config_prefix, context); } } diff --git a/tests/integration/test_reloading_storage_configuration/test.py b/tests/integration/test_reloading_storage_configuration/test.py index edcba4e8a60..485069e1808 100644 --- a/tests/integration/test_reloading_storage_configuration/test.py +++ b/tests/integration/test_reloading_storage_configuration/test.py @@ -7,8 +7,10 @@ import xml.etree.ElementTree as ET import helpers.client import helpers.cluster +from helpers.test_tools import TSV import pytest + cluster = helpers.cluster.ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', @@ -76,6 +78,37 @@ def add_disk(node, name, path, separate_file=False): else: tree.write(os.path.join(node.config_d_dir, "storage_configuration.xml")) +def update_disk(node, name, path, keep_free_space_bytes, separate_file=False): + separate_configuration_path = os.path.join(node.config_d_dir, + "separate_configuration.xml") + + try: + if separate_file: + tree = ET.parse(separate_configuration_path) + else: + tree = ET.parse( + os.path.join(node.config_d_dir, "storage_configuration.xml")) + except: + tree = ET.ElementTree( + ET.fromstring('')) + + root = tree.getroot() + disk = root.find("storage_configuration").find("disks").find(name) + assert disk is not None + + new_path = disk.find("path") + assert new_path is not None + new_path.text = path + + new_keep_free_space_bytes = disk.find("keep_free_space_bytes") + assert new_keep_free_space_bytes is not None + new_keep_free_space_bytes.text = keep_free_space_bytes + + if separate_file: + tree.write(separate_configuration_path) + else: + tree.write(os.path.join(node.config_d_dir, "storage_configuration.xml")) + def add_policy(node, name, volumes): tree = ET.parse(os.path.join(node.config_d_dir, "storage_configuration.xml")) @@ -123,6 +156,38 @@ def test_add_disk(started_cluster): except: """""" +def test_update_disk(started_cluster): + try: + name = "test_update_disk" + engine = "MergeTree()" + + start_over() + node1.restart_clickhouse(kill=True) + time.sleep(2) + + node1.query(""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = {engine} + ORDER BY d + SETTINGS storage_policy='jbods_with_external' + """.format(name=name, engine=engine)) + + assert node1.query( "SELECT path, keep_free_space FROM system.disks where name = '%s'" % (name)) == TSV([ + ["/jbod2/", "10485760"]]) + + update_disk(node1, "jbod2", "/jbod2/", "20971520") + node1.query("SYSTEM RELOAD CONFIG") + + assert node1.query( + "SELECT path, keep_free_space FROM system.disks where name = '%s'" % (name)) == TSV([ + ["/jbod2/", "20971520"]]) + + finally: + try: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + except: + """""" def test_add_disk_to_separate_config(started_cluster): try: From e76b224f722517cb2833ff98c84223188af0e437 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Fri, 19 Feb 2021 20:09:26 +0800 Subject: [PATCH 006/198] fix code style --- src/Disks/DiskLocal.cpp | 2 +- src/Disks/DiskLocal.h | 2 +- src/Disks/IDisk.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index 86bf358b11f..03163aa1770 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -391,7 +391,7 @@ void DiskLocal::sync(int fd) const throw Exception("Cannot fsync", ErrorCodes::CANNOT_FSYNC); } -void DiskLocal::updateFromConfigIfChanged(const Poco::Util::AbstractConfiguration & config, +void DiskLocal::updateFromConfigIfChanged(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, const Context & context) { diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index 59fb6196ab3..8d32a8e2176 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -108,7 +108,7 @@ public: const String getType() const override { return "local"; } - void updateFromConfigIfChanged(const Poco::Util::AbstractConfiguration & config, + void updateFromConfigIfChanged(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, const Context & context) override; diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index ede057d8ef1..0ce3bddbf34 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -198,7 +198,7 @@ public: virtual Executor & getExecutor() { return *executor; } /// Reload config if config changed - virtual void updateFromConfigIfChanged(const Poco::Util::AbstractConfiguration & /* config */, + virtual void updateFromConfigIfChanged(const Poco::Util::AbstractConfiguration & /* config */, const String & /* config_prefix */, const Context & /* context */) { } From d370381980e2d5a80f9c1515096d58153b2a507d Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sat, 20 Feb 2021 09:47:53 +0800 Subject: [PATCH 007/198] fix code style --- src/Disks/DiskLocal.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index 03163aa1770..bd5e20ab2b9 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -401,7 +401,6 @@ void DiskLocal::updateFromConfigIfChanged(const Poco::Util::AbstractConfiguratio if (disk_path != new_disk_path) throw Exception("Disk path can't update from config " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); - if (keep_free_space_bytes != new_keep_free_space_bytes) keep_free_space_bytes = new_keep_free_space_bytes; } From 1980e75a962dab55bd54468f2c454ef946596911 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sat, 20 Feb 2021 11:01:11 +0800 Subject: [PATCH 008/198] fix code style --- src/Disks/DiskLocal.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index 2a31bc3f00a..77d2b70b36a 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -83,8 +83,6 @@ static void loadDiskLocalConfig(const String & name, } } - - class DiskLocalReservation : public IReservation { public: @@ -380,7 +378,6 @@ void DiskLocal::updateFromConfigIfChanged(const Poco::Util::AbstractConfiguratio keep_free_space_bytes = new_keep_free_space_bytes; } - DiskPtr DiskLocalReservation::getDisk(size_t i) const { if (i != 0) @@ -398,7 +395,6 @@ void DiskLocalReservation::update(UInt64 new_size) disk->reserved_bytes += size; } - DiskLocalReservation::~DiskLocalReservation() { try From 50d013231ca494056e8e7e8c26ff1fa3dc917549 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sat, 20 Feb 2021 11:33:49 +0800 Subject: [PATCH 009/198] fix test_reloading_storage_configuration --- .../test_reloading_storage_configuration/test.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_reloading_storage_configuration/test.py b/tests/integration/test_reloading_storage_configuration/test.py index 485069e1808..7fa37d0909c 100644 --- a/tests/integration/test_reloading_storage_configuration/test.py +++ b/tests/integration/test_reloading_storage_configuration/test.py @@ -173,16 +173,14 @@ def test_update_disk(started_cluster): SETTINGS storage_policy='jbods_with_external' """.format(name=name, engine=engine)) - assert node1.query( "SELECT path, keep_free_space FROM system.disks where name = '%s'" % (name)) == TSV([ + assert node1.query("SELECT path, keep_free_space FROM system.disks where name = 'jbod2'") == TSV([ ["/jbod2/", "10485760"]]) update_disk(node1, "jbod2", "/jbod2/", "20971520") node1.query("SYSTEM RELOAD CONFIG") - assert node1.query( - "SELECT path, keep_free_space FROM system.disks where name = '%s'" % (name)) == TSV([ + assert node1.query("SELECT path, keep_free_space FROM system.disks where name = 'jbod2'") == TSV([ ["/jbod2/", "20971520"]]) - finally: try: node1.query("DROP TABLE IF EXISTS {}".format(name)) From fa8ad1a7c6295839ecd36db734d809e47c105550 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 25 Jun 2021 19:22:39 +0300 Subject: [PATCH 010/198] reduce memory usage in queries with 'ORDER BY primary_key' --- .../gtest_blocks_size_merging_streams.cpp | 4 +- .../Merges/AggregatingSortedTransform.h | 2 +- .../Merges/CollapsingSortedTransform.h | 2 +- .../FinishAggregatingInOrderTransform.h | 2 +- .../Merges/GraphiteRollupSortedTransform.h | 2 +- src/Processors/Merges/IMergingTransform.cpp | 16 ++-- src/Processors/Merges/IMergingTransform.h | 7 +- .../Merges/MergingSortedTransform.cpp | 3 +- .../Merges/MergingSortedTransform.h | 1 + .../Merges/ReplacingSortedTransform.h | 2 +- .../Merges/SummingSortedTransform.h | 2 +- .../Merges/VersionedCollapsingTransform.h | 2 +- .../QueryPlan/FinishSortingStep.cpp | 5 +- .../QueryPlan/ReadFromMergeTree.cpp | 19 +++-- src/Processors/QueryPlan/ReadFromMergeTree.h | 2 +- .../Transforms/MergeSortingTransform.cpp | 1 + .../MergeTree/MergeTreeDataMergerMutator.cpp | 2 +- .../MergeTreeInOrderSelectProcessor.cpp | 80 ++++++++++++++++++ .../MergeTreeInOrderSelectProcessor.h | 39 +++++++++ .../MergeTreeReverseSelectProcessor.cpp | 66 ++------------- .../MergeTreeReverseSelectProcessor.h | 44 ++-------- .../MergeTree/MergeTreeSelectProcessor.cpp | 83 ++++--------------- .../MergeTree/MergeTreeSelectProcessor.h | 20 ++--- .../01926_order_by_desc_limit.reference | 2 + .../0_stateless/01926_order_by_desc_limit.sql | 21 +++++ 25 files changed, 226 insertions(+), 203 deletions(-) create mode 100644 src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp create mode 100644 src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h create mode 100644 tests/queries/0_stateless/01926_order_by_desc_limit.reference create mode 100644 tests/queries/0_stateless/01926_order_by_desc_limit.sql diff --git a/src/DataStreams/tests/gtest_blocks_size_merging_streams.cpp b/src/DataStreams/tests/gtest_blocks_size_merging_streams.cpp index 0ce450c4e6c..2fc2dc77011 100644 --- a/src/DataStreams/tests/gtest_blocks_size_merging_streams.cpp +++ b/src/DataStreams/tests/gtest_blocks_size_merging_streams.cpp @@ -84,7 +84,7 @@ TEST(MergingSortedTest, SimpleBlockSizeTest) EXPECT_EQ(pipe.numOutputPorts(), 3); auto transform = std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), sort_description, - DEFAULT_MERGE_BLOCK_SIZE, 0, nullptr, false, true); + DEFAULT_MERGE_BLOCK_SIZE, 0, false, nullptr, false, true); pipe.addTransform(std::move(transform)); @@ -129,7 +129,7 @@ TEST(MergingSortedTest, MoreInterestingBlockSizes) EXPECT_EQ(pipe.numOutputPorts(), 3); auto transform = std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), sort_description, - DEFAULT_MERGE_BLOCK_SIZE, 0, nullptr, false, true); + DEFAULT_MERGE_BLOCK_SIZE, 0, false, nullptr, false, true); pipe.addTransform(std::move(transform)); diff --git a/src/Processors/Merges/AggregatingSortedTransform.h b/src/Processors/Merges/AggregatingSortedTransform.h index a0425d4c376..b4a7d4c8106 100644 --- a/src/Processors/Merges/AggregatingSortedTransform.h +++ b/src/Processors/Merges/AggregatingSortedTransform.h @@ -16,7 +16,7 @@ public: const Block & header, size_t num_inputs, SortDescription description_, size_t max_block_size) : IMergingTransform( - num_inputs, header, header, true, + num_inputs, header, header, true, false, header, num_inputs, std::move(description_), diff --git a/src/Processors/Merges/CollapsingSortedTransform.h b/src/Processors/Merges/CollapsingSortedTransform.h index 9e6bd306eee..059e90c1de8 100644 --- a/src/Processors/Merges/CollapsingSortedTransform.h +++ b/src/Processors/Merges/CollapsingSortedTransform.h @@ -20,7 +20,7 @@ public: WriteBuffer * out_row_sources_buf_ = nullptr, bool use_average_block_sizes = false) : IMergingTransform( - num_inputs, header, header, true, + num_inputs, header, header, true, false, header, num_inputs, std::move(description_), diff --git a/src/Processors/Merges/FinishAggregatingInOrderTransform.h b/src/Processors/Merges/FinishAggregatingInOrderTransform.h index 4f9e53bd7d5..f3bcc9dd878 100644 --- a/src/Processors/Merges/FinishAggregatingInOrderTransform.h +++ b/src/Processors/Merges/FinishAggregatingInOrderTransform.h @@ -19,7 +19,7 @@ public: SortDescription description, size_t max_block_size) : IMergingTransform( - num_inputs, header, header, true, + num_inputs, header, header, true, false, header, num_inputs, params, diff --git a/src/Processors/Merges/GraphiteRollupSortedTransform.h b/src/Processors/Merges/GraphiteRollupSortedTransform.h index 5104801aa0d..9432482fe16 100644 --- a/src/Processors/Merges/GraphiteRollupSortedTransform.h +++ b/src/Processors/Merges/GraphiteRollupSortedTransform.h @@ -15,7 +15,7 @@ public: SortDescription description_, size_t max_block_size, Graphite::Params params_, time_t time_of_merge_) : IMergingTransform( - num_inputs, header, header, true, + num_inputs, header, header, true, false, header, num_inputs, std::move(description_), diff --git a/src/Processors/Merges/IMergingTransform.cpp b/src/Processors/Merges/IMergingTransform.cpp index eff786b150f..8be21e3e291 100644 --- a/src/Processors/Merges/IMergingTransform.cpp +++ b/src/Processors/Merges/IMergingTransform.cpp @@ -14,9 +14,11 @@ IMergingTransformBase::IMergingTransformBase( size_t num_inputs, const Block & input_header, const Block & output_header, - bool have_all_inputs_) + bool have_all_inputs_, + bool expected_one_block_) : IProcessor(InputPorts(num_inputs, input_header), {output_header}) , have_all_inputs(have_all_inputs_) + , expected_one_block(expected_one_block_) { } @@ -64,10 +66,7 @@ IProcessor::Status IMergingTransformBase::prepareInitializeInputs() continue; if (input_states[i].is_initialized) - { - // input.setNotNeeded(); continue; - } input.setNeeded(); @@ -77,12 +76,17 @@ IProcessor::Status IMergingTransformBase::prepareInitializeInputs() continue; } - auto chunk = input.pull(); + /// setNotNeeded after reading first chunk, because in optimismtic case + /// (e.g. with optimized 'ORDER BY primary_key LIMIT n' and small 'n') + /// we won't have to read any chunks anymore; + auto chunk = input.pull(expected_one_block); if (!chunk.hasRows()) { - if (!input.isFinished()) + { + input.setNeeded(); all_inputs_has_data = false; + } continue; } diff --git a/src/Processors/Merges/IMergingTransform.h b/src/Processors/Merges/IMergingTransform.h index ce673131ab6..695aa4eb896 100644 --- a/src/Processors/Merges/IMergingTransform.h +++ b/src/Processors/Merges/IMergingTransform.h @@ -16,7 +16,8 @@ public: size_t num_inputs, const Block & input_header, const Block & output_header, - bool have_all_inputs_); + bool have_all_inputs_, + bool expected_one_block_); OutputPort & getOutputPort() { return outputs.front(); } @@ -66,6 +67,7 @@ private: std::vector input_states; std::atomic have_all_inputs; bool is_initialized = false; + bool expected_one_block = false; IProcessor::Status prepareInitializeInputs(); }; @@ -81,8 +83,9 @@ public: const Block & input_header, const Block & output_header, bool have_all_inputs_, + bool expected_one_block_, Args && ... args) - : IMergingTransformBase(num_inputs, input_header, output_header, have_all_inputs_) + : IMergingTransformBase(num_inputs, input_header, output_header, have_all_inputs_, expected_one_block_) , algorithm(std::forward(args) ...) { } diff --git a/src/Processors/Merges/MergingSortedTransform.cpp b/src/Processors/Merges/MergingSortedTransform.cpp index ec1bdc59683..f185ec6ad8e 100644 --- a/src/Processors/Merges/MergingSortedTransform.cpp +++ b/src/Processors/Merges/MergingSortedTransform.cpp @@ -13,12 +13,13 @@ MergingSortedTransform::MergingSortedTransform( SortDescription description_, size_t max_block_size, UInt64 limit_, + bool expected_one_block_, WriteBuffer * out_row_sources_buf_, bool quiet_, bool use_average_block_sizes, bool have_all_inputs_) : IMergingTransform( - num_inputs, header, header, have_all_inputs_, + num_inputs, header, header, have_all_inputs_, expected_one_block_, header, num_inputs, std::move(description_), diff --git a/src/Processors/Merges/MergingSortedTransform.h b/src/Processors/Merges/MergingSortedTransform.h index 93bd36d8aec..0e616eb2c78 100644 --- a/src/Processors/Merges/MergingSortedTransform.h +++ b/src/Processors/Merges/MergingSortedTransform.h @@ -17,6 +17,7 @@ public: SortDescription description, size_t max_block_size, UInt64 limit_ = 0, + bool expected_one_block_ = false, WriteBuffer * out_row_sources_buf_ = nullptr, bool quiet_ = false, bool use_average_block_sizes = false, diff --git a/src/Processors/Merges/ReplacingSortedTransform.h b/src/Processors/Merges/ReplacingSortedTransform.h index 757e19e2cbe..c1df68f3a76 100644 --- a/src/Processors/Merges/ReplacingSortedTransform.h +++ b/src/Processors/Merges/ReplacingSortedTransform.h @@ -18,7 +18,7 @@ public: WriteBuffer * out_row_sources_buf_ = nullptr, bool use_average_block_sizes = false) : IMergingTransform( - num_inputs, header, header, true, + num_inputs, header, header, true, false, header, num_inputs, std::move(description_), diff --git a/src/Processors/Merges/SummingSortedTransform.h b/src/Processors/Merges/SummingSortedTransform.h index 22361bb1a44..730d6604950 100644 --- a/src/Processors/Merges/SummingSortedTransform.h +++ b/src/Processors/Merges/SummingSortedTransform.h @@ -19,7 +19,7 @@ public: const Names & partition_key_columns, size_t max_block_size) : IMergingTransform( - num_inputs, header, header, true, + num_inputs, header, header, true, false, header, num_inputs, std::move(description_), diff --git a/src/Processors/Merges/VersionedCollapsingTransform.h b/src/Processors/Merges/VersionedCollapsingTransform.h index f593734c603..6251f842898 100644 --- a/src/Processors/Merges/VersionedCollapsingTransform.h +++ b/src/Processors/Merges/VersionedCollapsingTransform.h @@ -19,7 +19,7 @@ public: WriteBuffer * out_row_sources_buf_ = nullptr, bool use_average_block_sizes = false) : IMergingTransform( - num_inputs, header, header, true, + num_inputs, header, header, true, false, header, num_inputs, std::move(description_), diff --git a/src/Processors/QueryPlan/FinishSortingStep.cpp b/src/Processors/QueryPlan/FinishSortingStep.cpp index a2e056b3029..6f21865592b 100644 --- a/src/Processors/QueryPlan/FinishSortingStep.cpp +++ b/src/Processors/QueryPlan/FinishSortingStep.cpp @@ -58,11 +58,14 @@ void FinishSortingStep::transformPipeline(QueryPipeline & pipeline, const BuildQ if (pipeline.getNumStreams() > 1) { UInt64 limit_for_merging = (need_finish_sorting ? 0 : limit); + bool expected_one_block = limit_for_merging && limit_for_merging < max_block_size; auto transform = std::make_shared( pipeline.getHeader(), pipeline.getNumStreams(), prefix_description, - max_block_size, limit_for_merging); + max_block_size, + limit_for_merging, + expected_one_block); pipeline.addTransform(std::move(transform)); } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index fd5de98b4c0..3698fa09020 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -175,12 +175,13 @@ template ProcessorPtr ReadFromMergeTree::createSource( const RangesInDataPart & part, const Names & required_columns, - bool use_uncompressed_cache) + bool use_uncompressed_cache, + bool one_range_per_task) { return std::make_shared( data, metadata_snapshot, part.data_part, max_block_size, preferred_block_size_bytes, preferred_max_column_in_block_size_bytes, required_columns, part.ranges, use_uncompressed_cache, - prewhere_info, true, reader_settings, virt_column_names, part.part_index_in_query); + prewhere_info, true, reader_settings, virt_column_names, part.part_index_in_query, one_range_per_task); } Pipe ReadFromMergeTree::readInOrder( @@ -190,11 +191,15 @@ Pipe ReadFromMergeTree::readInOrder( bool use_uncompressed_cache) { Pipes pipes; + /// For reading in order it makes sense to read only + /// one range per task to reduce number of read rows. + bool one_range_per_task = read_type != ReadType::Default; + for (const auto & part : parts_with_range) { auto source = read_type == ReadType::InReverseOrder - ? createSource(part, required_columns, use_uncompressed_cache) - : createSource(part, required_columns, use_uncompressed_cache); + ? createSource(part, required_columns, use_uncompressed_cache, one_range_per_task) + : createSource(part, required_columns, use_uncompressed_cache, one_range_per_task); pipes.emplace_back(std::move(source)); } @@ -445,6 +450,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( } parts_with_ranges.emplace_back(part); } + ranges_to_get_from_part = split_ranges(ranges_to_get_from_part, input_order_info->direction); new_parts.emplace_back(part.data_part, part.part_index_in_query, std::move(ranges_to_get_from_part)); } @@ -482,7 +488,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( pipe.getHeader(), pipe.numOutputPorts(), sort_description, - max_block_size); + max_block_size, + 0, true); pipe.addTransform(std::move(transform)); } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 6e1efffdb02..d44a6e80cf6 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -111,7 +111,7 @@ private: Pipe readInOrder(RangesInDataParts parts_with_range, Names required_columns, ReadType read_type, bool use_uncompressed_cache); template - ProcessorPtr createSource(const RangesInDataPart & part, const Names & required_columns, bool use_uncompressed_cache); + ProcessorPtr createSource(const RangesInDataPart & part, const Names & required_columns, bool use_uncompressed_cache, bool one_range_per_task); Pipe spreadMarkRangesAmongStreams( RangesInDataParts && parts_with_ranges, diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index 1806693db3a..ca78a29071e 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -200,6 +200,7 @@ void MergeSortingTransform::consume(Chunk chunk) description, max_merged_block_size, limit, + false, nullptr, quiet, use_average_block_sizes, diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 766d988500d..200ff4f681d 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -891,7 +891,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor { case MergeTreeData::MergingParams::Ordinary: merged_transform = std::make_unique( - header, pipes.size(), sort_description, merge_block_size, 0, rows_sources_write_buf.get(), true, blocks_are_granules_size); + header, pipes.size(), sort_description, merge_block_size, 0, false, rows_sources_write_buf.get(), true, blocks_are_granules_size); break; case MergeTreeData::MergingParams::Collapsing: diff --git a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp new file mode 100644 index 00000000000..8e959249d93 --- /dev/null +++ b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp @@ -0,0 +1,80 @@ +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int MEMORY_LIMIT_EXCEEDED; +} + +MergeTreeInOrderSelectProcessor::MergeTreeInOrderSelectProcessor( + const MergeTreeData & storage_, + const StorageMetadataPtr & metadata_snapshot_, + const MergeTreeData::DataPartPtr & owned_data_part_, + UInt64 max_block_size_rows_, + size_t preferred_block_size_bytes_, + size_t preferred_max_column_in_block_size_bytes_, + Names required_columns_, + MarkRanges mark_ranges_, + bool use_uncompressed_cache_, + const PrewhereInfoPtr & prewhere_info_, + bool check_columns_, + const MergeTreeReaderSettings & reader_settings_, + const Names & virt_column_names_, + bool one_range_per_task_, + bool quiet) + : MergeTreeSelectProcessor{ + storage_, metadata_snapshot_, owned_data_part_, max_block_size_rows_, + preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, + required_columns_, std::move(mark_ranges_), use_uncompressed_cache_, prewhere_info_, + check_columns_, reader_settings_, virt_column_names_, one_range_per_task_} +{ + if (!quiet) + LOG_DEBUG(log, "Reading {} ranges in order from part {}, approx. {} rows starting from {}", + all_mark_ranges.size(), data_part->name, total_rows, + data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); +} + +bool MergeTreeInOrderSelectProcessor::getNewTask() +try +{ + /// Produce no more than one task + if (all_mark_ranges.empty()) + { + finish(); + return false; + } + + auto size_predictor = (preferred_block_size_bytes == 0) + ? nullptr + : std::make_unique(data_part, ordered_names, metadata_snapshot->getSampleBlock()); + + MarkRanges mark_ranges_for_task; + if (one_range_per_task) + { + mark_ranges_for_task = { std::move(all_mark_ranges.front()) }; + all_mark_ranges.pop_front(); + } + else + { + mark_ranges_for_task = std::move(all_mark_ranges); + all_mark_ranges.clear(); + } + + task = std::make_unique( + data_part, mark_ranges_for_task, part_index_in_query, ordered_names, column_name_set, task_columns.columns, + task_columns.pre_columns, prewhere_info && prewhere_info->remove_prewhere_column, + task_columns.should_reorder, std::move(size_predictor)); + + return true; +} +catch (...) +{ + /// Suspicion of the broken part. A part is added to the queue for verification. + if (getCurrentExceptionCode() != ErrorCodes::MEMORY_LIMIT_EXCEEDED) + storage.reportBrokenPart(data_part->name); + throw; +} + +} diff --git a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h new file mode 100644 index 00000000000..1a300620278 --- /dev/null +++ b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h @@ -0,0 +1,39 @@ +#pragma once +#include + +namespace DB +{ + + +/// Used to read data from single part with select query in order of primary key. +/// Cares about PREWHERE, virtual columns, indexes etc. +/// To read data from multiple parts, Storage (MergeTree) creates multiple such objects. +class MergeTreeInOrderSelectProcessor : public MergeTreeSelectProcessor +{ +public: + MergeTreeInOrderSelectProcessor( + const MergeTreeData & storage, + const StorageMetadataPtr & metadata_snapshot, + const MergeTreeData::DataPartPtr & owned_data_part, + UInt64 max_block_size_rows, + size_t preferred_block_size_bytes, + size_t preferred_max_column_in_block_size_bytes, + Names required_columns_, + MarkRanges mark_ranges, + bool use_uncompressed_cache, + const PrewhereInfoPtr & prewhere_info, + bool check_columns, + const MergeTreeReaderSettings & reader_settings, + const Names & virt_column_names = {}, + bool one_range_per_task_ = false, + bool quiet = false); + + String getName() const override { return "MergeTreeInOrder"; } + +private: + bool getNewTask() override; + + Poco::Logger * log = &Poco::Logger::get("MergeTreeInOrderSelectProcessor"); +}; + +} diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index e9527efaa4a..c144a7d16e7 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -1,8 +1,4 @@ #include -#include -#include -#include - namespace DB { @@ -23,62 +19,27 @@ MergeTreeReverseSelectProcessor::MergeTreeReverseSelectProcessor( MarkRanges mark_ranges_, bool use_uncompressed_cache_, const PrewhereInfoPtr & prewhere_info_, - bool check_columns, + bool check_columns_, const MergeTreeReaderSettings & reader_settings_, const Names & virt_column_names_, - size_t part_index_in_query_, + bool one_range_per_task_, bool quiet) - : - MergeTreeBaseSelectProcessor{ - metadata_snapshot_->getSampleBlockForColumns(required_columns_, storage_.getVirtuals(), storage_.getStorageID()), - storage_, metadata_snapshot_, prewhere_info_, max_block_size_rows_, + : MergeTreeSelectProcessor{ + storage_, metadata_snapshot_, owned_data_part_, max_block_size_rows_, preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, - reader_settings_, use_uncompressed_cache_, virt_column_names_}, - required_columns{std::move(required_columns_)}, - data_part{owned_data_part_}, - all_mark_ranges(std::move(mark_ranges_)), - part_index_in_query(part_index_in_query_), - path(data_part->getFullRelativePath()) + required_columns_, std::move(mark_ranges_), use_uncompressed_cache_, prewhere_info_, + check_columns_, reader_settings_, virt_column_names_, one_range_per_task_} { - /// Let's estimate total number of rows for progress bar. - for (const auto & range : all_mark_ranges) - total_marks_count += range.end - range.begin; - - size_t total_rows = data_part->index_granularity.getRowsCountInRanges(all_mark_ranges); - if (!quiet) LOG_DEBUG(log, "Reading {} ranges in reverse order from part {}, approx. {} rows starting from {}", all_mark_ranges.size(), data_part->name, total_rows, data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); - - addTotalRowsApprox(total_rows); - - ordered_names = header_without_virtual_columns.getNames(); - - task_columns = getReadTaskColumns(storage, metadata_snapshot, data_part, required_columns, prewhere_info, check_columns); - - /// will be used to distinguish between PREWHERE and WHERE columns when applying filter - const auto & column_names = task_columns.columns.getNames(); - column_name_set = NameSet{column_names.begin(), column_names.end()}; - - if (use_uncompressed_cache) - owned_uncompressed_cache = storage.getContext()->getUncompressedCache(); - - owned_mark_cache = storage.getContext()->getMarkCache(); - - reader = data_part->getReader(task_columns.columns, metadata_snapshot, - all_mark_ranges, owned_uncompressed_cache.get(), - owned_mark_cache.get(), reader_settings); - - if (prewhere_info) - pre_reader = data_part->getReader(task_columns.pre_columns, metadata_snapshot, all_mark_ranges, - owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings); } bool MergeTreeReverseSelectProcessor::getNewTask() try { - if ((chunks.empty() && all_mark_ranges.empty()) || total_marks_count == 0) + if (chunks.empty() && all_mark_ranges.empty()) { finish(); return false; @@ -141,17 +102,4 @@ Chunk MergeTreeReverseSelectProcessor::readFromPart() return res; } -void MergeTreeReverseSelectProcessor::finish() -{ - /** Close the files (before destroying the object). - * When many sources are created, but simultaneously reading only a few of them, - * buffers don't waste memory. - */ - reader.reset(); - pre_reader.reset(); - data_part.reset(); -} - -MergeTreeReverseSelectProcessor::~MergeTreeReverseSelectProcessor() = default; - } diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h index c9fd06c5534..6f507f57138 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h @@ -1,19 +1,15 @@ #pragma once -#include -#include -#include -#include -#include -#include +#include namespace DB { /// Used to read data from single part with select query +/// in reverse order of primary key. /// Cares about PREWHERE, virtual columns, indexes etc. /// To read data from multiple parts, Storage (MergeTree) creates multiple such objects. -class MergeTreeReverseSelectProcessor : public MergeTreeBaseSelectProcessor +class MergeTreeReverseSelectProcessor : public MergeTreeSelectProcessor { public: MergeTreeReverseSelectProcessor( @@ -30,46 +26,16 @@ public: bool check_columns, const MergeTreeReaderSettings & reader_settings, const Names & virt_column_names = {}, - size_t part_index_in_query = 0, + bool one_range_per_task_ = false, bool quiet = false); - ~MergeTreeReverseSelectProcessor() override; - String getName() const override { return "MergeTreeReverse"; } - /// Closes readers and unlock part locks - void finish(); - -protected: - +private: bool getNewTask() override; Chunk readFromPart() override; -private: - Block header; - - /// Used by Task - Names required_columns; - /// Names from header. Used in order to order columns in read blocks. - Names ordered_names; - NameSet column_name_set; - - MergeTreeReadTaskColumns task_columns; - - /// Data part will not be removed if the pointer owns it - MergeTreeData::DataPartPtr data_part; - - /// Mark ranges we should read (in ascending order) - MarkRanges all_mark_ranges; - /// Total number of marks we should read - size_t total_marks_count = 0; - /// Value of _part_index virtual column (used only in SelectExecutor) - size_t part_index_in_query = 0; - - String path; - Chunks chunks; - Poco::Logger * log = &Poco::Logger::get("MergeTreeReverseSelectProcessor"); }; diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 980afa170e9..4bb4799f396 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -26,10 +26,8 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( bool check_columns_, const MergeTreeReaderSettings & reader_settings_, const Names & virt_column_names_, - size_t part_index_in_query_, - bool quiet) - : - MergeTreeBaseSelectProcessor{ + bool one_range_per_task_) + : MergeTreeBaseSelectProcessor{ metadata_snapshot_->getSampleBlockForColumns(required_columns_, storage_.getVirtuals(), storage_.getStorageID()), storage_, metadata_snapshot_, prewhere_info_, max_block_size_rows_, preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, @@ -37,78 +35,33 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( required_columns{std::move(required_columns_)}, data_part{owned_data_part_}, all_mark_ranges(std::move(mark_ranges_)), - part_index_in_query(part_index_in_query_), - check_columns(check_columns_) + one_range_per_task(one_range_per_task_), + check_columns(check_columns_), + total_rows(data_part->index_granularity.getRowsCountInRanges(all_mark_ranges)) { - /// Let's estimate total number of rows for progress bar. - for (const auto & range : all_mark_ranges) - total_marks_count += range.end - range.begin; - - size_t total_rows = data_part->index_granularity.getRowsCountInRanges(all_mark_ranges); - - if (!quiet) - LOG_DEBUG(log, "Reading {} ranges from part {}, approx. {} rows starting from {}", - all_mark_ranges.size(), data_part->name, total_rows, - data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); - - addTotalRowsApprox(total_rows); - ordered_names = header_without_virtual_columns.getNames(); -} - - -bool MergeTreeSelectProcessor::getNewTask() -try -{ - /// Produce no more than one task - if (!is_first_task || total_marks_count == 0) - { - finish(); - return false; - } - is_first_task = false; + /// will be used to distinguish between PREWHERE and WHERE columns when applying filter + const auto & column_names = task_columns.columns.getNames(); + column_name_set = NameSet{column_names.begin(), column_names.end()}; task_columns = getReadTaskColumns( storage, metadata_snapshot, data_part, required_columns, prewhere_info, check_columns); - auto size_predictor = (preferred_block_size_bytes == 0) - ? nullptr - : std::make_unique(data_part, ordered_names, metadata_snapshot->getSampleBlock()); + if (use_uncompressed_cache) + owned_uncompressed_cache = storage.getContext()->getUncompressedCache(); - /// will be used to distinguish between PREWHERE and WHERE columns when applying filter - const auto & column_names = task_columns.columns.getNames(); - column_name_set = NameSet{column_names.begin(), column_names.end()}; + owned_mark_cache = storage.getContext()->getMarkCache(); - task = std::make_unique( - data_part, all_mark_ranges, part_index_in_query, ordered_names, column_name_set, task_columns.columns, - task_columns.pre_columns, prewhere_info && prewhere_info->remove_prewhere_column, - task_columns.should_reorder, std::move(size_predictor)); + reader = data_part->getReader(task_columns.columns, metadata_snapshot, all_mark_ranges, + owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings); - if (!reader) - { - if (use_uncompressed_cache) - owned_uncompressed_cache = storage.getContext()->getUncompressedCache(); - - owned_mark_cache = storage.getContext()->getMarkCache(); - - reader = data_part->getReader(task_columns.columns, metadata_snapshot, all_mark_ranges, + if (prewhere_info) + pre_reader = data_part->getReader(task_columns.pre_columns, metadata_snapshot, all_mark_ranges, owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings); - if (prewhere_info) - pre_reader = data_part->getReader(task_columns.pre_columns, metadata_snapshot, all_mark_ranges, - owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings); - } - - return true; + addTotalRowsApprox(total_rows); + ordered_names = header_without_virtual_columns.getNames(); } -catch (...) -{ - /// Suspicion of the broken part. A part is added to the queue for verification. - if (getCurrentExceptionCode() != ErrorCodes::MEMORY_LIMIT_EXCEEDED) - storage.reportBrokenPart(data_part->name); - throw; -} - void MergeTreeSelectProcessor::finish() { @@ -121,8 +74,6 @@ void MergeTreeSelectProcessor::finish() data_part.reset(); } - MergeTreeSelectProcessor::~MergeTreeSelectProcessor() = default; - } diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 925c437f1ce..7b938a77d69 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -1,6 +1,6 @@ #pragma once #include -#include +#include #include #include #include @@ -30,21 +30,18 @@ public: bool check_columns, const MergeTreeReaderSettings & reader_settings, const Names & virt_column_names = {}, - size_t part_index_in_query = 0, - bool quiet = false); + bool one_range_per_task_ = false); ~MergeTreeSelectProcessor() override; - String getName() const override { return "MergeTree"; } + String getName() const override = 0; /// Closes readers and unlock part locks void finish(); protected: - bool getNewTask() override; - -private: + bool getNewTask() override = 0; /// Used by Task Names required_columns; @@ -59,15 +56,14 @@ private: /// Mark ranges we should read (in ascending order) MarkRanges all_mark_ranges; - /// Total number of marks we should read - size_t total_marks_count = 0; /// Value of _part_index virtual column (used only in SelectExecutor) size_t part_index_in_query = 0; + /// If true, every task will be created only with one range. + /// It reduces amount of read data for queries with small LIMIT. + bool one_range_per_task = false; bool check_columns; - bool is_first_task = true; - - Poco::Logger * log = &Poco::Logger::get("MergeTreeSelectProcessor"); + size_t total_rows; }; } diff --git a/tests/queries/0_stateless/01926_order_by_desc_limit.reference b/tests/queries/0_stateless/01926_order_by_desc_limit.reference new file mode 100644 index 00000000000..6ed281c757a --- /dev/null +++ b/tests/queries/0_stateless/01926_order_by_desc_limit.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/01926_order_by_desc_limit.sql b/tests/queries/0_stateless/01926_order_by_desc_limit.sql new file mode 100644 index 00000000000..7ea102e11e9 --- /dev/null +++ b/tests/queries/0_stateless/01926_order_by_desc_limit.sql @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS order_by_desc; + +CREATE TABLE order_by_desc (u UInt32, s String) +ENGINE MergeTree ORDER BY u PARTITION BY u % 100 +SETTINGS index_granularity = 1024; + +INSERT INTO order_by_desc SELECT number, repeat('a', 1024) FROM numbers(1024 * 300); +OPTIMIZE TABLE order_by_desc FINAL; + +SELECT s FROM order_by_desc ORDER BY u DESC LIMIT 10 FORMAT Null +SETTINGS max_memory_usage = '400M'; + +SELECT s FROM order_by_desc ORDER BY u LIMIT 10 FORMAT Null +SETTINGS max_memory_usage = '400M'; + +SYSTEM FLUSH LOGS; + +SELECT read_rows < 110000 FROM system.query_log +WHERE type = 'QueryFinish' AND current_database = currentDatabase() +AND event_time > now() - INTERVAL 10 SECOND +AND lower(query) LIKE lower('SELECT s FROM order_by_desc ORDER BY u%'); From fafae3ab8d05d63adf9fc169707723d52d14b983 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 25 Jun 2021 19:58:25 +0300 Subject: [PATCH 011/198] fix tests --- src/Storages/MergeTree/MergeTreeSelectProcessor.cpp | 5 ----- .../01551_mergetree_read_in_order_spread.reference | 6 +++--- tests/queries/0_stateless/01861_explain_pipeline.reference | 4 ++-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 4bb4799f396..f849bfff1e9 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -7,11 +7,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int MEMORY_LIMIT_EXCEEDED; -} - MergeTreeSelectProcessor::MergeTreeSelectProcessor( const MergeTreeData & storage_, const StorageMetadataPtr & metadata_snapshot_, diff --git a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference index 2843b305f0a..cdc595a3c57 100644 --- a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference +++ b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference @@ -9,9 +9,9 @@ ExpressionTransform (SettingQuotaAndLimits) (ReadFromMergeTree) ExpressionTransform × 4 - MergeTree 0 → 1 + MergeTreeInOrder 0 → 1 MergingSortedTransform 2 → 1 ExpressionTransform × 2 - MergeTree × 2 0 → 1 + MergeTreeInOrder × 2 0 → 1 ExpressionTransform - MergeTree 0 → 1 + MergeTreeInOrder 0 → 1 diff --git a/tests/queries/0_stateless/01861_explain_pipeline.reference b/tests/queries/0_stateless/01861_explain_pipeline.reference index 9d62fb9f6b8..63ba55f5a04 100644 --- a/tests/queries/0_stateless/01861_explain_pipeline.reference +++ b/tests/queries/0_stateless/01861_explain_pipeline.reference @@ -5,7 +5,7 @@ ExpressionTransform ExpressionTransform ReplacingSorted 2 → 1 ExpressionTransform × 2 - MergeTree × 2 0 → 1 + MergeTreeInOrder × 2 0 → 1 0 0 1 1 2 2 @@ -22,4 +22,4 @@ ExpressionTransform × 2 Copy × 2 1 → 2 AddingSelector × 2 ExpressionTransform × 2 - MergeTree × 2 0 → 1 + MergeTreeInOrder × 2 0 → 1 From 2828b4ed73eb196c1b5210065e46e7a1dc5a4eeb Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Sun, 27 Jun 2021 17:43:20 +0300 Subject: [PATCH 012/198] fix tests --- src/Storages/ya.make | 1 + tests/queries/skip_list.json | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Storages/ya.make b/src/Storages/ya.make index d8692524210..61369177ee7 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -58,6 +58,7 @@ SRCS( MergeTree/MergeTreeDataSelectExecutor.cpp MergeTree/MergeTreeDataWriter.cpp MergeTree/MergeTreeDeduplicationLog.cpp + MergeTree/MergeTreeInOrderSelectProcessor.cpp MergeTree/MergeTreeIndexAggregatorBloomFilter.cpp MergeTree/MergeTreeIndexBloomFilter.cpp MergeTree/MergeTreeIndexConditionBloomFilter.cpp diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 52c2d468498..fb69464bf23 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -516,7 +516,8 @@ "01913_if_int_decimal", "01913_join_push_down_bug", "01921_with_fill_with_totals", - "01924_argmax_bitmap_state" + "01924_argmax_bitmap_state", + "01926_order_by_desc_limit" ], "parallel": [ From d9a77e3a1a9dd16ac5a6c08e0a86886732d7a493 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 1 Jul 2021 16:21:38 +0300 Subject: [PATCH 013/198] improve CREATE OR REPLACE query --- src/Interpreters/InterpreterCreateQuery.cpp | 77 +++++++++---------- src/Interpreters/InterpreterRenameQuery.cpp | 16 +++- src/Interpreters/InterpreterRenameQuery.h | 3 + src/Interpreters/executeQuery.cpp | 70 ++++++++++++----- src/Interpreters/executeQuery.h | 4 + src/Parsers/ASTRenameQuery.h | 3 + src/Storages/StorageDistributed.cpp | 5 ++ src/Storages/StorageDistributed.h | 1 + .../0_stateless/01157_replace_table.reference | 20 +++++ .../0_stateless/01157_replace_table.sql | 48 ++++++++++++ tests/queries/skip_list.json | 1 + 11 files changed, 186 insertions(+), 62 deletions(-) create mode 100644 tests/queries/0_stateless/01157_replace_table.reference create mode 100644 tests/queries/0_stateless/01157_replace_table.sql diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index de858bdbdc5..ccd92fb650c 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -31,6 +32,7 @@ #include #include +#include #include #include #include @@ -796,36 +798,6 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data create.uuid = UUIDHelpers::Nil; create.to_inner_uuid = UUIDHelpers::Nil; } - - if (create.replace_table) - { - if (database->getUUID() == UUIDHelpers::Nil) - throw Exception(ErrorCodes::INCORRECT_QUERY, - "{} query is supported only for Atomic databases", - create.create_or_replace ? "CREATE OR REPLACE TABLE" : "REPLACE TABLE"); - - UUID uuid_of_table_to_replace; - if (create.create_or_replace) - { - uuid_of_table_to_replace = getContext()->tryResolveStorageID(StorageID(create.database, create.table)).uuid; - if (uuid_of_table_to_replace == UUIDHelpers::Nil) - { - /// Convert to usual CREATE - create.replace_table = false; - assert(!database->isTableExist(create.table, getContext())); - } - else - create.table = "_tmp_replace_" + toString(uuid_of_table_to_replace); - } - else - { - uuid_of_table_to_replace = getContext()->resolveStorageID(StorageID(create.database, create.table)).uuid; - if (uuid_of_table_to_replace == UUIDHelpers::Nil) - throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", - backQuoteIfNeed(create.database), backQuoteIfNeed(create.table)); - create.table = "_tmp_replace_" + toString(uuid_of_table_to_replace); - } - } } @@ -1105,11 +1077,27 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, { auto ast_drop = std::make_shared(); String table_to_replace_name = create.table; - bool created = false; - bool replaced = false; + { + auto database = DatabaseCatalog::instance().getDatabase(create.database); + if (database->getUUID() == UUIDHelpers::Nil) + throw Exception(ErrorCodes::INCORRECT_QUERY, + "{} query is supported only for Atomic databases", + create.create_or_replace ? "CREATE OR REPLACE TABLE" : "REPLACE TABLE"); + + + UInt64 name_hash = sipHash64(create.database + create.table); + UInt16 random_suffix = thread_local_rng(); + create.table = fmt::format("_tmp_replace_{}_{}", + getHexUIntLowercase(name_hash), + getHexUIntLowercase(random_suffix)); + } + + bool created = false; + bool renamed = false; try { + /// Create temporary table (random name will be generated) [[maybe_unused]] bool done = doCreateTable(create, properties); assert(done); ast_drop->table = create.table; @@ -1117,9 +1105,12 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, ast_drop->database = create.database; ast_drop->kind = ASTDropQuery::Drop; created = true; - if (!create.replace_table) - return fillTableIfNeeded(create); + /// Try fill temporary table + BlockIO fill_io = fillTableIfNeeded(create); + executeTrivialBlockIO(fill_io, getContext()); + + /// Replace target table with created one auto ast_rename = std::make_shared(); ASTRenameQuery::Element elem { @@ -1130,19 +1121,27 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, ast_rename->elements.push_back(std::move(elem)); ast_rename->exchange = true; ast_rename->dictionary = create.is_dictionary; + /// Will execute ordinary RENAME instead of EXCHANGE if the target table does not exist + ast_rename->rename_if_cannot_exchange = create.create_or_replace; - InterpreterRenameQuery(ast_rename, getContext()).execute(); - replaced = true; + InterpreterRenameQuery interpreter_rename{ast_rename, getContext()}; + interpreter_rename.execute(); + renamed = true; - InterpreterDropQuery(ast_drop, getContext()).execute(); + if (!interpreter_rename.renamedInsteadOfExchange()) + { + /// Target table was replaced with new one, drop old table + InterpreterDropQuery(ast_drop, getContext()).execute(); + } create.table = table_to_replace_name; - return fillTableIfNeeded(create); + return {}; } catch (...) { - if (created && create.replace_table && !replaced) + /// Drop temporary table if it was successfully created, but was not renamed to target name + if (created && !renamed) InterpreterDropQuery(ast_drop, getContext()).execute(); throw; } diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp index 515559ad903..989cdad93d5 100644 --- a/src/Interpreters/InterpreterRenameQuery.cpp +++ b/src/Interpreters/InterpreterRenameQuery.cpp @@ -76,8 +76,20 @@ BlockIO InterpreterRenameQuery::executeToTables(const ASTRenameQuery & rename, c for (const auto & elem : descriptions) { - if (!rename.exchange) + bool exchange_tables = rename.exchange; + if (exchange_tables) + { + bool allow_rename_instead_of_exchange = descriptions.size() == 1 && rename.rename_if_cannot_exchange; + if (allow_rename_instead_of_exchange && !database_catalog.isTableExist(StorageID(elem.to_database_name, elem.to_table_name), getContext())) + { + exchange_tables = false; + renamed_instead_of_exchange = true; + } + } + else + { database_catalog.assertTableDoesntExist(StorageID(elem.to_database_name, elem.to_table_name), getContext()); + } DatabasePtr database = database_catalog.getDatabase(elem.from_database_name); if (typeid_cast(database.get()) @@ -100,7 +112,7 @@ BlockIO InterpreterRenameQuery::executeToTables(const ASTRenameQuery & rename, c elem.from_table_name, *database_catalog.getDatabase(elem.to_database_name), elem.to_table_name, - rename.exchange, + exchange_tables, rename.dictionary); } } diff --git a/src/Interpreters/InterpreterRenameQuery.h b/src/Interpreters/InterpreterRenameQuery.h index 49fdd50f52d..dfcd741754e 100644 --- a/src/Interpreters/InterpreterRenameQuery.h +++ b/src/Interpreters/InterpreterRenameQuery.h @@ -55,6 +55,8 @@ public: BlockIO execute() override; void extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr & ast, ContextPtr) const override; + bool renamedInsteadOfExchange() const { return renamed_instead_of_exchange; } + private: BlockIO executeToTables(const ASTRenameQuery & rename, const RenameDescriptions & descriptions, TableGuards & ddl_guards); static BlockIO executeToDatabase(const ASTRenameQuery & rename, const RenameDescriptions & descriptions); @@ -62,6 +64,7 @@ private: AccessRightsElements getRequiredAccess() const; ASTPtr query_ptr; + bool renamed_instead_of_exchange{false}; }; } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index c69a5bcd3e1..8f9a8b8a3e9 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -1060,26 +1060,30 @@ void executeQuery( } else if (pipeline.initialized()) { - const ASTQueryWithOutput * ast_query_with_output = dynamic_cast(ast.get()); - - WriteBuffer * out_buf = &ostr; - std::optional out_file_buf; - if (ast_query_with_output && ast_query_with_output->out_file) + if (pipeline.isCompleted()) { - if (!allow_into_outfile) - throw Exception("INTO OUTFILE is not allowed", ErrorCodes::INTO_OUTFILE_NOT_ALLOWED); - - const auto & out_file = typeid_cast(*ast_query_with_output->out_file).value.safeGet(); - out_file_buf.emplace(out_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_EXCL | O_CREAT); - out_buf = &*out_file_buf; + pipeline.setProgressCallback(context->getProgressCallback()); } - - String format_name = ast_query_with_output && (ast_query_with_output->format != nullptr) - ? getIdentifierName(ast_query_with_output->format) - : context->getDefaultFormat(); - - if (!pipeline.isCompleted()) + else { + const ASTQueryWithOutput * ast_query_with_output = dynamic_cast(ast.get()); + + WriteBuffer * out_buf = &ostr; + std::optional out_file_buf; + if (ast_query_with_output && ast_query_with_output->out_file) + { + if (!allow_into_outfile) + throw Exception("INTO OUTFILE is not allowed", ErrorCodes::INTO_OUTFILE_NOT_ALLOWED); + + const auto & out_file = typeid_cast(*ast_query_with_output->out_file).value.safeGet(); + out_file_buf.emplace(out_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_EXCL | O_CREAT); + out_buf = &*out_file_buf; + } + + String format_name = ast_query_with_output && (ast_query_with_output->format != nullptr) + ? getIdentifierName(ast_query_with_output->format) + : context->getDefaultFormat(); + pipeline.addSimpleTransform([](const Block & header) { return std::make_shared(header); @@ -1105,10 +1109,6 @@ void executeQuery( pipeline.setOutputFormat(std::move(out)); } - else - { - pipeline.setProgressCallback(context->getProgressCallback()); - } { auto executor = pipeline.execute(); @@ -1125,4 +1125,32 @@ void executeQuery( streams.onFinish(); } +void executeTrivialBlockIO(BlockIO & streams, ContextPtr context) +{ + try + { + if (streams.out) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Query stream requires input, but no input buffer provided, it's a bug"); + if (streams.in) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Query stream requires output, but no output buffer provided, it's a bug"); + + if (!streams.pipeline.initialized()) + return; + + if (!streams.pipeline.isCompleted()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Query pipeline requires output, but no output buffer provided, it's a bug"); + + streams.pipeline.setProgressCallback(context->getProgressCallback()); + auto executor = streams.pipeline.execute(); + executor->execute(streams.pipeline.getNumThreads()); + } + catch (...) + { + streams.onException(); + throw; + } + + streams.onFinish(); +} + } diff --git a/src/Interpreters/executeQuery.h b/src/Interpreters/executeQuery.h index 6448b26a652..5eebb09c36b 100644 --- a/src/Interpreters/executeQuery.h +++ b/src/Interpreters/executeQuery.h @@ -53,4 +53,8 @@ BlockIO executeQuery( bool allow_processors /// If can use processors pipeline ); +/// Executes BlockIO returned from executeQuery(...) +/// if built pipeline does not require any input and does not produce any output. +void executeTrivialBlockIO(BlockIO & streams, ContextPtr context); + } diff --git a/src/Parsers/ASTRenameQuery.h b/src/Parsers/ASTRenameQuery.h index 611f81dc9e9..e0c58e3462e 100644 --- a/src/Parsers/ASTRenameQuery.h +++ b/src/Parsers/ASTRenameQuery.h @@ -34,6 +34,9 @@ public: bool database{false}; /// For RENAME DATABASE bool dictionary{false}; /// For RENAME DICTIONARY + /// Special flag for CREATE OR REPLACE. Do not throw if the second table does not exist. + bool rename_if_cannot_exchange{false}; + /** Get the text that identifies this element. */ String getID(char) const override { return "Rename"; } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index f4d6ec5c6f7..f30f2fb8e09 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1137,6 +1137,11 @@ ActionLock StorageDistributed::getActionLock(StorageActionBlockType type) return {}; } +void StorageDistributed::flush() +{ + flushClusterNodesAllData(getContext()); +} + void StorageDistributed::flushClusterNodesAllData(ContextPtr local_context) { /// Sync SYSTEM FLUSH DISTRIBUTED with TRUNCATE diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index c63abbc6aa4..1a74b48a615 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -98,6 +98,7 @@ public: void startup() override; void shutdown() override; + void flush() override; void drop() override; bool storesDataOnDisk() const override { return true; } diff --git a/tests/queries/0_stateless/01157_replace_table.reference b/tests/queries/0_stateless/01157_replace_table.reference new file mode 100644 index 00000000000..9fddaf99847 --- /dev/null +++ b/tests/queries/0_stateless/01157_replace_table.reference @@ -0,0 +1,20 @@ +test flush on replace +1 s1 +2 s2 +3 s3 +exception on create and fill +0 +1 1 s1 +2 2 s2 +3 3 s3 +1 1 s1 +2 2 s2 +3 3 s3 +1 1 s1 +2 2 s2 +3 3 s3 +4 4 s4 +buf +dist +join +t diff --git a/tests/queries/0_stateless/01157_replace_table.sql b/tests/queries/0_stateless/01157_replace_table.sql new file mode 100644 index 00000000000..337290bb4d2 --- /dev/null +++ b/tests/queries/0_stateless/01157_replace_table.sql @@ -0,0 +1,48 @@ +drop table if exists t; +drop table if exists dist; +drop table if exists buf; +drop table if exists join; + +select 'test flush on replace'; +create table t (n UInt64, s String default 's' || toString(n)) engine=Memory; +create table dist (n int) engine=Distributed(test_shard_localhost, currentDatabase(), t); +create table buf (n int) engine=Buffer(currentDatabase(), dist, 1, 10, 100, 10, 100, 1000, 1000); + +insert into buf values (1); +replace table buf (n int) engine=Distributed(test_shard_localhost, currentDatabase(), dist); +replace table dist (n int) engine=Buffer(currentDatabase(), t, 1, 10, 100, 10, 100, 1000, 1000); + +insert into buf values (2); +replace table buf (n int) engine=Buffer(currentDatabase(), dist, 1, 10, 100, 10, 100, 1000, 1000); +replace table dist (n int) engine=Distributed(test_shard_localhost, currentDatabase(), t); + +insert into buf values (3); +replace table buf (n int) engine=Null; +replace table dist (n int) engine=Null; + +select * from t order by n; + +select 'exception on create and fill'; +-- table is not created if select fails +create or replace table join engine=Join(ANY, INNER, n) as select * from t where throwIf(n); -- { serverError 395 } +select count() from system.tables where database=currentDatabase() and name='join'; + +-- table is created and filled +create or replace table join engine=Join(ANY, INNER, n) as select * from t; +select * from numbers(10) as t any join join on t.number=join.n order by n; + +-- table is not replaced if select fails +insert into t(n) values (4); +replace table join engine=Join(ANY, INNER, n) as select * from t where throwIf(n); -- { serverError 395 } +select * from numbers(10) as t any join join on t.number=join.n order by n; + +-- table is replaced +replace table join engine=Join(ANY, INNER, n) as select * from t; +select * from numbers(10) as t any join join on t.number=join.n order by n; + +select name from system.tables where database=currentDatabase() order by name; + +drop table t; +drop table dist; +drop table buf; +drop table join; diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 7c1f998e91d..c9d517b0285 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -109,6 +109,7 @@ "00510_materizlized_view_and_deduplication_zookeeper", "00738_lock_for_inner_table", "01153_attach_mv_uuid", + "01157_replace_table", /// Sometimes cannot lock file most likely due to concurrent or adjacent tests, but we don't care how it works in Ordinary database. "rocksdb", "01914_exchange_dictionaries" /// Requires Atomic database From 1b2416007e383270eccd65b789903f28549e2287 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 1 Jul 2021 19:43:59 +0300 Subject: [PATCH 014/198] fix --- src/Interpreters/executeQuery.cpp | 43 +++++++++++++++-------------- src/Storages/StorageDistributed.cpp | 9 +++++- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 8f9a8b8a3e9..ae52b5992b2 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -77,6 +77,7 @@ namespace ErrorCodes { extern const int INTO_OUTFILE_NOT_ALLOWED; extern const int QUERY_WAS_CANCELLED; + extern const int LOGICAL_ERROR; } @@ -1060,30 +1061,26 @@ void executeQuery( } else if (pipeline.initialized()) { - if (pipeline.isCompleted()) + const ASTQueryWithOutput * ast_query_with_output = dynamic_cast(ast.get()); + + WriteBuffer * out_buf = &ostr; + std::optional out_file_buf; + if (ast_query_with_output && ast_query_with_output->out_file) { - pipeline.setProgressCallback(context->getProgressCallback()); + if (!allow_into_outfile) + throw Exception("INTO OUTFILE is not allowed", ErrorCodes::INTO_OUTFILE_NOT_ALLOWED); + + const auto & out_file = typeid_cast(*ast_query_with_output->out_file).value.safeGet(); + out_file_buf.emplace(out_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_EXCL | O_CREAT); + out_buf = &*out_file_buf; } - else + + String format_name = ast_query_with_output && (ast_query_with_output->format != nullptr) + ? getIdentifierName(ast_query_with_output->format) + : context->getDefaultFormat(); + + if (!pipeline.isCompleted()) { - const ASTQueryWithOutput * ast_query_with_output = dynamic_cast(ast.get()); - - WriteBuffer * out_buf = &ostr; - std::optional out_file_buf; - if (ast_query_with_output && ast_query_with_output->out_file) - { - if (!allow_into_outfile) - throw Exception("INTO OUTFILE is not allowed", ErrorCodes::INTO_OUTFILE_NOT_ALLOWED); - - const auto & out_file = typeid_cast(*ast_query_with_output->out_file).value.safeGet(); - out_file_buf.emplace(out_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_EXCL | O_CREAT); - out_buf = &*out_file_buf; - } - - String format_name = ast_query_with_output && (ast_query_with_output->format != nullptr) - ? getIdentifierName(ast_query_with_output->format) - : context->getDefaultFormat(); - pipeline.addSimpleTransform([](const Block & header) { return std::make_shared(header); @@ -1109,6 +1106,10 @@ void executeQuery( pipeline.setOutputFormat(std::move(out)); } + else + { + pipeline.setProgressCallback(context->getProgressCallback()); + } { auto executor = pipeline.execute(); diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index f30f2fb8e09..740b2cfbfe9 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1139,7 +1139,14 @@ ActionLock StorageDistributed::getActionLock(StorageActionBlockType type) void StorageDistributed::flush() { - flushClusterNodesAllData(getContext()); + try + { + flushClusterNodesAllData(getContext()); + } + catch (...) + { + tryLogCurrentException(log, "Cannot flush"); + } } void StorageDistributed::flushClusterNodesAllData(ContextPtr local_context) From aa3f0b2032fb22d12fe7c909220a52ed0429f0f5 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 2 Jul 2021 19:39:55 +0300 Subject: [PATCH 015/198] fix --- src/Interpreters/Context.cpp | 7 + src/Interpreters/Context.h | 2 + src/Interpreters/DDLTask.cpp | 4 +- src/Interpreters/DDLTask.h | 13 +- src/Interpreters/InterpreterCreateQuery.cpp | 46 +- src/Parsers/New/AST/CreateDictionaryQuery.cpp | 12 +- src/Parsers/New/AST/CreateDictionaryQuery.h | 7 +- src/Parsers/New/AST/CreateTableQuery.cpp | 17 +- src/Parsers/New/AST/CreateTableQuery.h | 8 +- src/Parsers/New/ClickHouseParser.cpp | 4619 +++++++++-------- src/Parsers/New/ClickHouseParser.g4 | 4 +- src/Parsers/New/ClickHouseParser.h | 4 + .../0_stateless/01157_replace_table.sql | 3 + .../01185_create_or_replace_table.reference | 6 +- .../01185_create_or_replace_table.sql | 37 +- .../01915_create_or_replace_dictionary.sql | 38 +- tests/queries/skip_list.json | 2 +- 17 files changed, 2496 insertions(+), 2333 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index a0fce58b472..9b51fc5933d 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -2677,6 +2677,13 @@ ZooKeeperMetadataTransactionPtr Context::getZooKeeperMetadataTransaction() const return metadata_transaction; } +void Context::resetZooKeeperMetadataTransaction() +{ + assert(metadata_transaction); + assert(hasQueryContext()); + metadata_transaction = nullptr; +} + PartUUIDsPtr Context::getPartUUIDs() const { auto lock = getLock(); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 7990bd7420b..3556783b695 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -782,6 +782,8 @@ public: void initZooKeeperMetadataTransaction(ZooKeeperMetadataTransactionPtr txn, bool attach_existing = false); /// Returns context of current distributed DDL query or nullptr. ZooKeeperMetadataTransactionPtr getZooKeeperMetadataTransaction() const; + /// Removes context of current distributed DDL. + void resetZooKeeperMetadataTransaction(); struct MySQLWireContext { diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index 4fb44738d8d..bf8380f5af6 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -22,6 +22,7 @@ namespace ErrorCodes extern const int UNKNOWN_FORMAT_VERSION; extern const int UNKNOWN_TYPE_OF_QUERY; extern const int INCONSISTENT_CLUSTER_DEFINITION; + extern const int LOGICAL_ERROR; } HostID HostID::fromString(const String & host_port_str) @@ -401,7 +402,8 @@ UInt32 DDLTaskBase::getLogEntryNumber(const String & log_entry_name) void ZooKeeperMetadataTransaction::commit() { - assert(state == CREATED); + if (state != CREATED) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Incorrect state ({}), it's a bug", state); state = FAILED; current_zookeeper->multi(ops); state = COMMITTED; diff --git a/src/Interpreters/DDLTask.h b/src/Interpreters/DDLTask.h index 703d691a358..00bb740f9c5 100644 --- a/src/Interpreters/DDLTask.h +++ b/src/Interpreters/DDLTask.h @@ -20,6 +20,11 @@ namespace fs = std::filesystem; namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + class ASTQueryWithOnCluster; using ZooKeeperPtr = std::shared_ptr; using ClusterPtr = std::shared_ptr; @@ -180,15 +185,19 @@ public: String getDatabaseZooKeeperPath() const { return zookeeper_path; } + ZooKeeperPtr getZooKeeper() const { return current_zookeeper; } + void addOp(Coordination::RequestPtr && op) { - assert(!isExecuted()); + if (isExecuted()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot add ZooKeeper operation because query is executed. It's a bug."); ops.emplace_back(op); } void moveOpsTo(Coordination::Requests & other_ops) { - assert(!isExecuted()); + if (isExecuted()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot add ZooKeeper operation because query is executed. It's a bug."); std::move(ops.begin(), ops.end(), std::back_inserter(other_ops)); ops.clear(); state = COMMITTED; diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index ccd92fb650c..c7ed4214e38 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -86,7 +87,6 @@ namespace ErrorCodes extern const int UNKNOWN_DATABASE; extern const int PATH_ACCESS_DENIED; extern const int NOT_IMPLEMENTED; - extern const int UNKNOWN_TABLE; } namespace fs = std::filesystem; @@ -1075,6 +1075,29 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, const InterpreterCreateQuery::TableProperties & properties) { + /// Replicated database requires separate contexts for each DDL query + ContextPtr current_context = getContext(); + ContextMutablePtr create_context = Context::createCopy(current_context); + create_context->setQueryContext(std::const_pointer_cast(current_context)); + + auto make_drop_context = [&](bool on_error) -> ContextMutablePtr + { + ContextMutablePtr drop_context = Context::createCopy(current_context); + drop_context->makeQueryContext(); + if (on_error) + return drop_context; + + if (auto txn = current_context->getZooKeeperMetadataTransaction()) + { + /// Execute drop as separate query, because [CREATE OR] REPLACE query can be considered as + /// successfully executed after RENAME/EXCHANGE query. + drop_context->resetZooKeeperMetadataTransaction(); + auto drop_txn = std::make_shared(txn->getZooKeeper(), txn->getDatabaseZooKeeperPath(), txn->isInitialQuery()); + drop_context->initZooKeeperMetadataTransaction(drop_txn); + } + return drop_context; + }; + auto ast_drop = std::make_shared(); String table_to_replace_name = create.table; @@ -1091,6 +1114,11 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, create.table = fmt::format("_tmp_replace_{}_{}", getHexUIntLowercase(name_hash), getHexUIntLowercase(random_suffix)); + + ast_drop->table = create.table; + ast_drop->is_dictionary = create.is_dictionary; + ast_drop->database = create.database; + ast_drop->kind = ASTDropQuery::Drop; } bool created = false; @@ -1098,12 +1126,8 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, try { /// Create temporary table (random name will be generated) - [[maybe_unused]] bool done = doCreateTable(create, properties); + [[maybe_unused]] bool done = InterpreterCreateQuery(query_ptr, create_context).doCreateTable(create, properties); assert(done); - ast_drop->table = create.table; - ast_drop->is_dictionary = create.is_dictionary; - ast_drop->database = create.database; - ast_drop->kind = ASTDropQuery::Drop; created = true; /// Try fill temporary table @@ -1124,14 +1148,15 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, /// Will execute ordinary RENAME instead of EXCHANGE if the target table does not exist ast_rename->rename_if_cannot_exchange = create.create_or_replace; - InterpreterRenameQuery interpreter_rename{ast_rename, getContext()}; + InterpreterRenameQuery interpreter_rename{ast_rename, current_context}; interpreter_rename.execute(); renamed = true; if (!interpreter_rename.renamedInsteadOfExchange()) { /// Target table was replaced with new one, drop old table - InterpreterDropQuery(ast_drop, getContext()).execute(); + auto drop_context = make_drop_context(false); + InterpreterDropQuery(ast_drop, drop_context).execute(); } create.table = table_to_replace_name; @@ -1142,7 +1167,10 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, { /// Drop temporary table if it was successfully created, but was not renamed to target name if (created && !renamed) - InterpreterDropQuery(ast_drop, getContext()).execute(); + { + auto drop_context = make_drop_context(true); + InterpreterDropQuery(ast_drop, drop_context).execute(); + } throw; } } diff --git a/src/Parsers/New/AST/CreateDictionaryQuery.cpp b/src/Parsers/New/AST/CreateDictionaryQuery.cpp index 75413df495b..dfc996054ee 100644 --- a/src/Parsers/New/AST/CreateDictionaryQuery.cpp +++ b/src/Parsers/New/AST/CreateDictionaryQuery.cpp @@ -226,11 +226,17 @@ CreateDictionaryQuery::CreateDictionaryQuery( PtrTo cluster, bool attach_, bool if_not_exists_, + bool create_or_replace_, + bool replace_, PtrTo identifier, PtrTo uuid, PtrTo schema, PtrTo engine) - : DDLQuery(cluster, {identifier, uuid, schema, engine}), attach(attach_), if_not_exists(if_not_exists_) + : DDLQuery(cluster, {identifier, uuid, schema, engine}) + , attach(attach_) + , if_not_exists(if_not_exists_) + , create_or_replace(create_or_replace_) + , replace(replace_) { } @@ -251,6 +257,8 @@ ASTPtr CreateDictionaryQuery::convertToOld() const query->is_dictionary = true; query->attach = attach; query->if_not_exists = if_not_exists; + query->create_or_replace = create_or_replace; + query->replace_table = replace; query->set(query->dictionary_attributes_list, get(SCHEMA)->convertToOld()); query->set(query->dictionary, get(ENGINE)->convertToOld()); @@ -272,7 +280,7 @@ antlrcpp::Any ParseTreeVisitor::visitCreateDictionaryStmt(ClickHouseParser::Crea auto schema = ctx->dictionarySchemaClause() ? visit(ctx->dictionarySchemaClause()).as>() : nullptr; auto engine = ctx->dictionaryEngineClause() ? visit(ctx->dictionaryEngineClause()).as>() : nullptr; return std::make_shared( - cluster, !!ctx->ATTACH(), !!ctx->IF(), visit(ctx->tableIdentifier()), uuid, schema, engine); + cluster, !!ctx->ATTACH(), !!ctx->IF(), !!ctx->OR(), !!ctx->REPLACE(), visit(ctx->tableIdentifier()), uuid, schema, engine); } antlrcpp::Any ParseTreeVisitor::visitDictionaryArgExpr(ClickHouseParser::DictionaryArgExprContext *ctx) diff --git a/src/Parsers/New/AST/CreateDictionaryQuery.h b/src/Parsers/New/AST/CreateDictionaryQuery.h index 3c5be3f391c..b070c6edd98 100644 --- a/src/Parsers/New/AST/CreateDictionaryQuery.h +++ b/src/Parsers/New/AST/CreateDictionaryQuery.h @@ -161,6 +161,8 @@ class CreateDictionaryQuery : public DDLQuery PtrTo cluster, bool attach, bool if_not_exists, + bool create_or_replace, + bool replace, PtrTo identifier, PtrTo uuid, PtrTo schema, @@ -177,7 +179,10 @@ class CreateDictionaryQuery : public DDLQuery ENGINE, // DictionaryEngineClause }; - const bool attach, if_not_exists; + const bool attach; + const bool if_not_exists; + const bool create_or_replace; + const bool replace; }; } diff --git a/src/Parsers/New/AST/CreateTableQuery.cpp b/src/Parsers/New/AST/CreateTableQuery.cpp index 1767c08451a..b24c34de9d2 100644 --- a/src/Parsers/New/AST/CreateTableQuery.cpp +++ b/src/Parsers/New/AST/CreateTableQuery.cpp @@ -99,12 +99,19 @@ CreateTableQuery::CreateTableQuery( bool attach_, bool temporary_, bool if_not_exists_, + bool create_or_replace_, + bool replace_, PtrTo identifier, PtrTo uuid, PtrTo schema, PtrTo engine, PtrTo query) - : DDLQuery(cluster, {identifier, uuid, schema, engine, query}), attach(attach_), temporary(temporary_), if_not_exists(if_not_exists_) + : DDLQuery(cluster, {identifier, uuid, schema, engine, query}) + , attach(attach_) + , temporary(temporary_) + , if_not_exists(if_not_exists_) + , create_or_replace(create_or_replace_) + , replace(replace_) { } @@ -125,6 +132,8 @@ ASTPtr CreateTableQuery::convertToOld() const query->attach = attach; query->if_not_exists = if_not_exists; query->temporary = temporary; + query->create_or_replace = create_or_replace; + query->replace_table = replace; if (has(SCHEMA)) { @@ -164,6 +173,10 @@ String CreateTableQuery::dumpInfo() const else info += "temporary=false, "; if (if_not_exists) info += "if_not_exists=true"; else info += "if_not_exists=false"; + if (create_or_replace) info += "create_or_replace=true"; + else info += "create_or_replace=false"; + if (replace) info += "replace=true"; + else info += "replace=false"; return info; } @@ -191,7 +204,7 @@ antlrcpp::Any ParseTreeVisitor::visitCreateTableStmt(ClickHouseParser::CreateTab auto engine = ctx->engineClause() ? visit(ctx->engineClause()).as>() : nullptr; auto query = ctx->subqueryClause() ? visit(ctx->subqueryClause()).as>() : nullptr; return std::make_shared( - cluster, !!ctx->ATTACH(), !!ctx->TEMPORARY(), !!ctx->IF(), visit(ctx->tableIdentifier()), uuid, schema, engine, query); + cluster, !!ctx->ATTACH(), !!ctx->TEMPORARY(), !!ctx->IF(), !!ctx->OR(), !!ctx->REPLACE(), visit(ctx->tableIdentifier()), uuid, schema, engine, query); } antlrcpp::Any ParseTreeVisitor::visitSchemaDescriptionClause(ClickHouseParser::SchemaDescriptionClauseContext *ctx) diff --git a/src/Parsers/New/AST/CreateTableQuery.h b/src/Parsers/New/AST/CreateTableQuery.h index 4fe19832b1d..39c43cda902 100644 --- a/src/Parsers/New/AST/CreateTableQuery.h +++ b/src/Parsers/New/AST/CreateTableQuery.h @@ -50,6 +50,8 @@ class CreateTableQuery : public DDLQuery bool attach, bool temporary, bool if_not_exists, + bool create_or_replace, + bool replace, PtrTo identifier, PtrTo uuid, PtrTo schema, @@ -68,7 +70,11 @@ class CreateTableQuery : public DDLQuery SUBQUERY, // SelectUnionQuery }; - const bool attach, temporary, if_not_exists; + const bool attach; + const bool temporary; + const bool if_not_exists; + const bool create_or_replace; + const bool replace; String dumpInfo() const override; }; diff --git a/src/Parsers/New/ClickHouseParser.cpp b/src/Parsers/New/ClickHouseParser.cpp index 174f838f19d..a9405aa0da1 100644 --- a/src/Parsers/New/ClickHouseParser.cpp +++ b/src/Parsers/New/ClickHouseParser.cpp @@ -107,6 +107,7 @@ ClickHouseParser::QueryStmtContext* ClickHouseParser::queryStmt() { case ClickHouseParser::KILL: case ClickHouseParser::OPTIMIZE: case ClickHouseParser::RENAME: + case ClickHouseParser::REPLACE: case ClickHouseParser::SELECT: case ClickHouseParser::SET: case ClickHouseParser::SHOW: @@ -2656,6 +2657,10 @@ tree::TerminalNode* ClickHouseParser::CreateDictionaryStmtContext::CREATE() { return getToken(ClickHouseParser::CREATE, 0); } +tree::TerminalNode* ClickHouseParser::CreateDictionaryStmtContext::REPLACE() { + return getToken(ClickHouseParser::REPLACE, 0); +} + tree::TerminalNode* ClickHouseParser::CreateDictionaryStmtContext::IF() { return getToken(ClickHouseParser::IF, 0); } @@ -2676,6 +2681,10 @@ ClickHouseParser::ClusterClauseContext* ClickHouseParser::CreateDictionaryStmtCo return getRuleContext(0); } +tree::TerminalNode* ClickHouseParser::CreateDictionaryStmtContext::OR() { + return getToken(ClickHouseParser::OR, 0); +} + ClickHouseParser::CreateDictionaryStmtContext::CreateDictionaryStmtContext(CreateStmtContext *ctx) { copyFrom(ctx); } antlrcpp::Any ClickHouseParser::CreateDictionaryStmtContext::accept(tree::ParseTreeVisitor *visitor) { @@ -2892,6 +2901,10 @@ tree::TerminalNode* ClickHouseParser::CreateTableStmtContext::CREATE() { return getToken(ClickHouseParser::CREATE, 0); } +tree::TerminalNode* ClickHouseParser::CreateTableStmtContext::REPLACE() { + return getToken(ClickHouseParser::REPLACE, 0); +} + tree::TerminalNode* ClickHouseParser::CreateTableStmtContext::TEMPORARY() { return getToken(ClickHouseParser::TEMPORARY, 0); } @@ -2928,6 +2941,10 @@ ClickHouseParser::SubqueryClauseContext* ClickHouseParser::CreateTableStmtContex return getRuleContext(0); } +tree::TerminalNode* ClickHouseParser::CreateTableStmtContext::OR() { + return getToken(ClickHouseParser::OR, 0); +} + ClickHouseParser::CreateTableStmtContext::CreateTableStmtContext(CreateStmtContext *ctx) { copyFrom(ctx); } antlrcpp::Any ClickHouseParser::CreateTableStmtContext::accept(tree::ParseTreeVisitor *visitor) { @@ -2945,9 +2962,9 @@ ClickHouseParser::CreateStmtContext* ClickHouseParser::createStmt() { exitRule(); }); try { - setState(654); + setState(670); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 71, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 75, _ctx)) { case 1: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); @@ -3003,55 +3020,78 @@ ClickHouseParser::CreateStmtContext* ClickHouseParser::createStmt() { case 2: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(531); - _la = _input->LA(1); - if (!(_la == ClickHouseParser::ATTACH + setState(538); + _errHandler->sync(this); + switch (_input->LA(1)) { + case ClickHouseParser::ATTACH: { + setState(531); + match(ClickHouseParser::ATTACH); + break; + } - || _la == ClickHouseParser::CREATE)) { - _errHandler->recoverInline(this); + case ClickHouseParser::CREATE: { + setState(532); + match(ClickHouseParser::CREATE); + setState(535); + _errHandler->sync(this); + + _la = _input->LA(1); + if (_la == ClickHouseParser::OR) { + setState(533); + match(ClickHouseParser::OR); + setState(534); + match(ClickHouseParser::REPLACE); + } + break; + } + + case ClickHouseParser::REPLACE: { + setState(537); + match(ClickHouseParser::REPLACE); + break; + } + + default: + throw NoViableAltException(this); } - else { - _errHandler->reportMatch(this); - consume(); - } - setState(532); + setState(540); match(ClickHouseParser::DICTIONARY); - setState(536); + setState(544); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 43, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 45, _ctx)) { case 1: { - setState(533); + setState(541); match(ClickHouseParser::IF); - setState(534); + setState(542); match(ClickHouseParser::NOT); - setState(535); + setState(543); match(ClickHouseParser::EXISTS); break; } } - setState(538); + setState(546); tableIdentifier(); - setState(540); + setState(548); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::UUID) { - setState(539); + setState(547); uuidClause(); } - setState(543); + setState(551); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ON) { - setState(542); + setState(550); clusterClause(); } - setState(545); + setState(553); dictionarySchemaClause(); - setState(546); + setState(554); dictionaryEngineClause(); break; } @@ -3059,7 +3099,7 @@ ClickHouseParser::CreateStmtContext* ClickHouseParser::createStmt() { case 3: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 3); - setState(548); + setState(556); _la = _input->LA(1); if (!(_la == ClickHouseParser::ATTACH @@ -3070,81 +3110,81 @@ ClickHouseParser::CreateStmtContext* ClickHouseParser::createStmt() { _errHandler->reportMatch(this); consume(); } - setState(549); + setState(557); match(ClickHouseParser::LIVE); - setState(550); + setState(558); match(ClickHouseParser::VIEW); - setState(554); + setState(562); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 46, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 48, _ctx)) { case 1: { - setState(551); + setState(559); match(ClickHouseParser::IF); - setState(552); + setState(560); match(ClickHouseParser::NOT); - setState(553); + setState(561); match(ClickHouseParser::EXISTS); break; } } - setState(556); + setState(564); tableIdentifier(); - setState(558); + setState(566); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::UUID) { - setState(557); + setState(565); uuidClause(); } - setState(561); + setState(569); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ON) { - setState(560); + setState(568); clusterClause(); } - setState(568); + setState(576); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::WITH) { - setState(563); + setState(571); match(ClickHouseParser::WITH); - setState(564); + setState(572); match(ClickHouseParser::TIMEOUT); - setState(566); + setState(574); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::DECIMAL_LITERAL) { - setState(565); + setState(573); match(ClickHouseParser::DECIMAL_LITERAL); } } - setState(571); + setState(579); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::TO) { - setState(570); + setState(578); destinationClause(); } - setState(574); + setState(582); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 52, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 54, _ctx)) { case 1: { - setState(573); + setState(581); tableSchemaClause(); break; } } - setState(576); + setState(584); subqueryClause(); break; } @@ -3152,7 +3192,7 @@ ClickHouseParser::CreateStmtContext* ClickHouseParser::createStmt() { case 4: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 4); - setState(578); + setState(586); _la = _input->LA(1); if (!(_la == ClickHouseParser::ATTACH @@ -3163,69 +3203,69 @@ ClickHouseParser::CreateStmtContext* ClickHouseParser::createStmt() { _errHandler->reportMatch(this); consume(); } - setState(579); + setState(587); match(ClickHouseParser::MATERIALIZED); - setState(580); + setState(588); match(ClickHouseParser::VIEW); - setState(584); + setState(592); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 53, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 55, _ctx)) { case 1: { - setState(581); + setState(589); match(ClickHouseParser::IF); - setState(582); + setState(590); match(ClickHouseParser::NOT); - setState(583); + setState(591); match(ClickHouseParser::EXISTS); break; } } - setState(586); + setState(594); tableIdentifier(); - setState(588); + setState(596); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::UUID) { - setState(587); + setState(595); uuidClause(); } - setState(591); + setState(599); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ON) { - setState(590); + setState(598); clusterClause(); } - setState(594); + setState(602); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::AS || _la == ClickHouseParser::LPAREN) { - setState(593); + setState(601); tableSchemaClause(); } - setState(601); + setState(609); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::TO: { - setState(596); + setState(604); destinationClause(); break; } case ClickHouseParser::ENGINE: { - setState(597); + setState(605); engineClause(); - setState(599); + setState(607); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::POPULATE) { - setState(598); + setState(606); match(ClickHouseParser::POPULATE); } break; @@ -3234,7 +3274,7 @@ ClickHouseParser::CreateStmtContext* ClickHouseParser::createStmt() { default: throw NoViableAltException(this); } - setState(603); + setState(611); subqueryClause(); break; } @@ -3242,85 +3282,108 @@ ClickHouseParser::CreateStmtContext* ClickHouseParser::createStmt() { case 5: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 5); - setState(605); - _la = _input->LA(1); - if (!(_la == ClickHouseParser::ATTACH + setState(620); + _errHandler->sync(this); + switch (_input->LA(1)) { + case ClickHouseParser::ATTACH: { + setState(613); + match(ClickHouseParser::ATTACH); + break; + } - || _la == ClickHouseParser::CREATE)) { - _errHandler->recoverInline(this); + case ClickHouseParser::CREATE: { + setState(614); + match(ClickHouseParser::CREATE); + setState(617); + _errHandler->sync(this); + + _la = _input->LA(1); + if (_la == ClickHouseParser::OR) { + setState(615); + match(ClickHouseParser::OR); + setState(616); + match(ClickHouseParser::REPLACE); + } + break; + } + + case ClickHouseParser::REPLACE: { + setState(619); + match(ClickHouseParser::REPLACE); + break; + } + + default: + throw NoViableAltException(this); } - else { - _errHandler->reportMatch(this); - consume(); - } - setState(607); + setState(623); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::TEMPORARY) { - setState(606); + setState(622); match(ClickHouseParser::TEMPORARY); } - setState(609); + setState(625); match(ClickHouseParser::TABLE); - setState(613); + setState(629); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 60, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 64, _ctx)) { case 1: { - setState(610); + setState(626); match(ClickHouseParser::IF); - setState(611); + setState(627); match(ClickHouseParser::NOT); - setState(612); + setState(628); match(ClickHouseParser::EXISTS); break; } } - setState(615); + setState(631); tableIdentifier(); - setState(617); + setState(633); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::UUID) { - setState(616); + setState(632); uuidClause(); } - setState(620); + setState(636); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ON) { - setState(619); + setState(635); clusterClause(); } - setState(623); + setState(639); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 63, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 67, _ctx)) { case 1: { - setState(622); + setState(638); tableSchemaClause(); break; } } - setState(626); + setState(642); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ENGINE) { - setState(625); + setState(641); engineClause(); } - setState(629); + setState(645); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::AS) { - setState(628); + setState(644); subqueryClause(); } break; @@ -3329,7 +3392,7 @@ ClickHouseParser::CreateStmtContext* ClickHouseParser::createStmt() { case 6: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 6); - setState(631); + setState(647); _la = _input->LA(1); if (!(_la == ClickHouseParser::ATTACH @@ -3340,63 +3403,63 @@ ClickHouseParser::CreateStmtContext* ClickHouseParser::createStmt() { _errHandler->reportMatch(this); consume(); } - setState(634); + setState(650); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::OR) { - setState(632); + setState(648); match(ClickHouseParser::OR); - setState(633); + setState(649); match(ClickHouseParser::REPLACE); } - setState(636); + setState(652); match(ClickHouseParser::VIEW); - setState(640); + setState(656); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 67, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 71, _ctx)) { case 1: { - setState(637); + setState(653); match(ClickHouseParser::IF); - setState(638); + setState(654); match(ClickHouseParser::NOT); - setState(639); + setState(655); match(ClickHouseParser::EXISTS); break; } } - setState(642); + setState(658); tableIdentifier(); - setState(644); + setState(660); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::UUID) { - setState(643); + setState(659); uuidClause(); } - setState(647); + setState(663); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ON) { - setState(646); + setState(662); clusterClause(); } - setState(650); + setState(666); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 70, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 74, _ctx)) { case 1: { - setState(649); + setState(665); tableSchemaClause(); break; } } - setState(652); + setState(668); subqueryClause(); break; } @@ -3465,23 +3528,23 @@ ClickHouseParser::DictionarySchemaClauseContext* ClickHouseParser::dictionarySch }); try { enterOuterAlt(_localctx, 1); - setState(656); + setState(672); match(ClickHouseParser::LPAREN); - setState(657); + setState(673); dictionaryAttrDfnt(); - setState(662); + setState(678); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(658); + setState(674); match(ClickHouseParser::COMMA); - setState(659); + setState(675); dictionaryAttrDfnt(); - setState(664); + setState(680); _errHandler->sync(this); _la = _input->LA(1); } - setState(665); + setState(681); match(ClickHouseParser::RPAREN); } @@ -3586,67 +3649,67 @@ ClickHouseParser::DictionaryAttrDfntContext* ClickHouseParser::dictionaryAttrDfn try { size_t alt; enterOuterAlt(_localctx, 1); - setState(667); + setState(683); identifier(); - setState(668); + setState(684); columnTypeExpr(); - setState(690); + setState(706); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 74, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 78, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { - setState(688); + setState(704); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 73, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 77, _ctx)) { case 1: { - setState(669); + setState(685); if (!(!_localctx->attrs.count("default"))) throw FailedPredicateException(this, "!$attrs.count(\"default\")"); - setState(670); + setState(686); match(ClickHouseParser::DEFAULT); - setState(671); + setState(687); literal(); _localctx->attrs.insert("default"); break; } case 2: { - setState(674); + setState(690); if (!(!_localctx->attrs.count("expression"))) throw FailedPredicateException(this, "!$attrs.count(\"expression\")"); - setState(675); + setState(691); match(ClickHouseParser::EXPRESSION); - setState(676); + setState(692); columnExpr(0); _localctx->attrs.insert("expression"); break; } case 3: { - setState(679); + setState(695); if (!(!_localctx->attrs.count("hierarchical"))) throw FailedPredicateException(this, "!$attrs.count(\"hierarchical\")"); - setState(680); + setState(696); match(ClickHouseParser::HIERARCHICAL); _localctx->attrs.insert("hierarchical"); break; } case 4: { - setState(682); + setState(698); if (!(!_localctx->attrs.count("injective"))) throw FailedPredicateException(this, "!$attrs.count(\"injective\")"); - setState(683); + setState(699); match(ClickHouseParser::INJECTIVE); _localctx->attrs.insert("injective"); break; } case 5: { - setState(685); + setState(701); if (!(!_localctx->attrs.count("is_object_id"))) throw FailedPredicateException(this, "!$attrs.count(\"is_object_id\")"); - setState(686); + setState(702); match(ClickHouseParser::IS_OBJECT_ID); _localctx->attrs.insert("is_object_id"); break; @@ -3654,9 +3717,9 @@ ClickHouseParser::DictionaryAttrDfntContext* ClickHouseParser::dictionaryAttrDfn } } - setState(692); + setState(708); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 74, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 78, _ctx); } } @@ -3741,70 +3804,70 @@ ClickHouseParser::DictionaryEngineClauseContext* ClickHouseParser::dictionaryEng try { size_t alt; enterOuterAlt(_localctx, 1); - setState(694); + setState(710); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 75, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 79, _ctx)) { case 1: { - setState(693); + setState(709); dictionaryPrimaryKeyClause(); break; } } - setState(718); + setState(734); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 77, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 81, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { - setState(716); + setState(732); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 76, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 80, _ctx)) { case 1: { - setState(696); + setState(712); if (!(!_localctx->clauses.count("source"))) throw FailedPredicateException(this, "!$clauses.count(\"source\")"); - setState(697); + setState(713); sourceClause(); _localctx->clauses.insert("source"); break; } case 2: { - setState(700); + setState(716); if (!(!_localctx->clauses.count("lifetime"))) throw FailedPredicateException(this, "!$clauses.count(\"lifetime\")"); - setState(701); + setState(717); lifetimeClause(); _localctx->clauses.insert("lifetime"); break; } case 3: { - setState(704); + setState(720); if (!(!_localctx->clauses.count("layout"))) throw FailedPredicateException(this, "!$clauses.count(\"layout\")"); - setState(705); + setState(721); layoutClause(); _localctx->clauses.insert("layout"); break; } case 4: { - setState(708); + setState(724); if (!(!_localctx->clauses.count("range"))) throw FailedPredicateException(this, "!$clauses.count(\"range\")"); - setState(709); + setState(725); rangeClause(); _localctx->clauses.insert("range"); break; } case 5: { - setState(712); + setState(728); if (!(!_localctx->clauses.count("settings"))) throw FailedPredicateException(this, "!$clauses.count(\"settings\")"); - setState(713); + setState(729); dictionarySettingsClause(); _localctx->clauses.insert("settings"); break; @@ -3812,9 +3875,9 @@ ClickHouseParser::DictionaryEngineClauseContext* ClickHouseParser::dictionaryEng } } - setState(720); + setState(736); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 77, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 81, _ctx); } } @@ -3866,11 +3929,11 @@ ClickHouseParser::DictionaryPrimaryKeyClauseContext* ClickHouseParser::dictionar }); try { enterOuterAlt(_localctx, 1); - setState(721); + setState(737); match(ClickHouseParser::PRIMARY); - setState(722); + setState(738); match(ClickHouseParser::KEY); - setState(723); + setState(739); columnExprList(); } @@ -3931,9 +3994,9 @@ ClickHouseParser::DictionaryArgExprContext* ClickHouseParser::dictionaryArgExpr( }); try { enterOuterAlt(_localctx, 1); - setState(725); + setState(741); identifier(); - setState(732); + setState(748); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::AFTER: @@ -4115,16 +4178,16 @@ ClickHouseParser::DictionaryArgExprContext* ClickHouseParser::dictionaryArgExpr( case ClickHouseParser::JSON_FALSE: case ClickHouseParser::JSON_TRUE: case ClickHouseParser::IDENTIFIER: { - setState(726); + setState(742); identifier(); - setState(729); + setState(745); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::LPAREN) { - setState(727); + setState(743); match(ClickHouseParser::LPAREN); - setState(728); + setState(744); match(ClickHouseParser::RPAREN); } break; @@ -4141,7 +4204,7 @@ ClickHouseParser::DictionaryArgExprContext* ClickHouseParser::dictionaryArgExpr( case ClickHouseParser::DASH: case ClickHouseParser::DOT: case ClickHouseParser::PLUS: { - setState(731); + setState(747); literal(); break; } @@ -4220,15 +4283,15 @@ ClickHouseParser::SourceClauseContext* ClickHouseParser::sourceClause() { }); try { enterOuterAlt(_localctx, 1); - setState(734); + setState(750); match(ClickHouseParser::SOURCE); - setState(735); + setState(751); match(ClickHouseParser::LPAREN); - setState(736); + setState(752); identifier(); - setState(737); + setState(753); match(ClickHouseParser::LPAREN); - setState(741); + setState(757); _errHandler->sync(this); _la = _input->LA(1); while ((((_la & ~ 0x3fULL) == 0) && @@ -4411,15 +4474,15 @@ ClickHouseParser::SourceClauseContext* ClickHouseParser::sourceClause() { | (1ULL << (ClickHouseParser::JSON_FALSE - 128)) | (1ULL << (ClickHouseParser::JSON_TRUE - 128)) | (1ULL << (ClickHouseParser::IDENTIFIER - 128)))) != 0)) { - setState(738); + setState(754); dictionaryArgExpr(); - setState(743); + setState(759); _errHandler->sync(this); _la = _input->LA(1); } - setState(744); + setState(760); match(ClickHouseParser::RPAREN); - setState(745); + setState(761); match(ClickHouseParser::RPAREN); } @@ -4487,39 +4550,39 @@ ClickHouseParser::LifetimeClauseContext* ClickHouseParser::lifetimeClause() { }); try { enterOuterAlt(_localctx, 1); - setState(747); + setState(763); match(ClickHouseParser::LIFETIME); - setState(748); + setState(764); match(ClickHouseParser::LPAREN); - setState(758); + setState(774); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::DECIMAL_LITERAL: { - setState(749); + setState(765); match(ClickHouseParser::DECIMAL_LITERAL); break; } case ClickHouseParser::MIN: { - setState(750); + setState(766); match(ClickHouseParser::MIN); - setState(751); + setState(767); match(ClickHouseParser::DECIMAL_LITERAL); - setState(752); + setState(768); match(ClickHouseParser::MAX); - setState(753); + setState(769); match(ClickHouseParser::DECIMAL_LITERAL); break; } case ClickHouseParser::MAX: { - setState(754); + setState(770); match(ClickHouseParser::MAX); - setState(755); + setState(771); match(ClickHouseParser::DECIMAL_LITERAL); - setState(756); + setState(772); match(ClickHouseParser::MIN); - setState(757); + setState(773); match(ClickHouseParser::DECIMAL_LITERAL); break; } @@ -4527,7 +4590,7 @@ ClickHouseParser::LifetimeClauseContext* ClickHouseParser::lifetimeClause() { default: throw NoViableAltException(this); } - setState(760); + setState(776); match(ClickHouseParser::RPAREN); } @@ -4600,15 +4663,15 @@ ClickHouseParser::LayoutClauseContext* ClickHouseParser::layoutClause() { }); try { enterOuterAlt(_localctx, 1); - setState(762); + setState(778); match(ClickHouseParser::LAYOUT); - setState(763); + setState(779); match(ClickHouseParser::LPAREN); - setState(764); + setState(780); identifier(); - setState(765); + setState(781); match(ClickHouseParser::LPAREN); - setState(769); + setState(785); _errHandler->sync(this); _la = _input->LA(1); while ((((_la & ~ 0x3fULL) == 0) && @@ -4791,15 +4854,15 @@ ClickHouseParser::LayoutClauseContext* ClickHouseParser::layoutClause() { | (1ULL << (ClickHouseParser::JSON_FALSE - 128)) | (1ULL << (ClickHouseParser::JSON_TRUE - 128)) | (1ULL << (ClickHouseParser::IDENTIFIER - 128)))) != 0)) { - setState(766); + setState(782); dictionaryArgExpr(); - setState(771); + setState(787); _errHandler->sync(this); _la = _input->LA(1); } - setState(772); + setState(788); match(ClickHouseParser::RPAREN); - setState(773); + setState(789); match(ClickHouseParser::RPAREN); } @@ -4867,33 +4930,33 @@ ClickHouseParser::RangeClauseContext* ClickHouseParser::rangeClause() { }); try { enterOuterAlt(_localctx, 1); - setState(775); + setState(791); match(ClickHouseParser::RANGE); - setState(776); + setState(792); match(ClickHouseParser::LPAREN); - setState(787); + setState(803); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::MIN: { - setState(777); + setState(793); match(ClickHouseParser::MIN); - setState(778); + setState(794); identifier(); - setState(779); + setState(795); match(ClickHouseParser::MAX); - setState(780); + setState(796); identifier(); break; } case ClickHouseParser::MAX: { - setState(782); + setState(798); match(ClickHouseParser::MAX); - setState(783); + setState(799); identifier(); - setState(784); + setState(800); match(ClickHouseParser::MIN); - setState(785); + setState(801); identifier(); break; } @@ -4901,7 +4964,7 @@ ClickHouseParser::RangeClauseContext* ClickHouseParser::rangeClause() { default: throw NoViableAltException(this); } - setState(789); + setState(805); match(ClickHouseParser::RPAREN); } @@ -4957,13 +5020,13 @@ ClickHouseParser::DictionarySettingsClauseContext* ClickHouseParser::dictionaryS }); try { enterOuterAlt(_localctx, 1); - setState(791); + setState(807); match(ClickHouseParser::SETTINGS); - setState(792); + setState(808); match(ClickHouseParser::LPAREN); - setState(793); + setState(809); settingExprList(); - setState(794); + setState(810); match(ClickHouseParser::RPAREN); } @@ -5019,11 +5082,11 @@ ClickHouseParser::ClusterClauseContext* ClickHouseParser::clusterClause() { }); try { enterOuterAlt(_localctx, 1); - setState(796); + setState(812); match(ClickHouseParser::ON); - setState(797); + setState(813); match(ClickHouseParser::CLUSTER); - setState(800); + setState(816); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::AFTER: @@ -5205,13 +5268,13 @@ ClickHouseParser::ClusterClauseContext* ClickHouseParser::clusterClause() { case ClickHouseParser::JSON_FALSE: case ClickHouseParser::JSON_TRUE: case ClickHouseParser::IDENTIFIER: { - setState(798); + setState(814); identifier(); break; } case ClickHouseParser::STRING_LITERAL: { - setState(799); + setState(815); match(ClickHouseParser::STRING_LITERAL); break; } @@ -5265,9 +5328,9 @@ ClickHouseParser::UuidClauseContext* ClickHouseParser::uuidClause() { }); try { enterOuterAlt(_localctx, 1); - setState(802); + setState(818); match(ClickHouseParser::UUID); - setState(803); + setState(819); match(ClickHouseParser::STRING_LITERAL); } @@ -5315,9 +5378,9 @@ ClickHouseParser::DestinationClauseContext* ClickHouseParser::destinationClause( }); try { enterOuterAlt(_localctx, 1); - setState(805); + setState(821); match(ClickHouseParser::TO); - setState(806); + setState(822); tableIdentifier(); } @@ -5365,9 +5428,9 @@ ClickHouseParser::SubqueryClauseContext* ClickHouseParser::subqueryClause() { }); try { enterOuterAlt(_localctx, 1); - setState(808); + setState(824); match(ClickHouseParser::AS); - setState(809); + setState(825); selectUnionStmt(); } @@ -5474,29 +5537,29 @@ ClickHouseParser::TableSchemaClauseContext* ClickHouseParser::tableSchemaClause( exitRule(); }); try { - setState(826); + setState(842); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 86, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 90, _ctx)) { case 1: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(811); + setState(827); match(ClickHouseParser::LPAREN); - setState(812); + setState(828); tableElementExpr(); - setState(817); + setState(833); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(813); + setState(829); match(ClickHouseParser::COMMA); - setState(814); + setState(830); tableElementExpr(); - setState(819); + setState(835); _errHandler->sync(this); _la = _input->LA(1); } - setState(820); + setState(836); match(ClickHouseParser::RPAREN); break; } @@ -5504,9 +5567,9 @@ ClickHouseParser::TableSchemaClauseContext* ClickHouseParser::tableSchemaClause( case 2: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(822); + setState(838); match(ClickHouseParser::AS); - setState(823); + setState(839); tableIdentifier(); break; } @@ -5514,9 +5577,9 @@ ClickHouseParser::TableSchemaClauseContext* ClickHouseParser::tableSchemaClause( case 3: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 3); - setState(824); + setState(840); match(ClickHouseParser::AS); - setState(825); + setState(841); tableFunctionExpr(); break; } @@ -5613,71 +5676,71 @@ ClickHouseParser::EngineClauseContext* ClickHouseParser::engineClause() { try { size_t alt; enterOuterAlt(_localctx, 1); - setState(828); + setState(844); engineExpr(); - setState(855); + setState(871); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 88, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 92, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { - setState(853); + setState(869); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 87, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 91, _ctx)) { case 1: { - setState(829); + setState(845); if (!(!_localctx->clauses.count("orderByClause"))) throw FailedPredicateException(this, "!$clauses.count(\"orderByClause\")"); - setState(830); + setState(846); orderByClause(); _localctx->clauses.insert("orderByClause"); break; } case 2: { - setState(833); + setState(849); if (!(!_localctx->clauses.count("partitionByClause"))) throw FailedPredicateException(this, "!$clauses.count(\"partitionByClause\")"); - setState(834); + setState(850); partitionByClause(); _localctx->clauses.insert("partitionByClause"); break; } case 3: { - setState(837); + setState(853); if (!(!_localctx->clauses.count("primaryKeyClause"))) throw FailedPredicateException(this, "!$clauses.count(\"primaryKeyClause\")"); - setState(838); + setState(854); primaryKeyClause(); _localctx->clauses.insert("primaryKeyClause"); break; } case 4: { - setState(841); + setState(857); if (!(!_localctx->clauses.count("sampleByClause"))) throw FailedPredicateException(this, "!$clauses.count(\"sampleByClause\")"); - setState(842); + setState(858); sampleByClause(); _localctx->clauses.insert("sampleByClause"); break; } case 5: { - setState(845); + setState(861); if (!(!_localctx->clauses.count("ttlClause"))) throw FailedPredicateException(this, "!$clauses.count(\"ttlClause\")"); - setState(846); + setState(862); ttlClause(); _localctx->clauses.insert("ttlClause"); break; } case 6: { - setState(849); + setState(865); if (!(!_localctx->clauses.count("settingsClause"))) throw FailedPredicateException(this, "!$clauses.count(\"settingsClause\")"); - setState(850); + setState(866); settingsClause(); _localctx->clauses.insert("settingsClause"); break; @@ -5685,9 +5748,9 @@ ClickHouseParser::EngineClauseContext* ClickHouseParser::engineClause() { } } - setState(857); + setState(873); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 88, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 92, _ctx); } } @@ -5739,11 +5802,11 @@ ClickHouseParser::PartitionByClauseContext* ClickHouseParser::partitionByClause( }); try { enterOuterAlt(_localctx, 1); - setState(858); + setState(874); match(ClickHouseParser::PARTITION); - setState(859); + setState(875); match(ClickHouseParser::BY); - setState(860); + setState(876); columnExpr(0); } @@ -5795,11 +5858,11 @@ ClickHouseParser::PrimaryKeyClauseContext* ClickHouseParser::primaryKeyClause() }); try { enterOuterAlt(_localctx, 1); - setState(862); + setState(878); match(ClickHouseParser::PRIMARY); - setState(863); + setState(879); match(ClickHouseParser::KEY); - setState(864); + setState(880); columnExpr(0); } @@ -5851,11 +5914,11 @@ ClickHouseParser::SampleByClauseContext* ClickHouseParser::sampleByClause() { }); try { enterOuterAlt(_localctx, 1); - setState(866); + setState(882); match(ClickHouseParser::SAMPLE); - setState(867); + setState(883); match(ClickHouseParser::BY); - setState(868); + setState(884); columnExpr(0); } @@ -5916,23 +5979,23 @@ ClickHouseParser::TtlClauseContext* ClickHouseParser::ttlClause() { try { size_t alt; enterOuterAlt(_localctx, 1); - setState(870); + setState(886); match(ClickHouseParser::TTL); - setState(871); + setState(887); ttlExpr(); - setState(876); + setState(892); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 89, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 93, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { - setState(872); + setState(888); match(ClickHouseParser::COMMA); - setState(873); + setState(889); ttlExpr(); } - setState(878); + setState(894); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 89, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 93, _ctx); } } @@ -5997,26 +6060,26 @@ ClickHouseParser::EngineExprContext* ClickHouseParser::engineExpr() { }); try { enterOuterAlt(_localctx, 1); - setState(879); + setState(895); match(ClickHouseParser::ENGINE); - setState(881); + setState(897); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::EQ_SINGLE) { - setState(880); + setState(896); match(ClickHouseParser::EQ_SINGLE); } - setState(883); + setState(899); identifierOrNull(); - setState(889); + setState(905); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 92, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 96, _ctx)) { case 1: { - setState(884); + setState(900); match(ClickHouseParser::LPAREN); - setState(886); + setState(902); _errHandler->sync(this); _la = _input->LA(1); @@ -6214,10 +6277,10 @@ ClickHouseParser::EngineExprContext* ClickHouseParser::engineExpr() { | (1ULL << (ClickHouseParser::LBRACKET - 197)) | (1ULL << (ClickHouseParser::LPAREN - 197)) | (1ULL << (ClickHouseParser::PLUS - 197)))) != 0)) { - setState(885); + setState(901); columnExprList(); } - setState(888); + setState(904); match(ClickHouseParser::RPAREN); break; } @@ -6333,13 +6396,13 @@ ClickHouseParser::TableElementExprContext* ClickHouseParser::tableElementExpr() exitRule(); }); try { - setState(901); + setState(917); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 93, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 97, _ctx)) { case 1: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(891); + setState(907); tableColumnDfnt(); break; } @@ -6347,13 +6410,13 @@ ClickHouseParser::TableElementExprContext* ClickHouseParser::tableElementExpr() case 2: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(892); + setState(908); match(ClickHouseParser::CONSTRAINT); - setState(893); + setState(909); identifier(); - setState(894); + setState(910); match(ClickHouseParser::CHECK); - setState(895); + setState(911); columnExpr(0); break; } @@ -6361,9 +6424,9 @@ ClickHouseParser::TableElementExprContext* ClickHouseParser::tableElementExpr() case 3: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 3); - setState(897); + setState(913); match(ClickHouseParser::INDEX); - setState(898); + setState(914); tableIndexDfnt(); break; } @@ -6371,9 +6434,9 @@ ClickHouseParser::TableElementExprContext* ClickHouseParser::tableElementExpr() case 4: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 4); - setState(899); + setState(915); match(ClickHouseParser::PROJECTION); - setState(900); + setState(916); tableProjectionDfnt(); break; } @@ -6449,73 +6512,25 @@ ClickHouseParser::TableColumnDfntContext* ClickHouseParser::tableColumnDfnt() { exitRule(); }); try { - setState(935); + setState(951); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 102, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 106, _ctx)) { case 1: { enterOuterAlt(_localctx, 1); - setState(903); + setState(919); nestedIdentifier(); - setState(904); + setState(920); columnTypeExpr(); - setState(906); + setState(922); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ALIAS || _la == ClickHouseParser::DEFAULT || _la == ClickHouseParser::MATERIALIZED) { - setState(905); + setState(921); tableColumnPropertyExpr(); } - setState(910); - _errHandler->sync(this); - - _la = _input->LA(1); - if (_la == ClickHouseParser::COMMENT) { - setState(908); - match(ClickHouseParser::COMMENT); - setState(909); - match(ClickHouseParser::STRING_LITERAL); - } - setState(913); - _errHandler->sync(this); - - _la = _input->LA(1); - if (_la == ClickHouseParser::CODEC) { - setState(912); - codecExpr(); - } - setState(917); - _errHandler->sync(this); - - _la = _input->LA(1); - if (_la == ClickHouseParser::TTL) { - setState(915); - match(ClickHouseParser::TTL); - setState(916); - columnExpr(0); - } - break; - } - - case 2: { - enterOuterAlt(_localctx, 2); - setState(919); - nestedIdentifier(); - setState(921); - _errHandler->sync(this); - - switch (getInterpreter()->adaptivePredict(_input, 98, _ctx)) { - case 1: { - setState(920); - columnTypeExpr(); - break; - } - - } - setState(923); - tableColumnPropertyExpr(); setState(926); _errHandler->sync(this); @@ -6547,6 +6562,54 @@ ClickHouseParser::TableColumnDfntContext* ClickHouseParser::tableColumnDfnt() { break; } + case 2: { + enterOuterAlt(_localctx, 2); + setState(935); + nestedIdentifier(); + setState(937); + _errHandler->sync(this); + + switch (getInterpreter()->adaptivePredict(_input, 102, _ctx)) { + case 1: { + setState(936); + columnTypeExpr(); + break; + } + + } + setState(939); + tableColumnPropertyExpr(); + setState(942); + _errHandler->sync(this); + + _la = _input->LA(1); + if (_la == ClickHouseParser::COMMENT) { + setState(940); + match(ClickHouseParser::COMMENT); + setState(941); + match(ClickHouseParser::STRING_LITERAL); + } + setState(945); + _errHandler->sync(this); + + _la = _input->LA(1); + if (_la == ClickHouseParser::CODEC) { + setState(944); + codecExpr(); + } + setState(949); + _errHandler->sync(this); + + _la = _input->LA(1); + if (_la == ClickHouseParser::TTL) { + setState(947); + match(ClickHouseParser::TTL); + setState(948); + columnExpr(0); + } + break; + } + } } @@ -6603,7 +6666,7 @@ ClickHouseParser::TableColumnPropertyExprContext* ClickHouseParser::tableColumnP }); try { enterOuterAlt(_localctx, 1); - setState(937); + setState(953); _la = _input->LA(1); if (!(_la == ClickHouseParser::ALIAS @@ -6614,7 +6677,7 @@ ClickHouseParser::TableColumnPropertyExprContext* ClickHouseParser::tableColumnP _errHandler->reportMatch(this); consume(); } - setState(938); + setState(954); columnExpr(0); } @@ -6678,17 +6741,17 @@ ClickHouseParser::TableIndexDfntContext* ClickHouseParser::tableIndexDfnt() { }); try { enterOuterAlt(_localctx, 1); - setState(940); + setState(956); nestedIdentifier(); - setState(941); + setState(957); columnExpr(0); - setState(942); + setState(958); match(ClickHouseParser::TYPE); - setState(943); + setState(959); columnTypeExpr(); - setState(944); + setState(960); match(ClickHouseParser::GRANULARITY); - setState(945); + setState(961); match(ClickHouseParser::DECIMAL_LITERAL); } @@ -6736,9 +6799,9 @@ ClickHouseParser::TableProjectionDfntContext* ClickHouseParser::tableProjectionD }); try { enterOuterAlt(_localctx, 1); - setState(947); + setState(963); nestedIdentifier(); - setState(948); + setState(964); projectionSelectStmt(); } @@ -6807,25 +6870,25 @@ ClickHouseParser::CodecExprContext* ClickHouseParser::codecExpr() { }); try { enterOuterAlt(_localctx, 1); - setState(950); + setState(966); match(ClickHouseParser::CODEC); - setState(951); + setState(967); match(ClickHouseParser::LPAREN); - setState(952); + setState(968); codecArgExpr(); - setState(957); + setState(973); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(953); + setState(969); match(ClickHouseParser::COMMA); - setState(954); + setState(970); codecArgExpr(); - setState(959); + setState(975); _errHandler->sync(this); _la = _input->LA(1); } - setState(960); + setState(976); match(ClickHouseParser::RPAREN); } @@ -6882,16 +6945,16 @@ ClickHouseParser::CodecArgExprContext* ClickHouseParser::codecArgExpr() { }); try { enterOuterAlt(_localctx, 1); - setState(962); + setState(978); identifier(); - setState(968); + setState(984); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::LPAREN) { - setState(963); + setState(979); match(ClickHouseParser::LPAREN); - setState(965); + setState(981); _errHandler->sync(this); _la = _input->LA(1); @@ -7089,10 +7152,10 @@ ClickHouseParser::CodecArgExprContext* ClickHouseParser::codecArgExpr() { | (1ULL << (ClickHouseParser::LBRACKET - 197)) | (1ULL << (ClickHouseParser::LPAREN - 197)) | (1ULL << (ClickHouseParser::PLUS - 197)))) != 0)) { - setState(964); + setState(980); columnExprList(); } - setState(967); + setState(983); match(ClickHouseParser::RPAREN); } @@ -7157,34 +7220,34 @@ ClickHouseParser::TtlExprContext* ClickHouseParser::ttlExpr() { }); try { enterOuterAlt(_localctx, 1); - setState(970); + setState(986); columnExpr(0); - setState(978); + setState(994); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 106, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 110, _ctx)) { case 1: { - setState(971); + setState(987); match(ClickHouseParser::DELETE); break; } case 2: { - setState(972); + setState(988); match(ClickHouseParser::TO); - setState(973); + setState(989); match(ClickHouseParser::DISK); - setState(974); + setState(990); match(ClickHouseParser::STRING_LITERAL); break; } case 3: { - setState(975); + setState(991); match(ClickHouseParser::TO); - setState(976); + setState(992); match(ClickHouseParser::VOLUME); - setState(977); + setState(993); match(ClickHouseParser::STRING_LITERAL); break; } @@ -7245,7 +7308,7 @@ ClickHouseParser::DescribeStmtContext* ClickHouseParser::describeStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(980); + setState(996); _la = _input->LA(1); if (!(_la == ClickHouseParser::DESC @@ -7256,18 +7319,18 @@ ClickHouseParser::DescribeStmtContext* ClickHouseParser::describeStmt() { _errHandler->reportMatch(this); consume(); } - setState(982); + setState(998); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 107, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 111, _ctx)) { case 1: { - setState(981); + setState(997); match(ClickHouseParser::TABLE); break; } } - setState(984); + setState(1000); tableExpr(0); } @@ -7400,55 +7463,13 @@ ClickHouseParser::DropStmtContext* ClickHouseParser::dropStmt() { exitRule(); }); try { - setState(1017); + setState(1033); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 115, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 119, _ctx)) { case 1: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(986); - _la = _input->LA(1); - if (!(_la == ClickHouseParser::DETACH - - || _la == ClickHouseParser::DROP)) { - _errHandler->recoverInline(this); - } - else { - _errHandler->reportMatch(this); - consume(); - } - setState(987); - match(ClickHouseParser::DATABASE); - setState(990); - _errHandler->sync(this); - - switch (getInterpreter()->adaptivePredict(_input, 108, _ctx)) { - case 1: { - setState(988); - match(ClickHouseParser::IF); - setState(989); - match(ClickHouseParser::EXISTS); - break; - } - - } - setState(992); - databaseIdentifier(); - setState(994); - _errHandler->sync(this); - - _la = _input->LA(1); - if (_la == ClickHouseParser::ON) { - setState(993); - clusterClause(); - } - break; - } - - case 2: { - _localctx = dynamic_cast(_tracker.createInstance(_localctx)); - enterOuterAlt(_localctx, 2); - setState(996); + setState(1002); _la = _input->LA(1); if (!(_la == ClickHouseParser::DETACH @@ -7460,31 +7481,73 @@ ClickHouseParser::DropStmtContext* ClickHouseParser::dropStmt() { consume(); } setState(1003); + match(ClickHouseParser::DATABASE); + setState(1006); + _errHandler->sync(this); + + switch (getInterpreter()->adaptivePredict(_input, 112, _ctx)) { + case 1: { + setState(1004); + match(ClickHouseParser::IF); + setState(1005); + match(ClickHouseParser::EXISTS); + break; + } + + } + setState(1008); + databaseIdentifier(); + setState(1010); + _errHandler->sync(this); + + _la = _input->LA(1); + if (_la == ClickHouseParser::ON) { + setState(1009); + clusterClause(); + } + break; + } + + case 2: { + _localctx = dynamic_cast(_tracker.createInstance(_localctx)); + enterOuterAlt(_localctx, 2); + setState(1012); + _la = _input->LA(1); + if (!(_la == ClickHouseParser::DETACH + + || _la == ClickHouseParser::DROP)) { + _errHandler->recoverInline(this); + } + else { + _errHandler->reportMatch(this); + consume(); + } + setState(1019); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::DICTIONARY: { - setState(997); + setState(1013); match(ClickHouseParser::DICTIONARY); break; } case ClickHouseParser::TABLE: case ClickHouseParser::TEMPORARY: { - setState(999); + setState(1015); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::TEMPORARY) { - setState(998); + setState(1014); match(ClickHouseParser::TEMPORARY); } - setState(1001); + setState(1017); match(ClickHouseParser::TABLE); break; } case ClickHouseParser::VIEW: { - setState(1002); + setState(1018); match(ClickHouseParser::VIEW); break; } @@ -7492,37 +7555,37 @@ ClickHouseParser::DropStmtContext* ClickHouseParser::dropStmt() { default: throw NoViableAltException(this); } - setState(1007); + setState(1023); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 112, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 116, _ctx)) { case 1: { - setState(1005); + setState(1021); match(ClickHouseParser::IF); - setState(1006); + setState(1022); match(ClickHouseParser::EXISTS); break; } } - setState(1009); + setState(1025); tableIdentifier(); - setState(1011); + setState(1027); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ON) { - setState(1010); + setState(1026); clusterClause(); } - setState(1015); + setState(1031); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::NO) { - setState(1013); + setState(1029); match(ClickHouseParser::NO); - setState(1014); + setState(1030); match(ClickHouseParser::DELAY); } break; @@ -7620,17 +7683,17 @@ ClickHouseParser::ExistsStmtContext* ClickHouseParser::existsStmt() { exitRule(); }); try { - setState(1032); + setState(1048); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 118, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 122, _ctx)) { case 1: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(1019); + setState(1035); match(ClickHouseParser::EXISTS); - setState(1020); + setState(1036); match(ClickHouseParser::DATABASE); - setState(1021); + setState(1037); databaseIdentifier(); break; } @@ -7638,40 +7701,40 @@ ClickHouseParser::ExistsStmtContext* ClickHouseParser::existsStmt() { case 2: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(1022); + setState(1038); match(ClickHouseParser::EXISTS); - setState(1029); + setState(1045); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 117, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 121, _ctx)) { case 1: { - setState(1023); + setState(1039); match(ClickHouseParser::DICTIONARY); break; } case 2: { - setState(1025); + setState(1041); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::TEMPORARY) { - setState(1024); + setState(1040); match(ClickHouseParser::TEMPORARY); } - setState(1027); + setState(1043); match(ClickHouseParser::TABLE); break; } case 3: { - setState(1028); + setState(1044); match(ClickHouseParser::VIEW); break; } } - setState(1031); + setState(1047); tableIdentifier(); break; } @@ -7755,17 +7818,17 @@ ClickHouseParser::ExplainStmtContext* ClickHouseParser::explainStmt() { exitRule(); }); try { - setState(1040); + setState(1056); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 119, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 123, _ctx)) { case 1: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(1034); + setState(1050); match(ClickHouseParser::EXPLAIN); - setState(1035); + setState(1051); match(ClickHouseParser::AST); - setState(1036); + setState(1052); query(); break; } @@ -7773,11 +7836,11 @@ ClickHouseParser::ExplainStmtContext* ClickHouseParser::explainStmt() { case 2: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(1037); + setState(1053); match(ClickHouseParser::EXPLAIN); - setState(1038); + setState(1054); match(ClickHouseParser::SYNTAX); - setState(1039); + setState(1055); query(); break; } @@ -7853,51 +7916,51 @@ ClickHouseParser::InsertStmtContext* ClickHouseParser::insertStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1042); + setState(1058); match(ClickHouseParser::INSERT); - setState(1043); + setState(1059); match(ClickHouseParser::INTO); - setState(1045); + setState(1061); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 120, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 124, _ctx)) { case 1: { - setState(1044); + setState(1060); match(ClickHouseParser::TABLE); break; } } - setState(1050); + setState(1066); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 121, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 125, _ctx)) { case 1: { - setState(1047); + setState(1063); tableIdentifier(); break; } case 2: { - setState(1048); + setState(1064); match(ClickHouseParser::FUNCTION); - setState(1049); + setState(1065); tableFunctionExpr(); break; } } - setState(1053); + setState(1069); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 122, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 126, _ctx)) { case 1: { - setState(1052); + setState(1068); columnsClause(); break; } } - setState(1055); + setState(1071); dataClause(); } @@ -7962,23 +8025,23 @@ ClickHouseParser::ColumnsClauseContext* ClickHouseParser::columnsClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1057); + setState(1073); match(ClickHouseParser::LPAREN); - setState(1058); + setState(1074); nestedIdentifier(); - setState(1063); + setState(1079); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1059); + setState(1075); match(ClickHouseParser::COMMA); - setState(1060); + setState(1076); nestedIdentifier(); - setState(1065); + setState(1081); _errHandler->sync(this); _la = _input->LA(1); } - setState(1066); + setState(1082); match(ClickHouseParser::RPAREN); } @@ -8069,15 +8132,15 @@ ClickHouseParser::DataClauseContext* ClickHouseParser::dataClause() { exitRule(); }); try { - setState(1077); + setState(1093); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::FORMAT: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(1068); + setState(1084); match(ClickHouseParser::FORMAT); - setState(1069); + setState(1085); identifier(); break; } @@ -8085,7 +8148,7 @@ ClickHouseParser::DataClauseContext* ClickHouseParser::dataClause() { case ClickHouseParser::VALUES: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(1070); + setState(1086); match(ClickHouseParser::VALUES); break; } @@ -8095,17 +8158,17 @@ ClickHouseParser::DataClauseContext* ClickHouseParser::dataClause() { case ClickHouseParser::LPAREN: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 3); - setState(1071); + setState(1087); selectUnionStmt(); - setState(1073); + setState(1089); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::SEMICOLON) { - setState(1072); + setState(1088); match(ClickHouseParser::SEMICOLON); } - setState(1075); + setState(1091); match(ClickHouseParser::EOF); break; } @@ -8188,28 +8251,28 @@ ClickHouseParser::KillStmtContext* ClickHouseParser::killStmt() { try { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(1079); + setState(1095); match(ClickHouseParser::KILL); - setState(1080); + setState(1096); match(ClickHouseParser::MUTATION); - setState(1082); + setState(1098); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ON) { - setState(1081); + setState(1097); clusterClause(); } - setState(1084); + setState(1100); whereClause(); - setState(1086); + setState(1102); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ASYNC || _la == ClickHouseParser::SYNC || _la == ClickHouseParser::TEST) { - setState(1085); + setState(1101); _la = _input->LA(1); if (!(_la == ClickHouseParser::ASYNC || _la == ClickHouseParser::SYNC @@ -8288,42 +8351,42 @@ ClickHouseParser::OptimizeStmtContext* ClickHouseParser::optimizeStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1088); + setState(1104); match(ClickHouseParser::OPTIMIZE); - setState(1089); + setState(1105); match(ClickHouseParser::TABLE); - setState(1090); + setState(1106); tableIdentifier(); - setState(1092); + setState(1108); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ON) { - setState(1091); + setState(1107); clusterClause(); } - setState(1095); + setState(1111); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::PARTITION) { - setState(1094); + setState(1110); partitionClause(); } - setState(1098); + setState(1114); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::FINAL) { - setState(1097); + setState(1113); match(ClickHouseParser::FINAL); } - setState(1101); + setState(1117); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::DEDUPLICATE) { - setState(1100); + setState(1116); match(ClickHouseParser::DEDUPLICATE); } @@ -8401,38 +8464,38 @@ ClickHouseParser::RenameStmtContext* ClickHouseParser::renameStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1103); + setState(1119); match(ClickHouseParser::RENAME); - setState(1104); + setState(1120); match(ClickHouseParser::TABLE); - setState(1105); + setState(1121); tableIdentifier(); - setState(1106); + setState(1122); match(ClickHouseParser::TO); - setState(1107); + setState(1123); tableIdentifier(); - setState(1115); + setState(1131); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1108); + setState(1124); match(ClickHouseParser::COMMA); - setState(1109); + setState(1125); tableIdentifier(); - setState(1110); + setState(1126); match(ClickHouseParser::TO); - setState(1111); + setState(1127); tableIdentifier(); - setState(1117); + setState(1133); _errHandler->sync(this); _la = _input->LA(1); } - setState(1119); + setState(1135); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ON) { - setState(1118); + setState(1134); clusterClause(); } @@ -8502,37 +8565,37 @@ ClickHouseParser::ProjectionSelectStmtContext* ClickHouseParser::projectionSelec }); try { enterOuterAlt(_localctx, 1); - setState(1121); + setState(1137); match(ClickHouseParser::LPAREN); - setState(1123); + setState(1139); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::WITH) { - setState(1122); + setState(1138); withClause(); } - setState(1125); + setState(1141); match(ClickHouseParser::SELECT); - setState(1126); + setState(1142); columnExprList(); - setState(1128); + setState(1144); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::GROUP) { - setState(1127); + setState(1143); groupByClause(); } - setState(1131); + setState(1147); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ORDER) { - setState(1130); + setState(1146); projectionOrderByClause(); } - setState(1133); + setState(1149); match(ClickHouseParser::RPAREN); } @@ -8597,19 +8660,19 @@ ClickHouseParser::SelectUnionStmtContext* ClickHouseParser::selectUnionStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1135); + setState(1151); selectStmtWithParens(); - setState(1141); + setState(1157); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::UNION) { - setState(1136); + setState(1152); match(ClickHouseParser::UNION); - setState(1137); + setState(1153); match(ClickHouseParser::ALL); - setState(1138); + setState(1154); selectStmtWithParens(); - setState(1143); + setState(1159); _errHandler->sync(this); _la = _input->LA(1); } @@ -8666,24 +8729,24 @@ ClickHouseParser::SelectStmtWithParensContext* ClickHouseParser::selectStmtWithP exitRule(); }); try { - setState(1149); + setState(1165); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::SELECT: case ClickHouseParser::WITH: { enterOuterAlt(_localctx, 1); - setState(1144); + setState(1160); selectStmt(); break; } case ClickHouseParser::LPAREN: { enterOuterAlt(_localctx, 2); - setState(1145); + setState(1161); match(ClickHouseParser::LPAREN); - setState(1146); + setState(1162); selectUnionStmt(); - setState(1147); + setState(1163); match(ClickHouseParser::RPAREN); break; } @@ -8810,90 +8873,90 @@ ClickHouseParser::SelectStmtContext* ClickHouseParser::selectStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1152); + setState(1168); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::WITH) { - setState(1151); + setState(1167); withClause(); } - setState(1154); + setState(1170); match(ClickHouseParser::SELECT); - setState(1156); + setState(1172); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 140, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 144, _ctx)) { case 1: { - setState(1155); + setState(1171); match(ClickHouseParser::DISTINCT); break; } } - setState(1159); + setState(1175); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 141, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 145, _ctx)) { case 1: { - setState(1158); + setState(1174); topClause(); break; } } - setState(1161); + setState(1177); columnExprList(); - setState(1163); + setState(1179); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::FROM) { - setState(1162); + setState(1178); fromClause(); } - setState(1166); + setState(1182); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ARRAY || _la == ClickHouseParser::INNER || _la == ClickHouseParser::LEFT) { - setState(1165); + setState(1181); arrayJoinClause(); } - setState(1169); + setState(1185); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::PREWHERE) { - setState(1168); + setState(1184); prewhereClause(); } - setState(1172); + setState(1188); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::WHERE) { - setState(1171); + setState(1187); whereClause(); } - setState(1175); + setState(1191); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::GROUP) { - setState(1174); + setState(1190); groupByClause(); } - setState(1179); + setState(1195); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 147, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 151, _ctx)) { case 1: { - setState(1177); + setState(1193); match(ClickHouseParser::WITH); - setState(1178); + setState(1194); _la = _input->LA(1); if (!(_la == ClickHouseParser::CUBE || _la == ClickHouseParser::ROLLUP)) { _errHandler->recoverInline(this); @@ -8906,57 +8969,57 @@ ClickHouseParser::SelectStmtContext* ClickHouseParser::selectStmt() { } } - setState(1183); + setState(1199); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::WITH) { - setState(1181); + setState(1197); match(ClickHouseParser::WITH); - setState(1182); + setState(1198); match(ClickHouseParser::TOTALS); } - setState(1186); + setState(1202); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::HAVING) { - setState(1185); + setState(1201); havingClause(); } - setState(1189); + setState(1205); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ORDER) { - setState(1188); + setState(1204); orderByClause(); } - setState(1192); + setState(1208); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 151, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 155, _ctx)) { case 1: { - setState(1191); + setState(1207); limitByClause(); break; } } - setState(1195); + setState(1211); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::LIMIT) { - setState(1194); + setState(1210); limitClause(); } - setState(1198); + setState(1214); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::SETTINGS) { - setState(1197); + setState(1213); settingsClause(); } @@ -9005,9 +9068,9 @@ ClickHouseParser::WithClauseContext* ClickHouseParser::withClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1200); + setState(1216); match(ClickHouseParser::WITH); - setState(1201); + setState(1217); columnExprList(); } @@ -9063,18 +9126,18 @@ ClickHouseParser::TopClauseContext* ClickHouseParser::topClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1203); + setState(1219); match(ClickHouseParser::TOP); - setState(1204); + setState(1220); match(ClickHouseParser::DECIMAL_LITERAL); - setState(1207); + setState(1223); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 154, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 158, _ctx)) { case 1: { - setState(1205); + setState(1221); match(ClickHouseParser::WITH); - setState(1206); + setState(1222); match(ClickHouseParser::TIES); break; } @@ -9126,9 +9189,9 @@ ClickHouseParser::FromClauseContext* ClickHouseParser::fromClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1209); + setState(1225); match(ClickHouseParser::FROM); - setState(1210); + setState(1226); joinExpr(0); } @@ -9189,14 +9252,14 @@ ClickHouseParser::ArrayJoinClauseContext* ClickHouseParser::arrayJoinClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1213); + setState(1229); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::INNER || _la == ClickHouseParser::LEFT) { - setState(1212); + setState(1228); _la = _input->LA(1); if (!(_la == ClickHouseParser::INNER @@ -9208,11 +9271,11 @@ ClickHouseParser::ArrayJoinClauseContext* ClickHouseParser::arrayJoinClause() { consume(); } } - setState(1215); + setState(1231); match(ClickHouseParser::ARRAY); - setState(1216); + setState(1232); match(ClickHouseParser::JOIN); - setState(1217); + setState(1233); columnExprList(); } @@ -9260,9 +9323,9 @@ ClickHouseParser::PrewhereClauseContext* ClickHouseParser::prewhereClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1219); + setState(1235); match(ClickHouseParser::PREWHERE); - setState(1220); + setState(1236); columnExpr(0); } @@ -9310,9 +9373,9 @@ ClickHouseParser::WhereClauseContext* ClickHouseParser::whereClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1222); + setState(1238); match(ClickHouseParser::WHERE); - setState(1223); + setState(1239); columnExpr(0); } @@ -9381,15 +9444,15 @@ ClickHouseParser::GroupByClauseContext* ClickHouseParser::groupByClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1225); + setState(1241); match(ClickHouseParser::GROUP); - setState(1226); + setState(1242); match(ClickHouseParser::BY); - setState(1233); + setState(1249); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 156, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 160, _ctx)) { case 1: { - setState(1227); + setState(1243); _la = _input->LA(1); if (!(_la == ClickHouseParser::CUBE || _la == ClickHouseParser::ROLLUP)) { _errHandler->recoverInline(this); @@ -9398,17 +9461,17 @@ ClickHouseParser::GroupByClauseContext* ClickHouseParser::groupByClause() { _errHandler->reportMatch(this); consume(); } - setState(1228); + setState(1244); match(ClickHouseParser::LPAREN); - setState(1229); + setState(1245); columnExprList(); - setState(1230); + setState(1246); match(ClickHouseParser::RPAREN); break; } case 2: { - setState(1232); + setState(1248); columnExprList(); break; } @@ -9460,9 +9523,9 @@ ClickHouseParser::HavingClauseContext* ClickHouseParser::havingClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1235); + setState(1251); match(ClickHouseParser::HAVING); - setState(1236); + setState(1252); columnExpr(0); } @@ -9514,11 +9577,11 @@ ClickHouseParser::OrderByClauseContext* ClickHouseParser::orderByClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1238); + setState(1254); match(ClickHouseParser::ORDER); - setState(1239); + setState(1255); match(ClickHouseParser::BY); - setState(1240); + setState(1256); orderExprList(); } @@ -9570,11 +9633,11 @@ ClickHouseParser::ProjectionOrderByClauseContext* ClickHouseParser::projectionOr }); try { enterOuterAlt(_localctx, 1); - setState(1242); + setState(1258); match(ClickHouseParser::ORDER); - setState(1243); + setState(1259); match(ClickHouseParser::BY); - setState(1244); + setState(1260); columnExprList(); } @@ -9630,13 +9693,13 @@ ClickHouseParser::LimitByClauseContext* ClickHouseParser::limitByClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1246); + setState(1262); match(ClickHouseParser::LIMIT); - setState(1247); + setState(1263); limitExpr(); - setState(1248); + setState(1264); match(ClickHouseParser::BY); - setState(1249); + setState(1265); columnExprList(); } @@ -9693,18 +9756,18 @@ ClickHouseParser::LimitClauseContext* ClickHouseParser::limitClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1251); + setState(1267); match(ClickHouseParser::LIMIT); - setState(1252); + setState(1268); limitExpr(); - setState(1255); + setState(1271); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::WITH) { - setState(1253); + setState(1269); match(ClickHouseParser::WITH); - setState(1254); + setState(1270); match(ClickHouseParser::TIES); } @@ -9753,9 +9816,9 @@ ClickHouseParser::SettingsClauseContext* ClickHouseParser::settingsClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1257); + setState(1273); match(ClickHouseParser::SETTINGS); - setState(1258); + setState(1274); settingExprList(); } @@ -9909,33 +9972,33 @@ ClickHouseParser::JoinExprContext* ClickHouseParser::joinExpr(int precedence) { try { size_t alt; enterOuterAlt(_localctx, 1); - setState(1272); + setState(1288); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 160, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 164, _ctx)) { case 1: { _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1261); + setState(1277); tableExpr(0); - setState(1263); + setState(1279); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 158, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 162, _ctx)) { case 1: { - setState(1262); + setState(1278); match(ClickHouseParser::FINAL); break; } } - setState(1266); + setState(1282); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 159, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 163, _ctx)) { case 1: { - setState(1265); + setState(1281); sampleClause(); break; } @@ -9948,38 +10011,38 @@ ClickHouseParser::JoinExprContext* ClickHouseParser::joinExpr(int precedence) { _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1268); + setState(1284); match(ClickHouseParser::LPAREN); - setState(1269); + setState(1285); joinExpr(0); - setState(1270); + setState(1286); match(ClickHouseParser::RPAREN); break; } } _ctx->stop = _input->LT(-1); - setState(1291); + setState(1307); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 164, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 168, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { if (!_parseListeners.empty()) triggerExitRuleEvent(); previousContext = _localctx; - setState(1289); + setState(1305); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 163, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 167, _ctx)) { case 1: { auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleJoinExpr); - setState(1274); + setState(1290); if (!(precpred(_ctx, 3))) throw FailedPredicateException(this, "precpred(_ctx, 3)"); - setState(1275); + setState(1291); joinOpCross(); - setState(1276); + setState(1292); joinExpr(4); break; } @@ -9988,17 +10051,17 @@ ClickHouseParser::JoinExprContext* ClickHouseParser::joinExpr(int precedence) { auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleJoinExpr); - setState(1278); + setState(1294); if (!(precpred(_ctx, 4))) throw FailedPredicateException(this, "precpred(_ctx, 4)"); - setState(1280); + setState(1296); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::GLOBAL || _la == ClickHouseParser::LOCAL) { - setState(1279); + setState(1295); _la = _input->LA(1); if (!(_la == ClickHouseParser::GLOBAL @@ -10010,7 +10073,7 @@ ClickHouseParser::JoinExprContext* ClickHouseParser::joinExpr(int precedence) { consume(); } } - setState(1283); + setState(1299); _errHandler->sync(this); _la = _input->LA(1); @@ -10024,23 +10087,23 @@ ClickHouseParser::JoinExprContext* ClickHouseParser::joinExpr(int precedence) { | (1ULL << (ClickHouseParser::LEFT - 81)) | (1ULL << (ClickHouseParser::RIGHT - 81)) | (1ULL << (ClickHouseParser::SEMI - 81)))) != 0)) { - setState(1282); + setState(1298); joinOp(); } - setState(1285); + setState(1301); match(ClickHouseParser::JOIN); - setState(1286); + setState(1302); joinExpr(0); - setState(1287); + setState(1303); joinConstraintClause(); break; } } } - setState(1293); + setState(1309); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 164, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 168, _ctx); } } catch (RecognitionException &e) { @@ -10169,17 +10232,17 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { exitRule(); }); try { - setState(1337); + setState(1353); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 178, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 182, _ctx)) { case 1: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(1303); + setState(1319); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 167, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 171, _ctx)) { case 1: { - setState(1295); + setState(1311); _errHandler->sync(this); _la = _input->LA(1); @@ -10187,7 +10250,7 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { ((1ULL << _la) & ((1ULL << ClickHouseParser::ALL) | (1ULL << ClickHouseParser::ANY) | (1ULL << ClickHouseParser::ASOF))) != 0)) { - setState(1294); + setState(1310); _la = _input->LA(1); if (!((((_la & ~ 0x3fULL) == 0) && ((1ULL << _la) & ((1ULL << ClickHouseParser::ALL) @@ -10200,15 +10263,15 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { consume(); } } - setState(1297); + setState(1313); match(ClickHouseParser::INNER); break; } case 2: { - setState(1298); + setState(1314); match(ClickHouseParser::INNER); - setState(1300); + setState(1316); _errHandler->sync(this); _la = _input->LA(1); @@ -10216,7 +10279,7 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { ((1ULL << _la) & ((1ULL << ClickHouseParser::ALL) | (1ULL << ClickHouseParser::ANY) | (1ULL << ClickHouseParser::ASOF))) != 0)) { - setState(1299); + setState(1315); _la = _input->LA(1); if (!((((_la & ~ 0x3fULL) == 0) && ((1ULL << _la) & ((1ULL << ClickHouseParser::ALL) @@ -10233,7 +10296,7 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { } case 3: { - setState(1302); + setState(1318); _la = _input->LA(1); if (!((((_la & ~ 0x3fULL) == 0) && ((1ULL << _la) & ((1ULL << ClickHouseParser::ALL) @@ -10255,11 +10318,11 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { case 2: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(1319); + setState(1335); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 172, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 176, _ctx)) { case 1: { - setState(1306); + setState(1322); _errHandler->sync(this); _la = _input->LA(1); @@ -10268,7 +10331,7 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { | (1ULL << ClickHouseParser::ANTI) | (1ULL << ClickHouseParser::ANY) | (1ULL << ClickHouseParser::ASOF))) != 0) || _la == ClickHouseParser::SEMI) { - setState(1305); + setState(1321); _la = _input->LA(1); if (!((((_la & ~ 0x3fULL) == 0) && ((1ULL << _la) & ((1ULL << ClickHouseParser::ALL) @@ -10282,7 +10345,7 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { consume(); } } - setState(1308); + setState(1324); _la = _input->LA(1); if (!(_la == ClickHouseParser::LEFT @@ -10293,19 +10356,19 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { _errHandler->reportMatch(this); consume(); } - setState(1310); + setState(1326); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::OUTER) { - setState(1309); + setState(1325); match(ClickHouseParser::OUTER); } break; } case 2: { - setState(1312); + setState(1328); _la = _input->LA(1); if (!(_la == ClickHouseParser::LEFT @@ -10316,15 +10379,15 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { _errHandler->reportMatch(this); consume(); } - setState(1314); + setState(1330); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::OUTER) { - setState(1313); + setState(1329); match(ClickHouseParser::OUTER); } - setState(1317); + setState(1333); _errHandler->sync(this); _la = _input->LA(1); @@ -10333,7 +10396,7 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { | (1ULL << ClickHouseParser::ANTI) | (1ULL << ClickHouseParser::ANY) | (1ULL << ClickHouseParser::ASOF))) != 0) || _la == ClickHouseParser::SEMI) { - setState(1316); + setState(1332); _la = _input->LA(1); if (!((((_la & ~ 0x3fULL) == 0) && ((1ULL << _la) & ((1ULL << ClickHouseParser::ALL) @@ -10357,18 +10420,18 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { case 3: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 3); - setState(1335); + setState(1351); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 177, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 181, _ctx)) { case 1: { - setState(1322); + setState(1338); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ALL || _la == ClickHouseParser::ANY) { - setState(1321); + setState(1337); _la = _input->LA(1); if (!(_la == ClickHouseParser::ALL @@ -10380,38 +10443,38 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { consume(); } } - setState(1324); + setState(1340); match(ClickHouseParser::FULL); - setState(1326); + setState(1342); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::OUTER) { - setState(1325); + setState(1341); match(ClickHouseParser::OUTER); } break; } case 2: { - setState(1328); + setState(1344); match(ClickHouseParser::FULL); - setState(1330); + setState(1346); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::OUTER) { - setState(1329); + setState(1345); match(ClickHouseParser::OUTER); } - setState(1333); + setState(1349); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ALL || _la == ClickHouseParser::ANY) { - setState(1332); + setState(1348); _la = _input->LA(1); if (!(_la == ClickHouseParser::ALL @@ -10489,21 +10552,21 @@ ClickHouseParser::JoinOpCrossContext* ClickHouseParser::joinOpCross() { exitRule(); }); try { - setState(1345); + setState(1361); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::CROSS: case ClickHouseParser::GLOBAL: case ClickHouseParser::LOCAL: { enterOuterAlt(_localctx, 1); - setState(1340); + setState(1356); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::GLOBAL || _la == ClickHouseParser::LOCAL) { - setState(1339); + setState(1355); _la = _input->LA(1); if (!(_la == ClickHouseParser::GLOBAL @@ -10515,16 +10578,16 @@ ClickHouseParser::JoinOpCrossContext* ClickHouseParser::joinOpCross() { consume(); } } - setState(1342); + setState(1358); match(ClickHouseParser::CROSS); - setState(1343); + setState(1359); match(ClickHouseParser::JOIN); break; } case ClickHouseParser::COMMA: { enterOuterAlt(_localctx, 2); - setState(1344); + setState(1360); match(ClickHouseParser::COMMA); break; } @@ -10589,36 +10652,36 @@ ClickHouseParser::JoinConstraintClauseContext* ClickHouseParser::joinConstraintC exitRule(); }); try { - setState(1356); + setState(1372); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 181, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 185, _ctx)) { case 1: { enterOuterAlt(_localctx, 1); - setState(1347); + setState(1363); match(ClickHouseParser::ON); - setState(1348); + setState(1364); columnExprList(); break; } case 2: { enterOuterAlt(_localctx, 2); - setState(1349); + setState(1365); match(ClickHouseParser::USING); - setState(1350); + setState(1366); match(ClickHouseParser::LPAREN); - setState(1351); + setState(1367); columnExprList(); - setState(1352); + setState(1368); match(ClickHouseParser::RPAREN); break; } case 3: { enterOuterAlt(_localctx, 3); - setState(1354); + setState(1370); match(ClickHouseParser::USING); - setState(1355); + setState(1371); columnExprList(); break; } @@ -10678,18 +10741,18 @@ ClickHouseParser::SampleClauseContext* ClickHouseParser::sampleClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1358); + setState(1374); match(ClickHouseParser::SAMPLE); - setState(1359); + setState(1375); ratioExpr(); - setState(1362); + setState(1378); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 182, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 186, _ctx)) { case 1: { - setState(1360); + setState(1376); match(ClickHouseParser::OFFSET); - setState(1361); + setState(1377); ratioExpr(); break; } @@ -10750,14 +10813,14 @@ ClickHouseParser::LimitExprContext* ClickHouseParser::limitExpr() { }); try { enterOuterAlt(_localctx, 1); - setState(1364); + setState(1380); columnExpr(0); - setState(1367); + setState(1383); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::OFFSET || _la == ClickHouseParser::COMMA) { - setState(1365); + setState(1381); _la = _input->LA(1); if (!(_la == ClickHouseParser::OFFSET || _la == ClickHouseParser::COMMA)) { _errHandler->recoverInline(this); @@ -10766,7 +10829,7 @@ ClickHouseParser::LimitExprContext* ClickHouseParser::limitExpr() { _errHandler->reportMatch(this); consume(); } - setState(1366); + setState(1382); columnExpr(0); } @@ -10824,21 +10887,21 @@ ClickHouseParser::OrderExprListContext* ClickHouseParser::orderExprList() { try { size_t alt; enterOuterAlt(_localctx, 1); - setState(1369); + setState(1385); orderExpr(); - setState(1374); + setState(1390); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 184, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 188, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { - setState(1370); + setState(1386); match(ClickHouseParser::COMMA); - setState(1371); + setState(1387); orderExpr(); } - setState(1376); + setState(1392); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 184, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 188, _ctx); } } @@ -10915,14 +10978,14 @@ ClickHouseParser::OrderExprContext* ClickHouseParser::orderExpr() { }); try { enterOuterAlt(_localctx, 1); - setState(1377); + setState(1393); columnExpr(0); - setState(1379); + setState(1395); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 185, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 189, _ctx)) { case 1: { - setState(1378); + setState(1394); _la = _input->LA(1); if (!((((_la & ~ 0x3fULL) == 0) && ((1ULL << _la) & ((1ULL << ClickHouseParser::ASCENDING) @@ -10938,14 +11001,14 @@ ClickHouseParser::OrderExprContext* ClickHouseParser::orderExpr() { } } - setState(1383); + setState(1399); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 186, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 190, _ctx)) { case 1: { - setState(1381); + setState(1397); match(ClickHouseParser::NULLS); - setState(1382); + setState(1398); _la = _input->LA(1); if (!(_la == ClickHouseParser::FIRST @@ -10960,14 +11023,14 @@ ClickHouseParser::OrderExprContext* ClickHouseParser::orderExpr() { } } - setState(1387); + setState(1403); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 187, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 191, _ctx)) { case 1: { - setState(1385); + setState(1401); match(ClickHouseParser::COLLATE); - setState(1386); + setState(1402); match(ClickHouseParser::STRING_LITERAL); break; } @@ -11023,16 +11086,16 @@ ClickHouseParser::RatioExprContext* ClickHouseParser::ratioExpr() { }); try { enterOuterAlt(_localctx, 1); - setState(1389); + setState(1405); numberLiteral(); - setState(1392); + setState(1408); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 188, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 192, _ctx)) { case 1: { - setState(1390); + setState(1406); match(ClickHouseParser::SLASH); - setState(1391); + setState(1407); numberLiteral(); break; } @@ -11093,21 +11156,21 @@ ClickHouseParser::SettingExprListContext* ClickHouseParser::settingExprList() { try { size_t alt; enterOuterAlt(_localctx, 1); - setState(1394); + setState(1410); settingExpr(); - setState(1399); + setState(1415); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 189, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 193, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { - setState(1395); + setState(1411); match(ClickHouseParser::COMMA); - setState(1396); + setState(1412); settingExpr(); } - setState(1401); + setState(1417); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 189, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 193, _ctx); } } @@ -11159,11 +11222,11 @@ ClickHouseParser::SettingExprContext* ClickHouseParser::settingExpr() { }); try { enterOuterAlt(_localctx, 1); - setState(1402); + setState(1418); identifier(); - setState(1403); + setState(1419); match(ClickHouseParser::EQ_SINGLE); - setState(1404); + setState(1420); literal(); } @@ -11211,9 +11274,9 @@ ClickHouseParser::SetStmtContext* ClickHouseParser::setStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1406); + setState(1422); match(ClickHouseParser::SET); - setState(1407); + setState(1423); settingExprList(); } @@ -11426,19 +11489,19 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { exitRule(); }); try { - setState(1451); + setState(1467); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 197, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 201, _ctx)) { case 1: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(1409); + setState(1425); match(ClickHouseParser::SHOW); - setState(1410); + setState(1426); match(ClickHouseParser::CREATE); - setState(1411); + setState(1427); match(ClickHouseParser::DATABASE); - setState(1412); + setState(1428); databaseIdentifier(); break; } @@ -11446,13 +11509,13 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { case 2: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(1413); + setState(1429); match(ClickHouseParser::SHOW); - setState(1414); + setState(1430); match(ClickHouseParser::CREATE); - setState(1415); + setState(1431); match(ClickHouseParser::DICTIONARY); - setState(1416); + setState(1432); tableIdentifier(); break; } @@ -11460,33 +11523,33 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { case 3: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 3); - setState(1417); + setState(1433); match(ClickHouseParser::SHOW); - setState(1418); + setState(1434); match(ClickHouseParser::CREATE); - setState(1420); + setState(1436); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 190, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 194, _ctx)) { case 1: { - setState(1419); + setState(1435); match(ClickHouseParser::TEMPORARY); break; } } - setState(1423); + setState(1439); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 191, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 195, _ctx)) { case 1: { - setState(1422); + setState(1438); match(ClickHouseParser::TABLE); break; } } - setState(1425); + setState(1441); tableIdentifier(); break; } @@ -11494,9 +11557,9 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { case 4: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 4); - setState(1426); + setState(1442); match(ClickHouseParser::SHOW); - setState(1427); + setState(1443); match(ClickHouseParser::DATABASES); break; } @@ -11504,18 +11567,18 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { case 5: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 5); - setState(1428); + setState(1444); match(ClickHouseParser::SHOW); - setState(1429); + setState(1445); match(ClickHouseParser::DICTIONARIES); - setState(1432); + setState(1448); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::FROM) { - setState(1430); + setState(1446); match(ClickHouseParser::FROM); - setState(1431); + setState(1447); databaseIdentifier(); } break; @@ -11524,26 +11587,26 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { case 6: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 6); - setState(1434); + setState(1450); match(ClickHouseParser::SHOW); - setState(1436); + setState(1452); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::TEMPORARY) { - setState(1435); + setState(1451); match(ClickHouseParser::TEMPORARY); } - setState(1438); + setState(1454); match(ClickHouseParser::TABLES); - setState(1441); + setState(1457); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::FROM || _la == ClickHouseParser::IN) { - setState(1439); + setState(1455); _la = _input->LA(1); if (!(_la == ClickHouseParser::FROM @@ -11554,22 +11617,22 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { _errHandler->reportMatch(this); consume(); } - setState(1440); + setState(1456); databaseIdentifier(); } - setState(1446); + setState(1462); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::LIKE: { - setState(1443); + setState(1459); match(ClickHouseParser::LIKE); - setState(1444); + setState(1460); match(ClickHouseParser::STRING_LITERAL); break; } case ClickHouseParser::WHERE: { - setState(1445); + setState(1461); whereClause(); break; } @@ -11585,12 +11648,12 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { default: break; } - setState(1449); + setState(1465); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::LIMIT) { - setState(1448); + setState(1464); limitClause(); } break; @@ -11703,62 +11766,62 @@ ClickHouseParser::SystemStmtContext* ClickHouseParser::systemStmt() { exitRule(); }); try { - setState(1487); + setState(1503); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 200, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 204, _ctx)) { case 1: { enterOuterAlt(_localctx, 1); - setState(1453); + setState(1469); match(ClickHouseParser::SYSTEM); - setState(1454); + setState(1470); match(ClickHouseParser::FLUSH); - setState(1455); + setState(1471); match(ClickHouseParser::DISTRIBUTED); - setState(1456); + setState(1472); tableIdentifier(); break; } case 2: { enterOuterAlt(_localctx, 2); - setState(1457); + setState(1473); match(ClickHouseParser::SYSTEM); - setState(1458); + setState(1474); match(ClickHouseParser::FLUSH); - setState(1459); + setState(1475); match(ClickHouseParser::LOGS); break; } case 3: { enterOuterAlt(_localctx, 3); - setState(1460); + setState(1476); match(ClickHouseParser::SYSTEM); - setState(1461); + setState(1477); match(ClickHouseParser::RELOAD); - setState(1462); + setState(1478); match(ClickHouseParser::DICTIONARIES); break; } case 4: { enterOuterAlt(_localctx, 4); - setState(1463); + setState(1479); match(ClickHouseParser::SYSTEM); - setState(1464); + setState(1480); match(ClickHouseParser::RELOAD); - setState(1465); + setState(1481); match(ClickHouseParser::DICTIONARY); - setState(1466); + setState(1482); tableIdentifier(); break; } case 5: { enterOuterAlt(_localctx, 5); - setState(1467); + setState(1483); match(ClickHouseParser::SYSTEM); - setState(1468); + setState(1484); _la = _input->LA(1); if (!(_la == ClickHouseParser::START @@ -11769,34 +11832,34 @@ ClickHouseParser::SystemStmtContext* ClickHouseParser::systemStmt() { _errHandler->reportMatch(this); consume(); } - setState(1476); + setState(1492); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::DISTRIBUTED: { - setState(1469); + setState(1485); match(ClickHouseParser::DISTRIBUTED); - setState(1470); + setState(1486); match(ClickHouseParser::SENDS); break; } case ClickHouseParser::FETCHES: { - setState(1471); + setState(1487); match(ClickHouseParser::FETCHES); break; } case ClickHouseParser::MERGES: case ClickHouseParser::TTL: { - setState(1473); + setState(1489); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::TTL) { - setState(1472); + setState(1488); match(ClickHouseParser::TTL); } - setState(1475); + setState(1491); match(ClickHouseParser::MERGES); break; } @@ -11804,16 +11867,16 @@ ClickHouseParser::SystemStmtContext* ClickHouseParser::systemStmt() { default: throw NoViableAltException(this); } - setState(1478); + setState(1494); tableIdentifier(); break; } case 6: { enterOuterAlt(_localctx, 6); - setState(1479); + setState(1495); match(ClickHouseParser::SYSTEM); - setState(1480); + setState(1496); _la = _input->LA(1); if (!(_la == ClickHouseParser::START @@ -11824,22 +11887,22 @@ ClickHouseParser::SystemStmtContext* ClickHouseParser::systemStmt() { _errHandler->reportMatch(this); consume(); } - setState(1481); + setState(1497); match(ClickHouseParser::REPLICATED); - setState(1482); + setState(1498); match(ClickHouseParser::SENDS); break; } case 7: { enterOuterAlt(_localctx, 7); - setState(1483); + setState(1499); match(ClickHouseParser::SYSTEM); - setState(1484); + setState(1500); match(ClickHouseParser::SYNC); - setState(1485); + setState(1501); match(ClickHouseParser::REPLICA); - setState(1486); + setState(1502); tableIdentifier(); break; } @@ -11912,51 +11975,51 @@ ClickHouseParser::TruncateStmtContext* ClickHouseParser::truncateStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1489); + setState(1505); match(ClickHouseParser::TRUNCATE); - setState(1491); + setState(1507); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 201, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 205, _ctx)) { case 1: { - setState(1490); + setState(1506); match(ClickHouseParser::TEMPORARY); break; } } - setState(1494); + setState(1510); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 202, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 206, _ctx)) { case 1: { - setState(1493); + setState(1509); match(ClickHouseParser::TABLE); break; } } - setState(1498); + setState(1514); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 203, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 207, _ctx)) { case 1: { - setState(1496); + setState(1512); match(ClickHouseParser::IF); - setState(1497); + setState(1513); match(ClickHouseParser::EXISTS); break; } } - setState(1500); + setState(1516); tableIdentifier(); - setState(1502); + setState(1518); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ON) { - setState(1501); + setState(1517); clusterClause(); } @@ -12005,9 +12068,9 @@ ClickHouseParser::UseStmtContext* ClickHouseParser::useStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1504); + setState(1520); match(ClickHouseParser::USE); - setState(1505); + setState(1521); databaseIdentifier(); } @@ -12068,26 +12131,26 @@ ClickHouseParser::WatchStmtContext* ClickHouseParser::watchStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1507); + setState(1523); match(ClickHouseParser::WATCH); - setState(1508); + setState(1524); tableIdentifier(); - setState(1510); + setState(1526); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::EVENTS) { - setState(1509); + setState(1525); match(ClickHouseParser::EVENTS); } - setState(1514); + setState(1530); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::LIMIT) { - setState(1512); + setState(1528); match(ClickHouseParser::LIMIT); - setState(1513); + setState(1529); match(ClickHouseParser::DECIMAL_LITERAL); } @@ -12283,13 +12346,13 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { exitRule(); }); try { - setState(1563); + setState(1579); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 211, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 215, _ctx)) { case 1: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(1516); + setState(1532); identifier(); break; } @@ -12297,29 +12360,29 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { case 2: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(1517); + setState(1533); identifier(); - setState(1518); + setState(1534); match(ClickHouseParser::LPAREN); - setState(1519); + setState(1535); identifier(); - setState(1520); + setState(1536); columnTypeExpr(); - setState(1527); + setState(1543); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1521); + setState(1537); match(ClickHouseParser::COMMA); - setState(1522); + setState(1538); identifier(); - setState(1523); + setState(1539); columnTypeExpr(); - setState(1529); + setState(1545); _errHandler->sync(this); _la = _input->LA(1); } - setState(1530); + setState(1546); match(ClickHouseParser::RPAREN); break; } @@ -12327,25 +12390,25 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { case 3: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 3); - setState(1532); + setState(1548); identifier(); - setState(1533); + setState(1549); match(ClickHouseParser::LPAREN); - setState(1534); + setState(1550); enumValue(); - setState(1539); + setState(1555); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1535); + setState(1551); match(ClickHouseParser::COMMA); - setState(1536); + setState(1552); enumValue(); - setState(1541); + setState(1557); _errHandler->sync(this); _la = _input->LA(1); } - setState(1542); + setState(1558); match(ClickHouseParser::RPAREN); break; } @@ -12353,25 +12416,25 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { case 4: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 4); - setState(1544); + setState(1560); identifier(); - setState(1545); + setState(1561); match(ClickHouseParser::LPAREN); - setState(1546); + setState(1562); columnTypeExpr(); - setState(1551); + setState(1567); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1547); + setState(1563); match(ClickHouseParser::COMMA); - setState(1548); + setState(1564); columnTypeExpr(); - setState(1553); + setState(1569); _errHandler->sync(this); _la = _input->LA(1); } - setState(1554); + setState(1570); match(ClickHouseParser::RPAREN); break; } @@ -12379,11 +12442,11 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { case 5: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 5); - setState(1556); + setState(1572); identifier(); - setState(1557); + setState(1573); match(ClickHouseParser::LPAREN); - setState(1559); + setState(1575); _errHandler->sync(this); _la = _input->LA(1); @@ -12581,10 +12644,10 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { | (1ULL << (ClickHouseParser::LBRACKET - 197)) | (1ULL << (ClickHouseParser::LPAREN - 197)) | (1ULL << (ClickHouseParser::PLUS - 197)))) != 0)) { - setState(1558); + setState(1574); columnExprList(); } - setState(1561); + setState(1577); match(ClickHouseParser::RPAREN); break; } @@ -12645,21 +12708,21 @@ ClickHouseParser::ColumnExprListContext* ClickHouseParser::columnExprList() { try { size_t alt; enterOuterAlt(_localctx, 1); - setState(1565); + setState(1581); columnsExpr(); - setState(1570); + setState(1586); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 212, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 216, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { - setState(1566); + setState(1582); match(ClickHouseParser::COMMA); - setState(1567); + setState(1583); columnsExpr(); } - setState(1572); + setState(1588); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 212, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 216, _ctx); } } @@ -12754,13 +12817,13 @@ ClickHouseParser::ColumnsExprContext* ClickHouseParser::columnsExpr() { exitRule(); }); try { - setState(1584); + setState(1600); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 214, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 218, _ctx)) { case 1: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(1576); + setState(1592); _errHandler->sync(this); _la = _input->LA(1); @@ -12944,12 +13007,12 @@ ClickHouseParser::ColumnsExprContext* ClickHouseParser::columnsExpr() { | (1ULL << (ClickHouseParser::JSON_FALSE - 128)) | (1ULL << (ClickHouseParser::JSON_TRUE - 128)) | (1ULL << (ClickHouseParser::IDENTIFIER - 128)))) != 0)) { - setState(1573); + setState(1589); tableIdentifier(); - setState(1574); + setState(1590); match(ClickHouseParser::DOT); } - setState(1578); + setState(1594); match(ClickHouseParser::ASTERISK); break; } @@ -12957,11 +13020,11 @@ ClickHouseParser::ColumnsExprContext* ClickHouseParser::columnsExpr() { case 2: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(1579); + setState(1595); match(ClickHouseParser::LPAREN); - setState(1580); + setState(1596); selectUnionStmt(); - setState(1581); + setState(1597); match(ClickHouseParser::RPAREN); break; } @@ -12969,7 +13032,7 @@ ClickHouseParser::ColumnsExprContext* ClickHouseParser::columnsExpr() { case 3: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 3); - setState(1583); + setState(1599); columnExpr(0); break; } @@ -13821,54 +13884,54 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence try { size_t alt; enterOuterAlt(_localctx, 1); - setState(1693); + setState(1709); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 225, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 229, _ctx)) { case 1: { _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1587); + setState(1603); match(ClickHouseParser::CASE); - setState(1589); + setState(1605); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 215, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 219, _ctx)) { case 1: { - setState(1588); + setState(1604); columnExpr(0); break; } } - setState(1596); + setState(1612); _errHandler->sync(this); _la = _input->LA(1); do { - setState(1591); + setState(1607); match(ClickHouseParser::WHEN); - setState(1592); + setState(1608); columnExpr(0); - setState(1593); + setState(1609); match(ClickHouseParser::THEN); - setState(1594); + setState(1610); columnExpr(0); - setState(1598); + setState(1614); _errHandler->sync(this); _la = _input->LA(1); } while (_la == ClickHouseParser::WHEN); - setState(1602); + setState(1618); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ELSE) { - setState(1600); + setState(1616); match(ClickHouseParser::ELSE); - setState(1601); + setState(1617); columnExpr(0); } - setState(1604); + setState(1620); match(ClickHouseParser::END); break; } @@ -13877,17 +13940,17 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1606); + setState(1622); match(ClickHouseParser::CAST); - setState(1607); + setState(1623); match(ClickHouseParser::LPAREN); - setState(1608); + setState(1624); columnExpr(0); - setState(1609); + setState(1625); match(ClickHouseParser::AS); - setState(1610); + setState(1626); columnTypeExpr(); - setState(1611); + setState(1627); match(ClickHouseParser::RPAREN); break; } @@ -13896,9 +13959,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1613); + setState(1629); match(ClickHouseParser::DATE); - setState(1614); + setState(1630); match(ClickHouseParser::STRING_LITERAL); break; } @@ -13907,17 +13970,17 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1615); + setState(1631); match(ClickHouseParser::EXTRACT); - setState(1616); + setState(1632); match(ClickHouseParser::LPAREN); - setState(1617); + setState(1633); interval(); - setState(1618); + setState(1634); match(ClickHouseParser::FROM); - setState(1619); + setState(1635); columnExpr(0); - setState(1620); + setState(1636); match(ClickHouseParser::RPAREN); break; } @@ -13926,11 +13989,11 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1622); + setState(1638); match(ClickHouseParser::INTERVAL); - setState(1623); + setState(1639); columnExpr(0); - setState(1624); + setState(1640); interval(); break; } @@ -13939,27 +14002,27 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1626); + setState(1642); match(ClickHouseParser::SUBSTRING); - setState(1627); + setState(1643); match(ClickHouseParser::LPAREN); - setState(1628); + setState(1644); columnExpr(0); - setState(1629); + setState(1645); match(ClickHouseParser::FROM); - setState(1630); + setState(1646); columnExpr(0); - setState(1633); + setState(1649); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::FOR) { - setState(1631); + setState(1647); match(ClickHouseParser::FOR); - setState(1632); + setState(1648); columnExpr(0); } - setState(1635); + setState(1651); match(ClickHouseParser::RPAREN); break; } @@ -13968,9 +14031,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1637); + setState(1653); match(ClickHouseParser::TIMESTAMP); - setState(1638); + setState(1654); match(ClickHouseParser::STRING_LITERAL); break; } @@ -13979,11 +14042,11 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1639); + setState(1655); match(ClickHouseParser::TRIM); - setState(1640); + setState(1656); match(ClickHouseParser::LPAREN); - setState(1641); + setState(1657); _la = _input->LA(1); if (!(_la == ClickHouseParser::BOTH || _la == ClickHouseParser::LEADING || _la == ClickHouseParser::TRAILING)) { _errHandler->recoverInline(this); @@ -13992,13 +14055,13 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _errHandler->reportMatch(this); consume(); } - setState(1642); + setState(1658); match(ClickHouseParser::STRING_LITERAL); - setState(1643); + setState(1659); match(ClickHouseParser::FROM); - setState(1644); + setState(1660); columnExpr(0); - setState(1645); + setState(1661); match(ClickHouseParser::RPAREN); break; } @@ -14007,16 +14070,16 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1647); + setState(1663); identifier(); - setState(1653); + setState(1669); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 220, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 224, _ctx)) { case 1: { - setState(1648); + setState(1664); match(ClickHouseParser::LPAREN); - setState(1650); + setState(1666); _errHandler->sync(this); _la = _input->LA(1); @@ -14214,29 +14277,29 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::LBRACKET - 197)) | (1ULL << (ClickHouseParser::LPAREN - 197)) | (1ULL << (ClickHouseParser::PLUS - 197)))) != 0)) { - setState(1649); + setState(1665); columnExprList(); } - setState(1652); + setState(1668); match(ClickHouseParser::RPAREN); break; } } - setState(1655); + setState(1671); match(ClickHouseParser::LPAREN); - setState(1657); + setState(1673); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 221, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 225, _ctx)) { case 1: { - setState(1656); + setState(1672); match(ClickHouseParser::DISTINCT); break; } } - setState(1660); + setState(1676); _errHandler->sync(this); _la = _input->LA(1); @@ -14434,10 +14497,10 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::LBRACKET - 197)) | (1ULL << (ClickHouseParser::LPAREN - 197)) | (1ULL << (ClickHouseParser::PLUS - 197)))) != 0)) { - setState(1659); + setState(1675); columnArgList(); } - setState(1662); + setState(1678); match(ClickHouseParser::RPAREN); break; } @@ -14446,7 +14509,7 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1664); + setState(1680); literal(); break; } @@ -14455,9 +14518,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1665); + setState(1681); match(ClickHouseParser::DASH); - setState(1666); + setState(1682); columnExpr(17); break; } @@ -14466,9 +14529,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1667); + setState(1683); match(ClickHouseParser::NOT); - setState(1668); + setState(1684); columnExpr(12); break; } @@ -14477,7 +14540,7 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1672); + setState(1688); _errHandler->sync(this); _la = _input->LA(1); @@ -14661,12 +14724,12 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::JSON_FALSE - 128)) | (1ULL << (ClickHouseParser::JSON_TRUE - 128)) | (1ULL << (ClickHouseParser::IDENTIFIER - 128)))) != 0)) { - setState(1669); + setState(1685); tableIdentifier(); - setState(1670); + setState(1686); match(ClickHouseParser::DOT); } - setState(1674); + setState(1690); match(ClickHouseParser::ASTERISK); break; } @@ -14675,11 +14738,11 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1675); + setState(1691); match(ClickHouseParser::LPAREN); - setState(1676); + setState(1692); selectUnionStmt(); - setState(1677); + setState(1693); match(ClickHouseParser::RPAREN); break; } @@ -14688,11 +14751,11 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1679); + setState(1695); match(ClickHouseParser::LPAREN); - setState(1680); + setState(1696); columnExpr(0); - setState(1681); + setState(1697); match(ClickHouseParser::RPAREN); break; } @@ -14701,11 +14764,11 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1683); + setState(1699); match(ClickHouseParser::LPAREN); - setState(1684); + setState(1700); columnExprList(); - setState(1685); + setState(1701); match(ClickHouseParser::RPAREN); break; } @@ -14714,9 +14777,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1687); + setState(1703); match(ClickHouseParser::LBRACKET); - setState(1689); + setState(1705); _errHandler->sync(this); _la = _input->LA(1); @@ -14914,10 +14977,10 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::LBRACKET - 197)) | (1ULL << (ClickHouseParser::LPAREN - 197)) | (1ULL << (ClickHouseParser::PLUS - 197)))) != 0)) { - setState(1688); + setState(1704); columnExprList(); } - setState(1691); + setState(1707); match(ClickHouseParser::RBRACKET); break; } @@ -14926,32 +14989,32 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1692); + setState(1708); columnIdentifier(); break; } } _ctx->stop = _input->LT(-1); - setState(1766); + setState(1782); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 234, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 238, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { if (!_parseListeners.empty()) triggerExitRuleEvent(); previousContext = _localctx; - setState(1764); + setState(1780); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 233, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 237, _ctx)) { case 1: { auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1695); + setState(1711); if (!(precpred(_ctx, 16))) throw FailedPredicateException(this, "precpred(_ctx, 16)"); - setState(1696); + setState(1712); _la = _input->LA(1); if (!(((((_la - 191) & ~ 0x3fULL) == 0) && ((1ULL << (_la - 191)) & ((1ULL << (ClickHouseParser::ASTERISK - 191)) @@ -14963,7 +15026,7 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _errHandler->reportMatch(this); consume(); } - setState(1697); + setState(1713); columnExpr(17); break; } @@ -14972,10 +15035,10 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1698); + setState(1714); if (!(precpred(_ctx, 15))) throw FailedPredicateException(this, "precpred(_ctx, 15)"); - setState(1699); + setState(1715); _la = _input->LA(1); if (!(((((_la - 196) & ~ 0x3fULL) == 0) && ((1ULL << (_la - 196)) & ((1ULL << (ClickHouseParser::CONCAT - 196)) @@ -14987,7 +15050,7 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _errHandler->reportMatch(this); consume(); } - setState(1700); + setState(1716); columnExpr(16); break; } @@ -14996,86 +15059,86 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1701); + setState(1717); if (!(precpred(_ctx, 14))) throw FailedPredicateException(this, "precpred(_ctx, 14)"); - setState(1720); + setState(1736); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 229, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 233, _ctx)) { case 1: { - setState(1702); + setState(1718); match(ClickHouseParser::EQ_DOUBLE); break; } case 2: { - setState(1703); + setState(1719); match(ClickHouseParser::EQ_SINGLE); break; } case 3: { - setState(1704); + setState(1720); match(ClickHouseParser::NOT_EQ); break; } case 4: { - setState(1705); + setState(1721); match(ClickHouseParser::LE); break; } case 5: { - setState(1706); + setState(1722); match(ClickHouseParser::GE); break; } case 6: { - setState(1707); + setState(1723); match(ClickHouseParser::LT); break; } case 7: { - setState(1708); + setState(1724); match(ClickHouseParser::GT); break; } case 8: { - setState(1710); + setState(1726); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::GLOBAL) { - setState(1709); + setState(1725); match(ClickHouseParser::GLOBAL); } - setState(1713); + setState(1729); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::NOT) { - setState(1712); + setState(1728); match(ClickHouseParser::NOT); } - setState(1715); + setState(1731); match(ClickHouseParser::IN); break; } case 9: { - setState(1717); + setState(1733); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::NOT) { - setState(1716); + setState(1732); match(ClickHouseParser::NOT); } - setState(1719); + setState(1735); _la = _input->LA(1); if (!(_la == ClickHouseParser::ILIKE @@ -15090,7 +15153,7 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence } } - setState(1722); + setState(1738); columnExpr(15); break; } @@ -15099,12 +15162,12 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1723); + setState(1739); if (!(precpred(_ctx, 11))) throw FailedPredicateException(this, "precpred(_ctx, 11)"); - setState(1724); + setState(1740); match(ClickHouseParser::AND); - setState(1725); + setState(1741); columnExpr(12); break; } @@ -15113,12 +15176,12 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1726); + setState(1742); if (!(precpred(_ctx, 10))) throw FailedPredicateException(this, "precpred(_ctx, 10)"); - setState(1727); + setState(1743); match(ClickHouseParser::OR); - setState(1728); + setState(1744); columnExpr(11); break; } @@ -15127,24 +15190,24 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1729); + setState(1745); if (!(precpred(_ctx, 9))) throw FailedPredicateException(this, "precpred(_ctx, 9)"); - setState(1731); + setState(1747); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::NOT) { - setState(1730); + setState(1746); match(ClickHouseParser::NOT); } - setState(1733); + setState(1749); match(ClickHouseParser::BETWEEN); - setState(1734); + setState(1750); columnExpr(0); - setState(1735); + setState(1751); match(ClickHouseParser::AND); - setState(1736); + setState(1752); columnExpr(10); break; } @@ -15153,16 +15216,16 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1738); + setState(1754); if (!(precpred(_ctx, 8))) throw FailedPredicateException(this, "precpred(_ctx, 8)"); - setState(1739); + setState(1755); match(ClickHouseParser::QUERY); - setState(1740); + setState(1756); columnExpr(0); - setState(1741); + setState(1757); match(ClickHouseParser::COLON); - setState(1742); + setState(1758); columnExpr(8); break; } @@ -15171,14 +15234,14 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1744); + setState(1760); if (!(precpred(_ctx, 19))) throw FailedPredicateException(this, "precpred(_ctx, 19)"); - setState(1745); + setState(1761); match(ClickHouseParser::LBRACKET); - setState(1746); + setState(1762); columnExpr(0); - setState(1747); + setState(1763); match(ClickHouseParser::RBRACKET); break; } @@ -15187,12 +15250,12 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1749); + setState(1765); if (!(precpred(_ctx, 18))) throw FailedPredicateException(this, "precpred(_ctx, 18)"); - setState(1750); + setState(1766); match(ClickHouseParser::DOT); - setState(1751); + setState(1767); match(ClickHouseParser::DECIMAL_LITERAL); break; } @@ -15201,20 +15264,20 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1752); + setState(1768); if (!(precpred(_ctx, 13))) throw FailedPredicateException(this, "precpred(_ctx, 13)"); - setState(1753); + setState(1769); match(ClickHouseParser::IS); - setState(1755); + setState(1771); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::NOT) { - setState(1754); + setState(1770); match(ClickHouseParser::NOT); } - setState(1757); + setState(1773); match(ClickHouseParser::NULL_SQL); break; } @@ -15223,10 +15286,10 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1758); + setState(1774); if (!(precpred(_ctx, 7))) throw FailedPredicateException(this, "precpred(_ctx, 7)"); - setState(1762); + setState(1778); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::DATE: @@ -15234,15 +15297,15 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence case ClickHouseParser::ID: case ClickHouseParser::KEY: case ClickHouseParser::IDENTIFIER: { - setState(1759); + setState(1775); alias(); break; } case ClickHouseParser::AS: { - setState(1760); + setState(1776); match(ClickHouseParser::AS); - setState(1761); + setState(1777); identifier(); break; } @@ -15255,9 +15318,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence } } - setState(1768); + setState(1784); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 234, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 238, _ctx); } } catch (RecognitionException &e) { @@ -15312,17 +15375,17 @@ ClickHouseParser::ColumnArgListContext* ClickHouseParser::columnArgList() { }); try { enterOuterAlt(_localctx, 1); - setState(1769); + setState(1785); columnArgExpr(); - setState(1774); + setState(1790); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1770); + setState(1786); match(ClickHouseParser::COMMA); - setState(1771); + setState(1787); columnArgExpr(); - setState(1776); + setState(1792); _errHandler->sync(this); _la = _input->LA(1); } @@ -15371,19 +15434,19 @@ ClickHouseParser::ColumnArgExprContext* ClickHouseParser::columnArgExpr() { exitRule(); }); try { - setState(1779); + setState(1795); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 236, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 240, _ctx)) { case 1: { enterOuterAlt(_localctx, 1); - setState(1777); + setState(1793); columnLambdaExpr(); break; } case 2: { enterOuterAlt(_localctx, 2); - setState(1778); + setState(1794); columnExpr(0); break; } @@ -15460,27 +15523,27 @@ ClickHouseParser::ColumnLambdaExprContext* ClickHouseParser::columnLambdaExpr() }); try { enterOuterAlt(_localctx, 1); - setState(1800); + setState(1816); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::LPAREN: { - setState(1781); + setState(1797); match(ClickHouseParser::LPAREN); - setState(1782); + setState(1798); identifier(); - setState(1787); + setState(1803); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1783); + setState(1799); match(ClickHouseParser::COMMA); - setState(1784); + setState(1800); identifier(); - setState(1789); + setState(1805); _errHandler->sync(this); _la = _input->LA(1); } - setState(1790); + setState(1806); match(ClickHouseParser::RPAREN); break; } @@ -15664,17 +15727,17 @@ ClickHouseParser::ColumnLambdaExprContext* ClickHouseParser::columnLambdaExpr() case ClickHouseParser::JSON_FALSE: case ClickHouseParser::JSON_TRUE: case ClickHouseParser::IDENTIFIER: { - setState(1792); + setState(1808); identifier(); - setState(1797); + setState(1813); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1793); + setState(1809); match(ClickHouseParser::COMMA); - setState(1794); + setState(1810); identifier(); - setState(1799); + setState(1815); _errHandler->sync(this); _la = _input->LA(1); } @@ -15684,9 +15747,9 @@ ClickHouseParser::ColumnLambdaExprContext* ClickHouseParser::columnLambdaExpr() default: throw NoViableAltException(this); } - setState(1802); + setState(1818); match(ClickHouseParser::ARROW); - setState(1803); + setState(1819); columnExpr(0); } @@ -15738,20 +15801,20 @@ ClickHouseParser::ColumnIdentifierContext* ClickHouseParser::columnIdentifier() }); try { enterOuterAlt(_localctx, 1); - setState(1808); + setState(1824); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 240, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 244, _ctx)) { case 1: { - setState(1805); + setState(1821); tableIdentifier(); - setState(1806); + setState(1822); match(ClickHouseParser::DOT); break; } } - setState(1810); + setState(1826); nestedIdentifier(); } @@ -15803,16 +15866,16 @@ ClickHouseParser::NestedIdentifierContext* ClickHouseParser::nestedIdentifier() }); try { enterOuterAlt(_localctx, 1); - setState(1812); + setState(1828); identifier(); - setState(1815); + setState(1831); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 241, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 245, _ctx)) { case 1: { - setState(1813); + setState(1829); match(ClickHouseParser::DOT); - setState(1814); + setState(1830); identifier(); break; } @@ -15942,15 +16005,15 @@ ClickHouseParser::TableExprContext* ClickHouseParser::tableExpr(int precedence) try { size_t alt; enterOuterAlt(_localctx, 1); - setState(1824); + setState(1840); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 242, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 246, _ctx)) { case 1: { _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1818); + setState(1834); tableIdentifier(); break; } @@ -15959,7 +16022,7 @@ ClickHouseParser::TableExprContext* ClickHouseParser::tableExpr(int precedence) _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1819); + setState(1835); tableFunctionExpr(); break; } @@ -15968,20 +16031,20 @@ ClickHouseParser::TableExprContext* ClickHouseParser::tableExpr(int precedence) _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1820); + setState(1836); match(ClickHouseParser::LPAREN); - setState(1821); + setState(1837); selectUnionStmt(); - setState(1822); + setState(1838); match(ClickHouseParser::RPAREN); break; } } _ctx->stop = _input->LT(-1); - setState(1834); + setState(1850); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 244, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 248, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { if (!_parseListeners.empty()) @@ -15990,10 +16053,10 @@ ClickHouseParser::TableExprContext* ClickHouseParser::tableExpr(int precedence) auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleTableExpr); - setState(1826); + setState(1842); if (!(precpred(_ctx, 1))) throw FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(1830); + setState(1846); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::DATE: @@ -16001,15 +16064,15 @@ ClickHouseParser::TableExprContext* ClickHouseParser::tableExpr(int precedence) case ClickHouseParser::ID: case ClickHouseParser::KEY: case ClickHouseParser::IDENTIFIER: { - setState(1827); + setState(1843); alias(); break; } case ClickHouseParser::AS: { - setState(1828); + setState(1844); match(ClickHouseParser::AS); - setState(1829); + setState(1845); identifier(); break; } @@ -16018,9 +16081,9 @@ ClickHouseParser::TableExprContext* ClickHouseParser::tableExpr(int precedence) throw NoViableAltException(this); } } - setState(1836); + setState(1852); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 244, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 248, _ctx); } } catch (RecognitionException &e) { @@ -16075,11 +16138,11 @@ ClickHouseParser::TableFunctionExprContext* ClickHouseParser::tableFunctionExpr( }); try { enterOuterAlt(_localctx, 1); - setState(1837); + setState(1853); identifier(); - setState(1838); + setState(1854); match(ClickHouseParser::LPAREN); - setState(1840); + setState(1856); _errHandler->sync(this); _la = _input->LA(1); @@ -16274,10 +16337,10 @@ ClickHouseParser::TableFunctionExprContext* ClickHouseParser::tableFunctionExpr( ((1ULL << (_la - 197)) & ((1ULL << (ClickHouseParser::DASH - 197)) | (1ULL << (ClickHouseParser::DOT - 197)) | (1ULL << (ClickHouseParser::PLUS - 197)))) != 0)) { - setState(1839); + setState(1855); tableArgList(); } - setState(1842); + setState(1858); match(ClickHouseParser::RPAREN); } @@ -16329,20 +16392,20 @@ ClickHouseParser::TableIdentifierContext* ClickHouseParser::tableIdentifier() { }); try { enterOuterAlt(_localctx, 1); - setState(1847); + setState(1863); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 246, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 250, _ctx)) { case 1: { - setState(1844); + setState(1860); databaseIdentifier(); - setState(1845); + setState(1861); match(ClickHouseParser::DOT); break; } } - setState(1849); + setState(1865); identifier(); } @@ -16399,17 +16462,17 @@ ClickHouseParser::TableArgListContext* ClickHouseParser::tableArgList() { }); try { enterOuterAlt(_localctx, 1); - setState(1851); + setState(1867); tableArgExpr(); - setState(1856); + setState(1872); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1852); + setState(1868); match(ClickHouseParser::COMMA); - setState(1853); + setState(1869); tableArgExpr(); - setState(1858); + setState(1874); _errHandler->sync(this); _la = _input->LA(1); } @@ -16462,26 +16525,26 @@ ClickHouseParser::TableArgExprContext* ClickHouseParser::tableArgExpr() { exitRule(); }); try { - setState(1862); + setState(1878); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 248, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 252, _ctx)) { case 1: { enterOuterAlt(_localctx, 1); - setState(1859); + setState(1875); nestedIdentifier(); break; } case 2: { enterOuterAlt(_localctx, 2); - setState(1860); + setState(1876); tableFunctionExpr(); break; } case 3: { enterOuterAlt(_localctx, 3); - setState(1861); + setState(1877); literal(); break; } @@ -16529,7 +16592,7 @@ ClickHouseParser::DatabaseIdentifierContext* ClickHouseParser::databaseIdentifie }); try { enterOuterAlt(_localctx, 1); - setState(1864); + setState(1880); identifier(); } @@ -16589,21 +16652,21 @@ ClickHouseParser::FloatingLiteralContext* ClickHouseParser::floatingLiteral() { exitRule(); }); try { - setState(1874); + setState(1890); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::FLOATING_LITERAL: { enterOuterAlt(_localctx, 1); - setState(1866); + setState(1882); match(ClickHouseParser::FLOATING_LITERAL); break; } case ClickHouseParser::DOT: { enterOuterAlt(_localctx, 2); - setState(1867); + setState(1883); match(ClickHouseParser::DOT); - setState(1868); + setState(1884); _la = _input->LA(1); if (!(_la == ClickHouseParser::OCTAL_LITERAL @@ -16619,16 +16682,16 @@ ClickHouseParser::FloatingLiteralContext* ClickHouseParser::floatingLiteral() { case ClickHouseParser::DECIMAL_LITERAL: { enterOuterAlt(_localctx, 3); - setState(1869); + setState(1885); match(ClickHouseParser::DECIMAL_LITERAL); - setState(1870); + setState(1886); match(ClickHouseParser::DOT); - setState(1872); + setState(1888); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 249, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 253, _ctx)) { case 1: { - setState(1871); + setState(1887); _la = _input->LA(1); if (!(_la == ClickHouseParser::OCTAL_LITERAL @@ -16720,14 +16783,14 @@ ClickHouseParser::NumberLiteralContext* ClickHouseParser::numberLiteral() { }); try { enterOuterAlt(_localctx, 1); - setState(1877); + setState(1893); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::DASH || _la == ClickHouseParser::PLUS) { - setState(1876); + setState(1892); _la = _input->LA(1); if (!(_la == ClickHouseParser::DASH @@ -16739,41 +16802,41 @@ ClickHouseParser::NumberLiteralContext* ClickHouseParser::numberLiteral() { consume(); } } - setState(1885); + setState(1901); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 252, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 256, _ctx)) { case 1: { - setState(1879); + setState(1895); floatingLiteral(); break; } case 2: { - setState(1880); + setState(1896); match(ClickHouseParser::OCTAL_LITERAL); break; } case 3: { - setState(1881); + setState(1897); match(ClickHouseParser::DECIMAL_LITERAL); break; } case 4: { - setState(1882); + setState(1898); match(ClickHouseParser::HEXADECIMAL_LITERAL); break; } case 5: { - setState(1883); + setState(1899); match(ClickHouseParser::INF); break; } case 6: { - setState(1884); + setState(1900); match(ClickHouseParser::NAN_SQL); break; } @@ -16828,7 +16891,7 @@ ClickHouseParser::LiteralContext* ClickHouseParser::literal() { exitRule(); }); try { - setState(1890); + setState(1906); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::INF: @@ -16841,21 +16904,21 @@ ClickHouseParser::LiteralContext* ClickHouseParser::literal() { case ClickHouseParser::DOT: case ClickHouseParser::PLUS: { enterOuterAlt(_localctx, 1); - setState(1887); + setState(1903); numberLiteral(); break; } case ClickHouseParser::STRING_LITERAL: { enterOuterAlt(_localctx, 2); - setState(1888); + setState(1904); match(ClickHouseParser::STRING_LITERAL); break; } case ClickHouseParser::NULL_SQL: { enterOuterAlt(_localctx, 3); - setState(1889); + setState(1905); match(ClickHouseParser::NULL_SQL); break; } @@ -16934,7 +16997,7 @@ ClickHouseParser::IntervalContext* ClickHouseParser::interval() { }); try { enterOuterAlt(_localctx, 1); - setState(1892); + setState(1908); _la = _input->LA(1); if (!(_la == ClickHouseParser::DAY || ((((_la - 73) & ~ 0x3fULL) == 0) && ((1ULL << (_la - 73)) & ((1ULL << (ClickHouseParser::HOUR - 73)) @@ -17669,7 +17732,7 @@ ClickHouseParser::KeywordContext* ClickHouseParser::keyword() { }); try { enterOuterAlt(_localctx, 1); - setState(1894); + setState(1910); _la = _input->LA(1); if (!((((_la & ~ 0x3fULL) == 0) && ((1ULL << _la) & ((1ULL << ClickHouseParser::AFTER) @@ -17903,7 +17966,7 @@ ClickHouseParser::KeywordForAliasContext* ClickHouseParser::keywordForAlias() { }); try { enterOuterAlt(_localctx, 1); - setState(1896); + setState(1912); _la = _input->LA(1); if (!(((((_la - 34) & ~ 0x3fULL) == 0) && ((1ULL << (_la - 34)) & ((1ULL << (ClickHouseParser::DATE - 34)) @@ -17961,12 +18024,12 @@ ClickHouseParser::AliasContext* ClickHouseParser::alias() { exitRule(); }); try { - setState(1900); + setState(1916); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::IDENTIFIER: { enterOuterAlt(_localctx, 1); - setState(1898); + setState(1914); match(ClickHouseParser::IDENTIFIER); break; } @@ -17976,7 +18039,7 @@ ClickHouseParser::AliasContext* ClickHouseParser::alias() { case ClickHouseParser::ID: case ClickHouseParser::KEY: { enterOuterAlt(_localctx, 2); - setState(1899); + setState(1915); keywordForAlias(); break; } @@ -18033,12 +18096,12 @@ ClickHouseParser::IdentifierContext* ClickHouseParser::identifier() { exitRule(); }); try { - setState(1905); + setState(1921); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::IDENTIFIER: { enterOuterAlt(_localctx, 1); - setState(1902); + setState(1918); match(ClickHouseParser::IDENTIFIER); break; } @@ -18052,7 +18115,7 @@ ClickHouseParser::IdentifierContext* ClickHouseParser::identifier() { case ClickHouseParser::WEEK: case ClickHouseParser::YEAR: { enterOuterAlt(_localctx, 2); - setState(1903); + setState(1919); interval(); break; } @@ -18228,7 +18291,7 @@ ClickHouseParser::IdentifierContext* ClickHouseParser::identifier() { case ClickHouseParser::JSON_FALSE: case ClickHouseParser::JSON_TRUE: { enterOuterAlt(_localctx, 3); - setState(1904); + setState(1920); keyword(); break; } @@ -18281,7 +18344,7 @@ ClickHouseParser::IdentifierOrNullContext* ClickHouseParser::identifierOrNull() exitRule(); }); try { - setState(1909); + setState(1925); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::AFTER: @@ -18464,14 +18527,14 @@ ClickHouseParser::IdentifierOrNullContext* ClickHouseParser::identifierOrNull() case ClickHouseParser::JSON_TRUE: case ClickHouseParser::IDENTIFIER: { enterOuterAlt(_localctx, 1); - setState(1907); + setState(1923); identifier(); break; } case ClickHouseParser::NULL_SQL: { enterOuterAlt(_localctx, 2); - setState(1908); + setState(1924); match(ClickHouseParser::NULL_SQL); break; } @@ -18529,11 +18592,11 @@ ClickHouseParser::EnumValueContext* ClickHouseParser::enumValue() { }); try { enterOuterAlt(_localctx, 1); - setState(1911); + setState(1927); match(ClickHouseParser::STRING_LITERAL); - setState(1912); + setState(1928); match(ClickHouseParser::EQ_SINGLE); - setState(1913); + setState(1929); numberLiteral(); } @@ -18751,7 +18814,7 @@ ClickHouseParser::Initializer::Initializer() { _serializedATN = { 0x3, 0x608b, 0xa72a, 0x8133, 0xb9ed, 0x417c, 0x3be7, 0x7786, 0x5964, - 0x3, 0xe0, 0x77e, 0x4, 0x2, 0x9, 0x2, 0x4, 0x3, 0x9, 0x3, 0x4, 0x4, + 0x3, 0xe0, 0x78e, 0x4, 0x2, 0x9, 0x2, 0x4, 0x3, 0x9, 0x3, 0x4, 0x4, 0x9, 0x4, 0x4, 0x5, 0x9, 0x5, 0x4, 0x6, 0x9, 0x6, 0x4, 0x7, 0x9, 0x7, 0x4, 0x8, 0x9, 0x8, 0x4, 0x9, 0x9, 0x9, 0x4, 0xa, 0x9, 0xa, 0x4, 0xb, 0x9, 0xb, 0x4, 0xc, 0x9, 0xc, 0x4, 0xd, 0x9, 0xd, 0x4, 0xe, 0x9, 0xe, @@ -18833,239 +18896,242 @@ ClickHouseParser::Initializer::Initializer() { 0xb, 0x3, 0xb, 0x5, 0xb, 0x206, 0xa, 0xb, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x20d, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x211, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x214, 0xa, 0xc, 0x3, 0xc, - 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x21b, 0xa, 0xc, 0x3, - 0xc, 0x3, 0xc, 0x5, 0xc, 0x21f, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x222, - 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, - 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x22d, 0xa, 0xc, 0x3, 0xc, 0x3, - 0xc, 0x5, 0xc, 0x231, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x234, 0xa, 0xc, - 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x239, 0xa, 0xc, 0x5, 0xc, 0x23b, - 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x23e, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x241, - 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, - 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x24b, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, - 0xc, 0x24f, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x252, 0xa, 0xc, 0x3, 0xc, - 0x5, 0xc, 0x255, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x25a, - 0xa, 0xc, 0x5, 0xc, 0x25c, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, - 0xc, 0x5, 0xc, 0x262, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, - 0x5, 0xc, 0x268, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x26c, 0xa, + 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x21a, 0xa, 0xc, 0x3, 0xc, 0x5, + 0xc, 0x21d, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, + 0x223, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x227, 0xa, 0xc, 0x3, + 0xc, 0x5, 0xc, 0x22a, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, + 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x235, 0xa, + 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x239, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, + 0x23c, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x241, 0xa, + 0xc, 0x5, 0xc, 0x243, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x246, 0xa, 0xc, + 0x3, 0xc, 0x5, 0xc, 0x249, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, + 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x253, 0xa, 0xc, + 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x257, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x25a, + 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x25d, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, + 0xc, 0x5, 0xc, 0x262, 0xa, 0xc, 0x5, 0xc, 0x264, 0xa, 0xc, 0x3, 0xc, + 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x26c, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x26f, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x272, - 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x275, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x278, - 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x27d, 0xa, 0xc, 0x3, - 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x283, 0xa, 0xc, 0x3, 0xc, - 0x3, 0xc, 0x5, 0xc, 0x287, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x28a, 0xa, - 0xc, 0x3, 0xc, 0x5, 0xc, 0x28d, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, - 0x291, 0xa, 0xc, 0x3, 0xd, 0x3, 0xd, 0x3, 0xd, 0x3, 0xd, 0x7, 0xd, 0x297, - 0xa, 0xd, 0xc, 0xd, 0xe, 0xd, 0x29a, 0xb, 0xd, 0x3, 0xd, 0x3, 0xd, 0x3, - 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, - 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, - 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x7, - 0xe, 0x2b3, 0xa, 0xe, 0xc, 0xe, 0xe, 0xe, 0x2b6, 0xb, 0xe, 0x3, 0xf, - 0x5, 0xf, 0x2b9, 0xa, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, - 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, - 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, - 0xf, 0x3, 0xf, 0x7, 0xf, 0x2cf, 0xa, 0xf, 0xc, 0xf, 0xe, 0xf, 0x2d2, - 0xb, 0xf, 0x3, 0x10, 0x3, 0x10, 0x3, 0x10, 0x3, 0x10, 0x3, 0x11, 0x3, - 0x11, 0x3, 0x11, 0x3, 0x11, 0x5, 0x11, 0x2dc, 0xa, 0x11, 0x3, 0x11, - 0x5, 0x11, 0x2df, 0xa, 0x11, 0x3, 0x12, 0x3, 0x12, 0x3, 0x12, 0x3, 0x12, - 0x3, 0x12, 0x7, 0x12, 0x2e6, 0xa, 0x12, 0xc, 0x12, 0xe, 0x12, 0x2e9, - 0xb, 0x12, 0x3, 0x12, 0x3, 0x12, 0x3, 0x12, 0x3, 0x13, 0x3, 0x13, 0x3, - 0x13, 0x3, 0x13, 0x3, 0x13, 0x3, 0x13, 0x3, 0x13, 0x3, 0x13, 0x3, 0x13, - 0x3, 0x13, 0x3, 0x13, 0x5, 0x13, 0x2f9, 0xa, 0x13, 0x3, 0x13, 0x3, 0x13, - 0x3, 0x14, 0x3, 0x14, 0x3, 0x14, 0x3, 0x14, 0x3, 0x14, 0x7, 0x14, 0x302, - 0xa, 0x14, 0xc, 0x14, 0xe, 0x14, 0x305, 0xb, 0x14, 0x3, 0x14, 0x3, 0x14, - 0x3, 0x14, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, - 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, - 0x5, 0x15, 0x316, 0xa, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x16, 0x3, 0x16, - 0x3, 0x16, 0x3, 0x16, 0x3, 0x16, 0x3, 0x17, 0x3, 0x17, 0x3, 0x17, 0x3, - 0x17, 0x5, 0x17, 0x323, 0xa, 0x17, 0x3, 0x18, 0x3, 0x18, 0x3, 0x18, - 0x3, 0x19, 0x3, 0x19, 0x3, 0x19, 0x3, 0x1a, 0x3, 0x1a, 0x3, 0x1a, 0x3, - 0x1b, 0x3, 0x1b, 0x3, 0x1b, 0x3, 0x1b, 0x7, 0x1b, 0x332, 0xa, 0x1b, - 0xc, 0x1b, 0xe, 0x1b, 0x335, 0xb, 0x1b, 0x3, 0x1b, 0x3, 0x1b, 0x3, 0x1b, - 0x3, 0x1b, 0x3, 0x1b, 0x3, 0x1b, 0x5, 0x1b, 0x33d, 0xa, 0x1b, 0x3, 0x1c, + 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x278, 0xa, + 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x27c, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, + 0x27f, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x282, 0xa, 0xc, 0x3, 0xc, 0x5, + 0xc, 0x285, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x288, 0xa, 0xc, 0x3, 0xc, + 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x28d, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, + 0xc, 0x3, 0xc, 0x5, 0xc, 0x293, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, + 0x297, 0xa, 0xc, 0x3, 0xc, 0x5, 0xc, 0x29a, 0xa, 0xc, 0x3, 0xc, 0x5, + 0xc, 0x29d, 0xa, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x2a1, 0xa, 0xc, + 0x3, 0xd, 0x3, 0xd, 0x3, 0xd, 0x3, 0xd, 0x7, 0xd, 0x2a7, 0xa, 0xd, 0xc, + 0xd, 0xe, 0xd, 0x2aa, 0xb, 0xd, 0x3, 0xd, 0x3, 0xd, 0x3, 0xe, 0x3, 0xe, + 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, + 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, + 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x7, 0xe, 0x2c3, 0xa, + 0xe, 0xc, 0xe, 0xe, 0xe, 0x2c6, 0xb, 0xe, 0x3, 0xf, 0x5, 0xf, 0x2c9, + 0xa, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, + 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, + 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, + 0x7, 0xf, 0x2df, 0xa, 0xf, 0xc, 0xf, 0xe, 0xf, 0x2e2, 0xb, 0xf, 0x3, + 0x10, 0x3, 0x10, 0x3, 0x10, 0x3, 0x10, 0x3, 0x11, 0x3, 0x11, 0x3, 0x11, + 0x3, 0x11, 0x5, 0x11, 0x2ec, 0xa, 0x11, 0x3, 0x11, 0x5, 0x11, 0x2ef, + 0xa, 0x11, 0x3, 0x12, 0x3, 0x12, 0x3, 0x12, 0x3, 0x12, 0x3, 0x12, 0x7, + 0x12, 0x2f6, 0xa, 0x12, 0xc, 0x12, 0xe, 0x12, 0x2f9, 0xb, 0x12, 0x3, + 0x12, 0x3, 0x12, 0x3, 0x12, 0x3, 0x13, 0x3, 0x13, 0x3, 0x13, 0x3, 0x13, + 0x3, 0x13, 0x3, 0x13, 0x3, 0x13, 0x3, 0x13, 0x3, 0x13, 0x3, 0x13, 0x3, + 0x13, 0x5, 0x13, 0x309, 0xa, 0x13, 0x3, 0x13, 0x3, 0x13, 0x3, 0x14, + 0x3, 0x14, 0x3, 0x14, 0x3, 0x14, 0x3, 0x14, 0x7, 0x14, 0x312, 0xa, 0x14, + 0xc, 0x14, 0xe, 0x14, 0x315, 0xb, 0x14, 0x3, 0x14, 0x3, 0x14, 0x3, 0x14, + 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, + 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x5, 0x15, + 0x326, 0xa, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x16, 0x3, 0x16, 0x3, 0x16, + 0x3, 0x16, 0x3, 0x16, 0x3, 0x17, 0x3, 0x17, 0x3, 0x17, 0x3, 0x17, 0x5, + 0x17, 0x333, 0xa, 0x17, 0x3, 0x18, 0x3, 0x18, 0x3, 0x18, 0x3, 0x19, + 0x3, 0x19, 0x3, 0x19, 0x3, 0x1a, 0x3, 0x1a, 0x3, 0x1a, 0x3, 0x1b, 0x3, + 0x1b, 0x3, 0x1b, 0x3, 0x1b, 0x7, 0x1b, 0x342, 0xa, 0x1b, 0xc, 0x1b, + 0xe, 0x1b, 0x345, 0xb, 0x1b, 0x3, 0x1b, 0x3, 0x1b, 0x3, 0x1b, 0x3, 0x1b, + 0x3, 0x1b, 0x3, 0x1b, 0x5, 0x1b, 0x34d, 0xa, 0x1b, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, - 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x7, 0x1c, 0x358, - 0xa, 0x1c, 0xc, 0x1c, 0xe, 0x1c, 0x35b, 0xb, 0x1c, 0x3, 0x1d, 0x3, 0x1d, - 0x3, 0x1d, 0x3, 0x1d, 0x3, 0x1e, 0x3, 0x1e, 0x3, 0x1e, 0x3, 0x1e, 0x3, - 0x1f, 0x3, 0x1f, 0x3, 0x1f, 0x3, 0x1f, 0x3, 0x20, 0x3, 0x20, 0x3, 0x20, - 0x3, 0x20, 0x7, 0x20, 0x36d, 0xa, 0x20, 0xc, 0x20, 0xe, 0x20, 0x370, - 0xb, 0x20, 0x3, 0x21, 0x3, 0x21, 0x5, 0x21, 0x374, 0xa, 0x21, 0x3, 0x21, - 0x3, 0x21, 0x3, 0x21, 0x5, 0x21, 0x379, 0xa, 0x21, 0x3, 0x21, 0x5, 0x21, - 0x37c, 0xa, 0x21, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, - 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x5, 0x22, 0x388, - 0xa, 0x22, 0x3, 0x23, 0x3, 0x23, 0x3, 0x23, 0x5, 0x23, 0x38d, 0xa, 0x23, - 0x3, 0x23, 0x3, 0x23, 0x5, 0x23, 0x391, 0xa, 0x23, 0x3, 0x23, 0x5, 0x23, - 0x394, 0xa, 0x23, 0x3, 0x23, 0x3, 0x23, 0x5, 0x23, 0x398, 0xa, 0x23, - 0x3, 0x23, 0x3, 0x23, 0x5, 0x23, 0x39c, 0xa, 0x23, 0x3, 0x23, 0x3, 0x23, - 0x3, 0x23, 0x5, 0x23, 0x3a1, 0xa, 0x23, 0x3, 0x23, 0x5, 0x23, 0x3a4, - 0xa, 0x23, 0x3, 0x23, 0x3, 0x23, 0x5, 0x23, 0x3a8, 0xa, 0x23, 0x5, 0x23, - 0x3aa, 0xa, 0x23, 0x3, 0x24, 0x3, 0x24, 0x3, 0x24, 0x3, 0x25, 0x3, 0x25, + 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x7, 0x1c, 0x368, 0xa, 0x1c, + 0xc, 0x1c, 0xe, 0x1c, 0x36b, 0xb, 0x1c, 0x3, 0x1d, 0x3, 0x1d, 0x3, 0x1d, + 0x3, 0x1d, 0x3, 0x1e, 0x3, 0x1e, 0x3, 0x1e, 0x3, 0x1e, 0x3, 0x1f, 0x3, + 0x1f, 0x3, 0x1f, 0x3, 0x1f, 0x3, 0x20, 0x3, 0x20, 0x3, 0x20, 0x3, 0x20, + 0x7, 0x20, 0x37d, 0xa, 0x20, 0xc, 0x20, 0xe, 0x20, 0x380, 0xb, 0x20, + 0x3, 0x21, 0x3, 0x21, 0x5, 0x21, 0x384, 0xa, 0x21, 0x3, 0x21, 0x3, 0x21, + 0x3, 0x21, 0x5, 0x21, 0x389, 0xa, 0x21, 0x3, 0x21, 0x5, 0x21, 0x38c, + 0xa, 0x21, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, + 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x5, 0x22, 0x398, + 0xa, 0x22, 0x3, 0x23, 0x3, 0x23, 0x3, 0x23, 0x5, 0x23, 0x39d, 0xa, 0x23, + 0x3, 0x23, 0x3, 0x23, 0x5, 0x23, 0x3a1, 0xa, 0x23, 0x3, 0x23, 0x5, 0x23, + 0x3a4, 0xa, 0x23, 0x3, 0x23, 0x3, 0x23, 0x5, 0x23, 0x3a8, 0xa, 0x23, + 0x3, 0x23, 0x3, 0x23, 0x5, 0x23, 0x3ac, 0xa, 0x23, 0x3, 0x23, 0x3, 0x23, + 0x3, 0x23, 0x5, 0x23, 0x3b1, 0xa, 0x23, 0x3, 0x23, 0x5, 0x23, 0x3b4, + 0xa, 0x23, 0x3, 0x23, 0x3, 0x23, 0x5, 0x23, 0x3b8, 0xa, 0x23, 0x5, 0x23, + 0x3ba, 0xa, 0x23, 0x3, 0x24, 0x3, 0x24, 0x3, 0x24, 0x3, 0x25, 0x3, 0x25, 0x3, 0x25, 0x3, 0x25, 0x3, 0x25, 0x3, 0x25, 0x3, 0x25, 0x3, 0x26, 0x3, 0x26, 0x3, 0x26, 0x3, 0x27, 0x3, 0x27, 0x3, 0x27, 0x3, 0x27, 0x3, 0x27, - 0x7, 0x27, 0x3be, 0xa, 0x27, 0xc, 0x27, 0xe, 0x27, 0x3c1, 0xb, 0x27, - 0x3, 0x27, 0x3, 0x27, 0x3, 0x28, 0x3, 0x28, 0x3, 0x28, 0x5, 0x28, 0x3c8, - 0xa, 0x28, 0x3, 0x28, 0x5, 0x28, 0x3cb, 0xa, 0x28, 0x3, 0x29, 0x3, 0x29, + 0x7, 0x27, 0x3ce, 0xa, 0x27, 0xc, 0x27, 0xe, 0x27, 0x3d1, 0xb, 0x27, + 0x3, 0x27, 0x3, 0x27, 0x3, 0x28, 0x3, 0x28, 0x3, 0x28, 0x5, 0x28, 0x3d8, + 0xa, 0x28, 0x3, 0x28, 0x5, 0x28, 0x3db, 0xa, 0x28, 0x3, 0x29, 0x3, 0x29, 0x3, 0x29, 0x3, 0x29, 0x3, 0x29, 0x3, 0x29, 0x3, 0x29, 0x3, 0x29, 0x5, - 0x29, 0x3d5, 0xa, 0x29, 0x3, 0x2a, 0x3, 0x2a, 0x5, 0x2a, 0x3d9, 0xa, + 0x29, 0x3e5, 0xa, 0x29, 0x3, 0x2a, 0x3, 0x2a, 0x5, 0x2a, 0x3e9, 0xa, 0x2a, 0x3, 0x2a, 0x3, 0x2a, 0x3, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x3, 0x2b, - 0x5, 0x2b, 0x3e1, 0xa, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x5, 0x2b, 0x3e5, - 0xa, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x5, 0x2b, 0x3ea, 0xa, 0x2b, - 0x3, 0x2b, 0x3, 0x2b, 0x5, 0x2b, 0x3ee, 0xa, 0x2b, 0x3, 0x2b, 0x3, 0x2b, - 0x5, 0x2b, 0x3f2, 0xa, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x5, 0x2b, 0x3f6, - 0xa, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x5, 0x2b, 0x3fa, 0xa, 0x2b, 0x5, 0x2b, - 0x3fc, 0xa, 0x2b, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, - 0x3, 0x2c, 0x5, 0x2c, 0x404, 0xa, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x5, 0x2c, - 0x408, 0xa, 0x2c, 0x3, 0x2c, 0x5, 0x2c, 0x40b, 0xa, 0x2c, 0x3, 0x2d, - 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x5, 0x2d, 0x413, - 0xa, 0x2d, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x5, 0x2e, 0x418, 0xa, 0x2e, - 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x5, 0x2e, 0x41d, 0xa, 0x2e, 0x3, 0x2e, - 0x5, 0x2e, 0x420, 0xa, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2f, 0x3, 0x2f, - 0x3, 0x2f, 0x3, 0x2f, 0x7, 0x2f, 0x428, 0xa, 0x2f, 0xc, 0x2f, 0xe, 0x2f, - 0x42b, 0xb, 0x2f, 0x3, 0x2f, 0x3, 0x2f, 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, - 0x3, 0x30, 0x3, 0x30, 0x5, 0x30, 0x434, 0xa, 0x30, 0x3, 0x30, 0x3, 0x30, - 0x5, 0x30, 0x438, 0xa, 0x30, 0x3, 0x31, 0x3, 0x31, 0x3, 0x31, 0x5, 0x31, - 0x43d, 0xa, 0x31, 0x3, 0x31, 0x3, 0x31, 0x5, 0x31, 0x441, 0xa, 0x31, - 0x3, 0x32, 0x3, 0x32, 0x3, 0x32, 0x3, 0x32, 0x5, 0x32, 0x447, 0xa, 0x32, - 0x3, 0x32, 0x5, 0x32, 0x44a, 0xa, 0x32, 0x3, 0x32, 0x5, 0x32, 0x44d, - 0xa, 0x32, 0x3, 0x32, 0x5, 0x32, 0x450, 0xa, 0x32, 0x3, 0x33, 0x3, 0x33, + 0x5, 0x2b, 0x3f1, 0xa, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x5, 0x2b, 0x3f5, + 0xa, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x5, 0x2b, 0x3fa, 0xa, 0x2b, + 0x3, 0x2b, 0x3, 0x2b, 0x5, 0x2b, 0x3fe, 0xa, 0x2b, 0x3, 0x2b, 0x3, 0x2b, + 0x5, 0x2b, 0x402, 0xa, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x5, 0x2b, 0x406, + 0xa, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x5, 0x2b, 0x40a, 0xa, 0x2b, 0x5, 0x2b, + 0x40c, 0xa, 0x2b, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, + 0x3, 0x2c, 0x5, 0x2c, 0x414, 0xa, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x5, 0x2c, + 0x418, 0xa, 0x2c, 0x3, 0x2c, 0x5, 0x2c, 0x41b, 0xa, 0x2c, 0x3, 0x2d, + 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x5, 0x2d, 0x423, + 0xa, 0x2d, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x5, 0x2e, 0x428, 0xa, 0x2e, + 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x5, 0x2e, 0x42d, 0xa, 0x2e, 0x3, 0x2e, + 0x5, 0x2e, 0x430, 0xa, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2f, 0x3, 0x2f, + 0x3, 0x2f, 0x3, 0x2f, 0x7, 0x2f, 0x438, 0xa, 0x2f, 0xc, 0x2f, 0xe, 0x2f, + 0x43b, 0xb, 0x2f, 0x3, 0x2f, 0x3, 0x2f, 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, + 0x3, 0x30, 0x3, 0x30, 0x5, 0x30, 0x444, 0xa, 0x30, 0x3, 0x30, 0x3, 0x30, + 0x5, 0x30, 0x448, 0xa, 0x30, 0x3, 0x31, 0x3, 0x31, 0x3, 0x31, 0x5, 0x31, + 0x44d, 0xa, 0x31, 0x3, 0x31, 0x3, 0x31, 0x5, 0x31, 0x451, 0xa, 0x31, + 0x3, 0x32, 0x3, 0x32, 0x3, 0x32, 0x3, 0x32, 0x5, 0x32, 0x457, 0xa, 0x32, + 0x3, 0x32, 0x5, 0x32, 0x45a, 0xa, 0x32, 0x3, 0x32, 0x5, 0x32, 0x45d, + 0xa, 0x32, 0x3, 0x32, 0x5, 0x32, 0x460, 0xa, 0x32, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, - 0x33, 0x3, 0x33, 0x7, 0x33, 0x45c, 0xa, 0x33, 0xc, 0x33, 0xe, 0x33, - 0x45f, 0xb, 0x33, 0x3, 0x33, 0x5, 0x33, 0x462, 0xa, 0x33, 0x3, 0x34, - 0x3, 0x34, 0x5, 0x34, 0x466, 0xa, 0x34, 0x3, 0x34, 0x3, 0x34, 0x3, 0x34, - 0x5, 0x34, 0x46b, 0xa, 0x34, 0x3, 0x34, 0x5, 0x34, 0x46e, 0xa, 0x34, + 0x33, 0x3, 0x33, 0x7, 0x33, 0x46c, 0xa, 0x33, 0xc, 0x33, 0xe, 0x33, + 0x46f, 0xb, 0x33, 0x3, 0x33, 0x5, 0x33, 0x472, 0xa, 0x33, 0x3, 0x34, + 0x3, 0x34, 0x5, 0x34, 0x476, 0xa, 0x34, 0x3, 0x34, 0x3, 0x34, 0x3, 0x34, + 0x5, 0x34, 0x47b, 0xa, 0x34, 0x3, 0x34, 0x5, 0x34, 0x47e, 0xa, 0x34, 0x3, 0x34, 0x3, 0x34, 0x3, 0x35, 0x3, 0x35, 0x3, 0x35, 0x3, 0x35, 0x7, - 0x35, 0x476, 0xa, 0x35, 0xc, 0x35, 0xe, 0x35, 0x479, 0xb, 0x35, 0x3, - 0x36, 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, 0x5, 0x36, 0x480, - 0xa, 0x36, 0x3, 0x37, 0x5, 0x37, 0x483, 0xa, 0x37, 0x3, 0x37, 0x3, 0x37, - 0x5, 0x37, 0x487, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x48a, 0xa, 0x37, - 0x3, 0x37, 0x3, 0x37, 0x5, 0x37, 0x48e, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, - 0x491, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x494, 0xa, 0x37, 0x3, 0x37, + 0x35, 0x486, 0xa, 0x35, 0xc, 0x35, 0xe, 0x35, 0x489, 0xb, 0x35, 0x3, + 0x36, 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, 0x5, 0x36, 0x490, + 0xa, 0x36, 0x3, 0x37, 0x5, 0x37, 0x493, 0xa, 0x37, 0x3, 0x37, 0x3, 0x37, 0x5, 0x37, 0x497, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x49a, 0xa, 0x37, - 0x3, 0x37, 0x3, 0x37, 0x5, 0x37, 0x49e, 0xa, 0x37, 0x3, 0x37, 0x3, 0x37, - 0x5, 0x37, 0x4a2, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x4a5, 0xa, 0x37, - 0x3, 0x37, 0x5, 0x37, 0x4a8, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x4ab, - 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x4ae, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, - 0x4b1, 0xa, 0x37, 0x3, 0x38, 0x3, 0x38, 0x3, 0x38, 0x3, 0x39, 0x3, 0x39, - 0x3, 0x39, 0x3, 0x39, 0x5, 0x39, 0x4ba, 0xa, 0x39, 0x3, 0x3a, 0x3, 0x3a, - 0x3, 0x3a, 0x3, 0x3b, 0x5, 0x3b, 0x4c0, 0xa, 0x3b, 0x3, 0x3b, 0x3, 0x3b, + 0x3, 0x37, 0x3, 0x37, 0x5, 0x37, 0x49e, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, + 0x4a1, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x4a4, 0xa, 0x37, 0x3, 0x37, + 0x5, 0x37, 0x4a7, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x4aa, 0xa, 0x37, + 0x3, 0x37, 0x3, 0x37, 0x5, 0x37, 0x4ae, 0xa, 0x37, 0x3, 0x37, 0x3, 0x37, + 0x5, 0x37, 0x4b2, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x4b5, 0xa, 0x37, + 0x3, 0x37, 0x5, 0x37, 0x4b8, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x4bb, + 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x4be, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, + 0x4c1, 0xa, 0x37, 0x3, 0x38, 0x3, 0x38, 0x3, 0x38, 0x3, 0x39, 0x3, 0x39, + 0x3, 0x39, 0x3, 0x39, 0x5, 0x39, 0x4ca, 0xa, 0x39, 0x3, 0x3a, 0x3, 0x3a, + 0x3, 0x3a, 0x3, 0x3b, 0x5, 0x3b, 0x4d0, 0xa, 0x3b, 0x3, 0x3b, 0x3, 0x3b, 0x3, 0x3b, 0x3, 0x3b, 0x3, 0x3c, 0x3, 0x3c, 0x3, 0x3c, 0x3, 0x3d, 0x3, 0x3d, 0x3, 0x3d, 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, - 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, 0x5, 0x3e, 0x4d4, 0xa, 0x3e, 0x3, 0x3f, + 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, 0x5, 0x3e, 0x4e4, 0xa, 0x3e, 0x3, 0x3f, 0x3, 0x3f, 0x3, 0x3f, 0x3, 0x40, 0x3, 0x40, 0x3, 0x40, 0x3, 0x40, 0x3, 0x41, 0x3, 0x41, 0x3, 0x41, 0x3, 0x41, 0x3, 0x42, 0x3, 0x42, 0x3, 0x42, 0x3, 0x42, 0x3, 0x42, 0x3, 0x43, 0x3, 0x43, 0x3, 0x43, 0x3, 0x43, 0x5, - 0x43, 0x4ea, 0xa, 0x43, 0x3, 0x44, 0x3, 0x44, 0x3, 0x44, 0x3, 0x45, - 0x3, 0x45, 0x3, 0x45, 0x5, 0x45, 0x4f2, 0xa, 0x45, 0x3, 0x45, 0x5, 0x45, - 0x4f5, 0xa, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x5, 0x45, - 0x4fb, 0xa, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, - 0x3, 0x45, 0x5, 0x45, 0x503, 0xa, 0x45, 0x3, 0x45, 0x5, 0x45, 0x506, - 0xa, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x7, 0x45, 0x50c, - 0xa, 0x45, 0xc, 0x45, 0xe, 0x45, 0x50f, 0xb, 0x45, 0x3, 0x46, 0x5, 0x46, - 0x512, 0xa, 0x46, 0x3, 0x46, 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, 0x517, - 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, 0x51a, 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, - 0x51d, 0xa, 0x46, 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, 0x521, 0xa, 0x46, - 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, 0x525, 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, - 0x528, 0xa, 0x46, 0x5, 0x46, 0x52a, 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, + 0x43, 0x4fa, 0xa, 0x43, 0x3, 0x44, 0x3, 0x44, 0x3, 0x44, 0x3, 0x45, + 0x3, 0x45, 0x3, 0x45, 0x5, 0x45, 0x502, 0xa, 0x45, 0x3, 0x45, 0x5, 0x45, + 0x505, 0xa, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x5, 0x45, + 0x50b, 0xa, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, + 0x3, 0x45, 0x5, 0x45, 0x513, 0xa, 0x45, 0x3, 0x45, 0x5, 0x45, 0x516, + 0xa, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x7, 0x45, 0x51c, + 0xa, 0x45, 0xc, 0x45, 0xe, 0x45, 0x51f, 0xb, 0x45, 0x3, 0x46, 0x5, 0x46, + 0x522, 0xa, 0x46, 0x3, 0x46, 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, 0x527, + 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, 0x52a, 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, 0x52d, 0xa, 0x46, 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, 0x531, 0xa, 0x46, 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, 0x535, 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, - 0x538, 0xa, 0x46, 0x5, 0x46, 0x53a, 0xa, 0x46, 0x5, 0x46, 0x53c, 0xa, - 0x46, 0x3, 0x47, 0x5, 0x47, 0x53f, 0xa, 0x47, 0x3, 0x47, 0x3, 0x47, - 0x3, 0x47, 0x5, 0x47, 0x544, 0xa, 0x47, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, + 0x538, 0xa, 0x46, 0x5, 0x46, 0x53a, 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, + 0x53d, 0xa, 0x46, 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, 0x541, 0xa, 0x46, + 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, 0x545, 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, + 0x548, 0xa, 0x46, 0x5, 0x46, 0x54a, 0xa, 0x46, 0x5, 0x46, 0x54c, 0xa, + 0x46, 0x3, 0x47, 0x5, 0x47, 0x54f, 0xa, 0x47, 0x3, 0x47, 0x3, 0x47, + 0x3, 0x47, 0x5, 0x47, 0x554, 0xa, 0x47, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x5, - 0x48, 0x54f, 0xa, 0x48, 0x3, 0x49, 0x3, 0x49, 0x3, 0x49, 0x3, 0x49, - 0x5, 0x49, 0x555, 0xa, 0x49, 0x3, 0x4a, 0x3, 0x4a, 0x3, 0x4a, 0x5, 0x4a, - 0x55a, 0xa, 0x4a, 0x3, 0x4b, 0x3, 0x4b, 0x3, 0x4b, 0x7, 0x4b, 0x55f, - 0xa, 0x4b, 0xc, 0x4b, 0xe, 0x4b, 0x562, 0xb, 0x4b, 0x3, 0x4c, 0x3, 0x4c, - 0x5, 0x4c, 0x566, 0xa, 0x4c, 0x3, 0x4c, 0x3, 0x4c, 0x5, 0x4c, 0x56a, - 0xa, 0x4c, 0x3, 0x4c, 0x3, 0x4c, 0x5, 0x4c, 0x56e, 0xa, 0x4c, 0x3, 0x4d, - 0x3, 0x4d, 0x3, 0x4d, 0x5, 0x4d, 0x573, 0xa, 0x4d, 0x3, 0x4e, 0x3, 0x4e, - 0x3, 0x4e, 0x7, 0x4e, 0x578, 0xa, 0x4e, 0xc, 0x4e, 0xe, 0x4e, 0x57b, + 0x48, 0x55f, 0xa, 0x48, 0x3, 0x49, 0x3, 0x49, 0x3, 0x49, 0x3, 0x49, + 0x5, 0x49, 0x565, 0xa, 0x49, 0x3, 0x4a, 0x3, 0x4a, 0x3, 0x4a, 0x5, 0x4a, + 0x56a, 0xa, 0x4a, 0x3, 0x4b, 0x3, 0x4b, 0x3, 0x4b, 0x7, 0x4b, 0x56f, + 0xa, 0x4b, 0xc, 0x4b, 0xe, 0x4b, 0x572, 0xb, 0x4b, 0x3, 0x4c, 0x3, 0x4c, + 0x5, 0x4c, 0x576, 0xa, 0x4c, 0x3, 0x4c, 0x3, 0x4c, 0x5, 0x4c, 0x57a, + 0xa, 0x4c, 0x3, 0x4c, 0x3, 0x4c, 0x5, 0x4c, 0x57e, 0xa, 0x4c, 0x3, 0x4d, + 0x3, 0x4d, 0x3, 0x4d, 0x5, 0x4d, 0x583, 0xa, 0x4d, 0x3, 0x4e, 0x3, 0x4e, + 0x3, 0x4e, 0x7, 0x4e, 0x588, 0xa, 0x4e, 0xc, 0x4e, 0xe, 0x4e, 0x58b, 0xb, 0x4e, 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x50, 0x3, 0x50, 0x3, 0x50, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x5, - 0x51, 0x58f, 0xa, 0x51, 0x3, 0x51, 0x5, 0x51, 0x592, 0xa, 0x51, 0x3, + 0x51, 0x59f, 0xa, 0x51, 0x3, 0x51, 0x5, 0x51, 0x5a2, 0xa, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, - 0x5, 0x51, 0x59b, 0xa, 0x51, 0x3, 0x51, 0x3, 0x51, 0x5, 0x51, 0x59f, - 0xa, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x5, 0x51, 0x5a4, 0xa, 0x51, - 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x5, 0x51, 0x5a9, 0xa, 0x51, 0x3, 0x51, - 0x5, 0x51, 0x5ac, 0xa, 0x51, 0x5, 0x51, 0x5ae, 0xa, 0x51, 0x3, 0x52, + 0x5, 0x51, 0x5ab, 0xa, 0x51, 0x3, 0x51, 0x3, 0x51, 0x5, 0x51, 0x5af, + 0xa, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x5, 0x51, 0x5b4, 0xa, 0x51, + 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x5, 0x51, 0x5b9, 0xa, 0x51, 0x3, 0x51, + 0x5, 0x51, 0x5bc, 0xa, 0x51, 0x5, 0x51, 0x5be, 0xa, 0x51, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x5, - 0x52, 0x5c4, 0xa, 0x52, 0x3, 0x52, 0x5, 0x52, 0x5c7, 0xa, 0x52, 0x3, + 0x52, 0x5d4, 0xa, 0x52, 0x3, 0x52, 0x5, 0x52, 0x5d7, 0xa, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, - 0x3, 0x52, 0x3, 0x52, 0x5, 0x52, 0x5d2, 0xa, 0x52, 0x3, 0x53, 0x3, 0x53, - 0x5, 0x53, 0x5d6, 0xa, 0x53, 0x3, 0x53, 0x5, 0x53, 0x5d9, 0xa, 0x53, - 0x3, 0x53, 0x3, 0x53, 0x5, 0x53, 0x5dd, 0xa, 0x53, 0x3, 0x53, 0x3, 0x53, - 0x5, 0x53, 0x5e1, 0xa, 0x53, 0x3, 0x54, 0x3, 0x54, 0x3, 0x54, 0x3, 0x55, - 0x3, 0x55, 0x3, 0x55, 0x5, 0x55, 0x5e9, 0xa, 0x55, 0x3, 0x55, 0x3, 0x55, - 0x5, 0x55, 0x5ed, 0xa, 0x55, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, - 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x7, 0x56, 0x5f8, - 0xa, 0x56, 0xc, 0x56, 0xe, 0x56, 0x5fb, 0xb, 0x56, 0x3, 0x56, 0x3, 0x56, - 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x7, 0x56, 0x604, - 0xa, 0x56, 0xc, 0x56, 0xe, 0x56, 0x607, 0xb, 0x56, 0x3, 0x56, 0x3, 0x56, - 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x7, 0x56, 0x610, - 0xa, 0x56, 0xc, 0x56, 0xe, 0x56, 0x613, 0xb, 0x56, 0x3, 0x56, 0x3, 0x56, - 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x5, 0x56, 0x61a, 0xa, 0x56, 0x3, 0x56, - 0x3, 0x56, 0x5, 0x56, 0x61e, 0xa, 0x56, 0x3, 0x57, 0x3, 0x57, 0x3, 0x57, - 0x7, 0x57, 0x623, 0xa, 0x57, 0xc, 0x57, 0xe, 0x57, 0x626, 0xb, 0x57, - 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x5, 0x58, 0x62b, 0xa, 0x58, 0x3, 0x58, - 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x5, 0x58, 0x633, - 0xa, 0x58, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x638, 0xa, 0x59, - 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x6, 0x59, 0x63f, - 0xa, 0x59, 0xd, 0x59, 0xe, 0x59, 0x640, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, - 0x645, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, + 0x3, 0x52, 0x3, 0x52, 0x5, 0x52, 0x5e2, 0xa, 0x52, 0x3, 0x53, 0x3, 0x53, + 0x5, 0x53, 0x5e6, 0xa, 0x53, 0x3, 0x53, 0x5, 0x53, 0x5e9, 0xa, 0x53, + 0x3, 0x53, 0x3, 0x53, 0x5, 0x53, 0x5ed, 0xa, 0x53, 0x3, 0x53, 0x3, 0x53, + 0x5, 0x53, 0x5f1, 0xa, 0x53, 0x3, 0x54, 0x3, 0x54, 0x3, 0x54, 0x3, 0x55, + 0x3, 0x55, 0x3, 0x55, 0x5, 0x55, 0x5f9, 0xa, 0x55, 0x3, 0x55, 0x3, 0x55, + 0x5, 0x55, 0x5fd, 0xa, 0x55, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, + 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x7, 0x56, 0x608, + 0xa, 0x56, 0xc, 0x56, 0xe, 0x56, 0x60b, 0xb, 0x56, 0x3, 0x56, 0x3, 0x56, + 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x7, 0x56, 0x614, + 0xa, 0x56, 0xc, 0x56, 0xe, 0x56, 0x617, 0xb, 0x56, 0x3, 0x56, 0x3, 0x56, + 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x7, 0x56, 0x620, + 0xa, 0x56, 0xc, 0x56, 0xe, 0x56, 0x623, 0xb, 0x56, 0x3, 0x56, 0x3, 0x56, + 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x5, 0x56, 0x62a, 0xa, 0x56, 0x3, 0x56, + 0x3, 0x56, 0x5, 0x56, 0x62e, 0xa, 0x56, 0x3, 0x57, 0x3, 0x57, 0x3, 0x57, + 0x7, 0x57, 0x633, 0xa, 0x57, 0xc, 0x57, 0xe, 0x57, 0x636, 0xb, 0x57, + 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x5, 0x58, 0x63b, 0xa, 0x58, 0x3, 0x58, + 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x5, 0x58, 0x643, + 0xa, 0x58, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x648, 0xa, 0x59, + 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x6, 0x59, 0x64f, + 0xa, 0x59, 0xd, 0x59, 0xe, 0x59, 0x650, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, + 0x655, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, - 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x664, + 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x674, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, - 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x675, 0xa, 0x59, 0x3, 0x59, - 0x5, 0x59, 0x678, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x67c, - 0xa, 0x59, 0x3, 0x59, 0x5, 0x59, 0x67f, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, + 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x685, 0xa, 0x59, 0x3, 0x59, + 0x5, 0x59, 0x688, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x68c, + 0xa, 0x59, 0x3, 0x59, 0x5, 0x59, 0x68f, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, - 0x59, 0x3, 0x59, 0x5, 0x59, 0x68b, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, + 0x59, 0x3, 0x59, 0x5, 0x59, 0x69b, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, - 0x5, 0x59, 0x69c, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6a0, + 0x5, 0x59, 0x6ac, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6b0, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, - 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6b1, 0xa, 0x59, 0x3, 0x59, - 0x5, 0x59, 0x6b4, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6b8, - 0xa, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6bb, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, + 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6c1, 0xa, 0x59, 0x3, 0x59, + 0x5, 0x59, 0x6c4, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6c8, + 0xa, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6cb, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, - 0x59, 0x5, 0x59, 0x6c6, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, + 0x59, 0x5, 0x59, 0x6d6, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, - 0x59, 0x6de, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, - 0x3, 0x59, 0x5, 0x59, 0x6e5, 0xa, 0x59, 0x7, 0x59, 0x6e7, 0xa, 0x59, - 0xc, 0x59, 0xe, 0x59, 0x6ea, 0xb, 0x59, 0x3, 0x5a, 0x3, 0x5a, 0x3, 0x5a, - 0x7, 0x5a, 0x6ef, 0xa, 0x5a, 0xc, 0x5a, 0xe, 0x5a, 0x6f2, 0xb, 0x5a, - 0x3, 0x5b, 0x3, 0x5b, 0x5, 0x5b, 0x6f6, 0xa, 0x5b, 0x3, 0x5c, 0x3, 0x5c, - 0x3, 0x5c, 0x3, 0x5c, 0x7, 0x5c, 0x6fc, 0xa, 0x5c, 0xc, 0x5c, 0xe, 0x5c, - 0x6ff, 0xb, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, - 0x7, 0x5c, 0x706, 0xa, 0x5c, 0xc, 0x5c, 0xe, 0x5c, 0x709, 0xb, 0x5c, - 0x5, 0x5c, 0x70b, 0xa, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5d, - 0x3, 0x5d, 0x3, 0x5d, 0x5, 0x5d, 0x713, 0xa, 0x5d, 0x3, 0x5d, 0x3, 0x5d, - 0x3, 0x5e, 0x3, 0x5e, 0x3, 0x5e, 0x5, 0x5e, 0x71a, 0xa, 0x5e, 0x3, 0x5f, + 0x59, 0x6ee, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, + 0x3, 0x59, 0x5, 0x59, 0x6f5, 0xa, 0x59, 0x7, 0x59, 0x6f7, 0xa, 0x59, + 0xc, 0x59, 0xe, 0x59, 0x6fa, 0xb, 0x59, 0x3, 0x5a, 0x3, 0x5a, 0x3, 0x5a, + 0x7, 0x5a, 0x6ff, 0xa, 0x5a, 0xc, 0x5a, 0xe, 0x5a, 0x702, 0xb, 0x5a, + 0x3, 0x5b, 0x3, 0x5b, 0x5, 0x5b, 0x706, 0xa, 0x5b, 0x3, 0x5c, 0x3, 0x5c, + 0x3, 0x5c, 0x3, 0x5c, 0x7, 0x5c, 0x70c, 0xa, 0x5c, 0xc, 0x5c, 0xe, 0x5c, + 0x70f, 0xb, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, + 0x7, 0x5c, 0x716, 0xa, 0x5c, 0xc, 0x5c, 0xe, 0x5c, 0x719, 0xb, 0x5c, + 0x5, 0x5c, 0x71b, 0xa, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5d, + 0x3, 0x5d, 0x3, 0x5d, 0x5, 0x5d, 0x723, 0xa, 0x5d, 0x3, 0x5d, 0x3, 0x5d, + 0x3, 0x5e, 0x3, 0x5e, 0x3, 0x5e, 0x5, 0x5e, 0x72a, 0xa, 0x5e, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x5, - 0x5f, 0x723, 0xa, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, - 0x5, 0x5f, 0x729, 0xa, 0x5f, 0x7, 0x5f, 0x72b, 0xa, 0x5f, 0xc, 0x5f, - 0xe, 0x5f, 0x72e, 0xb, 0x5f, 0x3, 0x60, 0x3, 0x60, 0x3, 0x60, 0x5, 0x60, - 0x733, 0xa, 0x60, 0x3, 0x60, 0x3, 0x60, 0x3, 0x61, 0x3, 0x61, 0x3, 0x61, - 0x5, 0x61, 0x73a, 0xa, 0x61, 0x3, 0x61, 0x3, 0x61, 0x3, 0x62, 0x3, 0x62, - 0x3, 0x62, 0x7, 0x62, 0x741, 0xa, 0x62, 0xc, 0x62, 0xe, 0x62, 0x744, - 0xb, 0x62, 0x3, 0x63, 0x3, 0x63, 0x3, 0x63, 0x5, 0x63, 0x749, 0xa, 0x63, + 0x5f, 0x733, 0xa, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, + 0x5, 0x5f, 0x739, 0xa, 0x5f, 0x7, 0x5f, 0x73b, 0xa, 0x5f, 0xc, 0x5f, + 0xe, 0x5f, 0x73e, 0xb, 0x5f, 0x3, 0x60, 0x3, 0x60, 0x3, 0x60, 0x5, 0x60, + 0x743, 0xa, 0x60, 0x3, 0x60, 0x3, 0x60, 0x3, 0x61, 0x3, 0x61, 0x3, 0x61, + 0x5, 0x61, 0x74a, 0xa, 0x61, 0x3, 0x61, 0x3, 0x61, 0x3, 0x62, 0x3, 0x62, + 0x3, 0x62, 0x7, 0x62, 0x751, 0xa, 0x62, 0xc, 0x62, 0xe, 0x62, 0x754, + 0xb, 0x62, 0x3, 0x63, 0x3, 0x63, 0x3, 0x63, 0x5, 0x63, 0x759, 0xa, 0x63, 0x3, 0x64, 0x3, 0x64, 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, 0x3, - 0x65, 0x3, 0x65, 0x5, 0x65, 0x753, 0xa, 0x65, 0x5, 0x65, 0x755, 0xa, - 0x65, 0x3, 0x66, 0x5, 0x66, 0x758, 0xa, 0x66, 0x3, 0x66, 0x3, 0x66, - 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x5, 0x66, 0x760, 0xa, 0x66, - 0x3, 0x67, 0x3, 0x67, 0x3, 0x67, 0x5, 0x67, 0x765, 0xa, 0x67, 0x3, 0x68, + 0x65, 0x3, 0x65, 0x5, 0x65, 0x763, 0xa, 0x65, 0x5, 0x65, 0x765, 0xa, + 0x65, 0x3, 0x66, 0x5, 0x66, 0x768, 0xa, 0x66, 0x3, 0x66, 0x3, 0x66, + 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x5, 0x66, 0x770, 0xa, 0x66, + 0x3, 0x67, 0x3, 0x67, 0x3, 0x67, 0x5, 0x67, 0x775, 0xa, 0x67, 0x3, 0x68, 0x3, 0x68, 0x3, 0x69, 0x3, 0x69, 0x3, 0x6a, 0x3, 0x6a, 0x3, 0x6b, 0x3, - 0x6b, 0x5, 0x6b, 0x76f, 0xa, 0x6b, 0x3, 0x6c, 0x3, 0x6c, 0x3, 0x6c, - 0x5, 0x6c, 0x774, 0xa, 0x6c, 0x3, 0x6d, 0x3, 0x6d, 0x5, 0x6d, 0x778, + 0x6b, 0x5, 0x6b, 0x77f, 0xa, 0x6b, 0x3, 0x6c, 0x3, 0x6c, 0x3, 0x6c, + 0x5, 0x6c, 0x784, 0xa, 0x6c, 0x3, 0x6d, 0x3, 0x6d, 0x5, 0x6d, 0x788, 0xa, 0x6d, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6e, 0x2, 0x5, 0x88, 0xb0, 0xbc, 0x6f, 0x2, 0x4, 0x6, 0x8, 0xa, 0xc, 0xe, 0x10, 0x12, 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e, 0x20, 0x22, 0x24, 0x26, 0x28, @@ -19094,57 +19160,57 @@ ClickHouseParser::Initializer::Initializer() { 0xb7, 0xe, 0x2, 0x4, 0x24, 0x26, 0x4a, 0x4c, 0x50, 0x52, 0x6a, 0x6c, 0x6c, 0x6e, 0x6f, 0x71, 0x72, 0x74, 0x7f, 0x82, 0x8b, 0x8d, 0xb2, 0xb4, 0xb6, 0xb8, 0xb9, 0x6, 0x2, 0x24, 0x24, 0x3e, 0x3e, 0x4c, 0x4c, 0x5a, - 0x5a, 0x2, 0x88f, 0x2, 0xea, 0x3, 0x2, 0x2, 0x2, 0x4, 0xfe, 0x3, 0x2, + 0x5a, 0x2, 0x8a5, 0x2, 0xea, 0x3, 0x2, 0x2, 0x2, 0x4, 0xfe, 0x3, 0x2, 0x2, 0x2, 0x6, 0x100, 0x3, 0x2, 0x2, 0x2, 0x8, 0x1e4, 0x3, 0x2, 0x2, 0x2, 0xa, 0x1e6, 0x3, 0x2, 0x2, 0x2, 0xc, 0x1ee, 0x3, 0x2, 0x2, 0x2, 0xe, 0x1f2, 0x3, 0x2, 0x2, 0x2, 0x10, 0x1f9, 0x3, 0x2, 0x2, 0x2, 0x12, - 0x1fb, 0x3, 0x2, 0x2, 0x2, 0x14, 0x201, 0x3, 0x2, 0x2, 0x2, 0x16, 0x290, - 0x3, 0x2, 0x2, 0x2, 0x18, 0x292, 0x3, 0x2, 0x2, 0x2, 0x1a, 0x29d, 0x3, - 0x2, 0x2, 0x2, 0x1c, 0x2b8, 0x3, 0x2, 0x2, 0x2, 0x1e, 0x2d3, 0x3, 0x2, - 0x2, 0x2, 0x20, 0x2d7, 0x3, 0x2, 0x2, 0x2, 0x22, 0x2e0, 0x3, 0x2, 0x2, - 0x2, 0x24, 0x2ed, 0x3, 0x2, 0x2, 0x2, 0x26, 0x2fc, 0x3, 0x2, 0x2, 0x2, - 0x28, 0x309, 0x3, 0x2, 0x2, 0x2, 0x2a, 0x319, 0x3, 0x2, 0x2, 0x2, 0x2c, - 0x31e, 0x3, 0x2, 0x2, 0x2, 0x2e, 0x324, 0x3, 0x2, 0x2, 0x2, 0x30, 0x327, - 0x3, 0x2, 0x2, 0x2, 0x32, 0x32a, 0x3, 0x2, 0x2, 0x2, 0x34, 0x33c, 0x3, - 0x2, 0x2, 0x2, 0x36, 0x33e, 0x3, 0x2, 0x2, 0x2, 0x38, 0x35c, 0x3, 0x2, - 0x2, 0x2, 0x3a, 0x360, 0x3, 0x2, 0x2, 0x2, 0x3c, 0x364, 0x3, 0x2, 0x2, - 0x2, 0x3e, 0x368, 0x3, 0x2, 0x2, 0x2, 0x40, 0x371, 0x3, 0x2, 0x2, 0x2, - 0x42, 0x387, 0x3, 0x2, 0x2, 0x2, 0x44, 0x3a9, 0x3, 0x2, 0x2, 0x2, 0x46, - 0x3ab, 0x3, 0x2, 0x2, 0x2, 0x48, 0x3ae, 0x3, 0x2, 0x2, 0x2, 0x4a, 0x3b5, - 0x3, 0x2, 0x2, 0x2, 0x4c, 0x3b8, 0x3, 0x2, 0x2, 0x2, 0x4e, 0x3c4, 0x3, - 0x2, 0x2, 0x2, 0x50, 0x3cc, 0x3, 0x2, 0x2, 0x2, 0x52, 0x3d6, 0x3, 0x2, - 0x2, 0x2, 0x54, 0x3fb, 0x3, 0x2, 0x2, 0x2, 0x56, 0x40a, 0x3, 0x2, 0x2, - 0x2, 0x58, 0x412, 0x3, 0x2, 0x2, 0x2, 0x5a, 0x414, 0x3, 0x2, 0x2, 0x2, - 0x5c, 0x423, 0x3, 0x2, 0x2, 0x2, 0x5e, 0x437, 0x3, 0x2, 0x2, 0x2, 0x60, - 0x439, 0x3, 0x2, 0x2, 0x2, 0x62, 0x442, 0x3, 0x2, 0x2, 0x2, 0x64, 0x451, - 0x3, 0x2, 0x2, 0x2, 0x66, 0x463, 0x3, 0x2, 0x2, 0x2, 0x68, 0x471, 0x3, - 0x2, 0x2, 0x2, 0x6a, 0x47f, 0x3, 0x2, 0x2, 0x2, 0x6c, 0x482, 0x3, 0x2, - 0x2, 0x2, 0x6e, 0x4b2, 0x3, 0x2, 0x2, 0x2, 0x70, 0x4b5, 0x3, 0x2, 0x2, - 0x2, 0x72, 0x4bb, 0x3, 0x2, 0x2, 0x2, 0x74, 0x4bf, 0x3, 0x2, 0x2, 0x2, - 0x76, 0x4c5, 0x3, 0x2, 0x2, 0x2, 0x78, 0x4c8, 0x3, 0x2, 0x2, 0x2, 0x7a, - 0x4cb, 0x3, 0x2, 0x2, 0x2, 0x7c, 0x4d5, 0x3, 0x2, 0x2, 0x2, 0x7e, 0x4d8, - 0x3, 0x2, 0x2, 0x2, 0x80, 0x4dc, 0x3, 0x2, 0x2, 0x2, 0x82, 0x4e0, 0x3, - 0x2, 0x2, 0x2, 0x84, 0x4e5, 0x3, 0x2, 0x2, 0x2, 0x86, 0x4eb, 0x3, 0x2, - 0x2, 0x2, 0x88, 0x4fa, 0x3, 0x2, 0x2, 0x2, 0x8a, 0x53b, 0x3, 0x2, 0x2, - 0x2, 0x8c, 0x543, 0x3, 0x2, 0x2, 0x2, 0x8e, 0x54e, 0x3, 0x2, 0x2, 0x2, - 0x90, 0x550, 0x3, 0x2, 0x2, 0x2, 0x92, 0x556, 0x3, 0x2, 0x2, 0x2, 0x94, - 0x55b, 0x3, 0x2, 0x2, 0x2, 0x96, 0x563, 0x3, 0x2, 0x2, 0x2, 0x98, 0x56f, - 0x3, 0x2, 0x2, 0x2, 0x9a, 0x574, 0x3, 0x2, 0x2, 0x2, 0x9c, 0x57c, 0x3, - 0x2, 0x2, 0x2, 0x9e, 0x580, 0x3, 0x2, 0x2, 0x2, 0xa0, 0x5ad, 0x3, 0x2, - 0x2, 0x2, 0xa2, 0x5d1, 0x3, 0x2, 0x2, 0x2, 0xa4, 0x5d3, 0x3, 0x2, 0x2, - 0x2, 0xa6, 0x5e2, 0x3, 0x2, 0x2, 0x2, 0xa8, 0x5e5, 0x3, 0x2, 0x2, 0x2, - 0xaa, 0x61d, 0x3, 0x2, 0x2, 0x2, 0xac, 0x61f, 0x3, 0x2, 0x2, 0x2, 0xae, - 0x632, 0x3, 0x2, 0x2, 0x2, 0xb0, 0x69f, 0x3, 0x2, 0x2, 0x2, 0xb2, 0x6eb, - 0x3, 0x2, 0x2, 0x2, 0xb4, 0x6f5, 0x3, 0x2, 0x2, 0x2, 0xb6, 0x70a, 0x3, - 0x2, 0x2, 0x2, 0xb8, 0x712, 0x3, 0x2, 0x2, 0x2, 0xba, 0x716, 0x3, 0x2, - 0x2, 0x2, 0xbc, 0x722, 0x3, 0x2, 0x2, 0x2, 0xbe, 0x72f, 0x3, 0x2, 0x2, - 0x2, 0xc0, 0x739, 0x3, 0x2, 0x2, 0x2, 0xc2, 0x73d, 0x3, 0x2, 0x2, 0x2, - 0xc4, 0x748, 0x3, 0x2, 0x2, 0x2, 0xc6, 0x74a, 0x3, 0x2, 0x2, 0x2, 0xc8, - 0x754, 0x3, 0x2, 0x2, 0x2, 0xca, 0x757, 0x3, 0x2, 0x2, 0x2, 0xcc, 0x764, - 0x3, 0x2, 0x2, 0x2, 0xce, 0x766, 0x3, 0x2, 0x2, 0x2, 0xd0, 0x768, 0x3, - 0x2, 0x2, 0x2, 0xd2, 0x76a, 0x3, 0x2, 0x2, 0x2, 0xd4, 0x76e, 0x3, 0x2, - 0x2, 0x2, 0xd6, 0x773, 0x3, 0x2, 0x2, 0x2, 0xd8, 0x777, 0x3, 0x2, 0x2, - 0x2, 0xda, 0x779, 0x3, 0x2, 0x2, 0x2, 0xdc, 0xe0, 0x5, 0x4, 0x3, 0x2, + 0x1fb, 0x3, 0x2, 0x2, 0x2, 0x14, 0x201, 0x3, 0x2, 0x2, 0x2, 0x16, 0x2a0, + 0x3, 0x2, 0x2, 0x2, 0x18, 0x2a2, 0x3, 0x2, 0x2, 0x2, 0x1a, 0x2ad, 0x3, + 0x2, 0x2, 0x2, 0x1c, 0x2c8, 0x3, 0x2, 0x2, 0x2, 0x1e, 0x2e3, 0x3, 0x2, + 0x2, 0x2, 0x20, 0x2e7, 0x3, 0x2, 0x2, 0x2, 0x22, 0x2f0, 0x3, 0x2, 0x2, + 0x2, 0x24, 0x2fd, 0x3, 0x2, 0x2, 0x2, 0x26, 0x30c, 0x3, 0x2, 0x2, 0x2, + 0x28, 0x319, 0x3, 0x2, 0x2, 0x2, 0x2a, 0x329, 0x3, 0x2, 0x2, 0x2, 0x2c, + 0x32e, 0x3, 0x2, 0x2, 0x2, 0x2e, 0x334, 0x3, 0x2, 0x2, 0x2, 0x30, 0x337, + 0x3, 0x2, 0x2, 0x2, 0x32, 0x33a, 0x3, 0x2, 0x2, 0x2, 0x34, 0x34c, 0x3, + 0x2, 0x2, 0x2, 0x36, 0x34e, 0x3, 0x2, 0x2, 0x2, 0x38, 0x36c, 0x3, 0x2, + 0x2, 0x2, 0x3a, 0x370, 0x3, 0x2, 0x2, 0x2, 0x3c, 0x374, 0x3, 0x2, 0x2, + 0x2, 0x3e, 0x378, 0x3, 0x2, 0x2, 0x2, 0x40, 0x381, 0x3, 0x2, 0x2, 0x2, + 0x42, 0x397, 0x3, 0x2, 0x2, 0x2, 0x44, 0x3b9, 0x3, 0x2, 0x2, 0x2, 0x46, + 0x3bb, 0x3, 0x2, 0x2, 0x2, 0x48, 0x3be, 0x3, 0x2, 0x2, 0x2, 0x4a, 0x3c5, + 0x3, 0x2, 0x2, 0x2, 0x4c, 0x3c8, 0x3, 0x2, 0x2, 0x2, 0x4e, 0x3d4, 0x3, + 0x2, 0x2, 0x2, 0x50, 0x3dc, 0x3, 0x2, 0x2, 0x2, 0x52, 0x3e6, 0x3, 0x2, + 0x2, 0x2, 0x54, 0x40b, 0x3, 0x2, 0x2, 0x2, 0x56, 0x41a, 0x3, 0x2, 0x2, + 0x2, 0x58, 0x422, 0x3, 0x2, 0x2, 0x2, 0x5a, 0x424, 0x3, 0x2, 0x2, 0x2, + 0x5c, 0x433, 0x3, 0x2, 0x2, 0x2, 0x5e, 0x447, 0x3, 0x2, 0x2, 0x2, 0x60, + 0x449, 0x3, 0x2, 0x2, 0x2, 0x62, 0x452, 0x3, 0x2, 0x2, 0x2, 0x64, 0x461, + 0x3, 0x2, 0x2, 0x2, 0x66, 0x473, 0x3, 0x2, 0x2, 0x2, 0x68, 0x481, 0x3, + 0x2, 0x2, 0x2, 0x6a, 0x48f, 0x3, 0x2, 0x2, 0x2, 0x6c, 0x492, 0x3, 0x2, + 0x2, 0x2, 0x6e, 0x4c2, 0x3, 0x2, 0x2, 0x2, 0x70, 0x4c5, 0x3, 0x2, 0x2, + 0x2, 0x72, 0x4cb, 0x3, 0x2, 0x2, 0x2, 0x74, 0x4cf, 0x3, 0x2, 0x2, 0x2, + 0x76, 0x4d5, 0x3, 0x2, 0x2, 0x2, 0x78, 0x4d8, 0x3, 0x2, 0x2, 0x2, 0x7a, + 0x4db, 0x3, 0x2, 0x2, 0x2, 0x7c, 0x4e5, 0x3, 0x2, 0x2, 0x2, 0x7e, 0x4e8, + 0x3, 0x2, 0x2, 0x2, 0x80, 0x4ec, 0x3, 0x2, 0x2, 0x2, 0x82, 0x4f0, 0x3, + 0x2, 0x2, 0x2, 0x84, 0x4f5, 0x3, 0x2, 0x2, 0x2, 0x86, 0x4fb, 0x3, 0x2, + 0x2, 0x2, 0x88, 0x50a, 0x3, 0x2, 0x2, 0x2, 0x8a, 0x54b, 0x3, 0x2, 0x2, + 0x2, 0x8c, 0x553, 0x3, 0x2, 0x2, 0x2, 0x8e, 0x55e, 0x3, 0x2, 0x2, 0x2, + 0x90, 0x560, 0x3, 0x2, 0x2, 0x2, 0x92, 0x566, 0x3, 0x2, 0x2, 0x2, 0x94, + 0x56b, 0x3, 0x2, 0x2, 0x2, 0x96, 0x573, 0x3, 0x2, 0x2, 0x2, 0x98, 0x57f, + 0x3, 0x2, 0x2, 0x2, 0x9a, 0x584, 0x3, 0x2, 0x2, 0x2, 0x9c, 0x58c, 0x3, + 0x2, 0x2, 0x2, 0x9e, 0x590, 0x3, 0x2, 0x2, 0x2, 0xa0, 0x5bd, 0x3, 0x2, + 0x2, 0x2, 0xa2, 0x5e1, 0x3, 0x2, 0x2, 0x2, 0xa4, 0x5e3, 0x3, 0x2, 0x2, + 0x2, 0xa6, 0x5f2, 0x3, 0x2, 0x2, 0x2, 0xa8, 0x5f5, 0x3, 0x2, 0x2, 0x2, + 0xaa, 0x62d, 0x3, 0x2, 0x2, 0x2, 0xac, 0x62f, 0x3, 0x2, 0x2, 0x2, 0xae, + 0x642, 0x3, 0x2, 0x2, 0x2, 0xb0, 0x6af, 0x3, 0x2, 0x2, 0x2, 0xb2, 0x6fb, + 0x3, 0x2, 0x2, 0x2, 0xb4, 0x705, 0x3, 0x2, 0x2, 0x2, 0xb6, 0x71a, 0x3, + 0x2, 0x2, 0x2, 0xb8, 0x722, 0x3, 0x2, 0x2, 0x2, 0xba, 0x726, 0x3, 0x2, + 0x2, 0x2, 0xbc, 0x732, 0x3, 0x2, 0x2, 0x2, 0xbe, 0x73f, 0x3, 0x2, 0x2, + 0x2, 0xc0, 0x749, 0x3, 0x2, 0x2, 0x2, 0xc2, 0x74d, 0x3, 0x2, 0x2, 0x2, + 0xc4, 0x758, 0x3, 0x2, 0x2, 0x2, 0xc6, 0x75a, 0x3, 0x2, 0x2, 0x2, 0xc8, + 0x764, 0x3, 0x2, 0x2, 0x2, 0xca, 0x767, 0x3, 0x2, 0x2, 0x2, 0xcc, 0x774, + 0x3, 0x2, 0x2, 0x2, 0xce, 0x776, 0x3, 0x2, 0x2, 0x2, 0xd0, 0x778, 0x3, + 0x2, 0x2, 0x2, 0xd2, 0x77a, 0x3, 0x2, 0x2, 0x2, 0xd4, 0x77e, 0x3, 0x2, + 0x2, 0x2, 0xd6, 0x783, 0x3, 0x2, 0x2, 0x2, 0xd8, 0x787, 0x3, 0x2, 0x2, + 0x2, 0xda, 0x789, 0x3, 0x2, 0x2, 0x2, 0xdc, 0xe0, 0x5, 0x4, 0x3, 0x2, 0xdd, 0xde, 0x7, 0x56, 0x2, 0x2, 0xde, 0xdf, 0x7, 0x7b, 0x2, 0x2, 0xdf, 0xe1, 0x7, 0xbf, 0x2, 0x2, 0xe0, 0xdd, 0x3, 0x2, 0x2, 0x2, 0xe0, 0xe1, 0x3, 0x2, 0x2, 0x2, 0xe1, 0xe4, 0x3, 0x2, 0x2, 0x2, 0xe2, 0xe3, 0x7, @@ -19341,870 +19407,881 @@ ClickHouseParser::Initializer::Initializer() { 0x2c, 0x17, 0x2, 0x210, 0x20f, 0x3, 0x2, 0x2, 0x2, 0x210, 0x211, 0x3, 0x2, 0x2, 0x2, 0x211, 0x213, 0x3, 0x2, 0x2, 0x2, 0x212, 0x214, 0x5, 0x40, 0x21, 0x2, 0x213, 0x212, 0x3, 0x2, 0x2, 0x2, 0x213, 0x214, 0x3, - 0x2, 0x2, 0x2, 0x214, 0x291, 0x3, 0x2, 0x2, 0x2, 0x215, 0x216, 0x9, - 0x3, 0x2, 0x2, 0x216, 0x21a, 0x7, 0x2f, 0x2, 0x2, 0x217, 0x218, 0x7, - 0x4d, 0x2, 0x2, 0x218, 0x219, 0x7, 0x72, 0x2, 0x2, 0x219, 0x21b, 0x7, - 0x38, 0x2, 0x2, 0x21a, 0x217, 0x3, 0x2, 0x2, 0x2, 0x21a, 0x21b, 0x3, - 0x2, 0x2, 0x2, 0x21b, 0x21c, 0x3, 0x2, 0x2, 0x2, 0x21c, 0x21e, 0x5, - 0xc0, 0x61, 0x2, 0x21d, 0x21f, 0x5, 0x2e, 0x18, 0x2, 0x21e, 0x21d, 0x3, - 0x2, 0x2, 0x2, 0x21e, 0x21f, 0x3, 0x2, 0x2, 0x2, 0x21f, 0x221, 0x3, - 0x2, 0x2, 0x2, 0x220, 0x222, 0x5, 0x2c, 0x17, 0x2, 0x221, 0x220, 0x3, - 0x2, 0x2, 0x2, 0x221, 0x222, 0x3, 0x2, 0x2, 0x2, 0x222, 0x223, 0x3, - 0x2, 0x2, 0x2, 0x223, 0x224, 0x5, 0x18, 0xd, 0x2, 0x224, 0x225, 0x5, - 0x1c, 0xf, 0x2, 0x225, 0x291, 0x3, 0x2, 0x2, 0x2, 0x226, 0x227, 0x9, - 0x3, 0x2, 0x2, 0x227, 0x228, 0x7, 0x63, 0x2, 0x2, 0x228, 0x22c, 0x7, - 0xb0, 0x2, 0x2, 0x229, 0x22a, 0x7, 0x4d, 0x2, 0x2, 0x22a, 0x22b, 0x7, - 0x72, 0x2, 0x2, 0x22b, 0x22d, 0x7, 0x38, 0x2, 0x2, 0x22c, 0x229, 0x3, - 0x2, 0x2, 0x2, 0x22c, 0x22d, 0x3, 0x2, 0x2, 0x2, 0x22d, 0x22e, 0x3, - 0x2, 0x2, 0x2, 0x22e, 0x230, 0x5, 0xc0, 0x61, 0x2, 0x22f, 0x231, 0x5, - 0x2e, 0x18, 0x2, 0x230, 0x22f, 0x3, 0x2, 0x2, 0x2, 0x230, 0x231, 0x3, - 0x2, 0x2, 0x2, 0x231, 0x233, 0x3, 0x2, 0x2, 0x2, 0x232, 0x234, 0x5, - 0x2c, 0x17, 0x2, 0x233, 0x232, 0x3, 0x2, 0x2, 0x2, 0x233, 0x234, 0x3, - 0x2, 0x2, 0x2, 0x234, 0x23a, 0x3, 0x2, 0x2, 0x2, 0x235, 0x236, 0x7, - 0xb6, 0x2, 0x2, 0x236, 0x238, 0x7, 0xa0, 0x2, 0x2, 0x237, 0x239, 0x7, - 0xbd, 0x2, 0x2, 0x238, 0x237, 0x3, 0x2, 0x2, 0x2, 0x238, 0x239, 0x3, - 0x2, 0x2, 0x2, 0x239, 0x23b, 0x3, 0x2, 0x2, 0x2, 0x23a, 0x235, 0x3, - 0x2, 0x2, 0x2, 0x23a, 0x23b, 0x3, 0x2, 0x2, 0x2, 0x23b, 0x23d, 0x3, - 0x2, 0x2, 0x2, 0x23c, 0x23e, 0x5, 0x30, 0x19, 0x2, 0x23d, 0x23c, 0x3, - 0x2, 0x2, 0x2, 0x23d, 0x23e, 0x3, 0x2, 0x2, 0x2, 0x23e, 0x240, 0x3, - 0x2, 0x2, 0x2, 0x23f, 0x241, 0x5, 0x34, 0x1b, 0x2, 0x240, 0x23f, 0x3, - 0x2, 0x2, 0x2, 0x240, 0x241, 0x3, 0x2, 0x2, 0x2, 0x241, 0x242, 0x3, - 0x2, 0x2, 0x2, 0x242, 0x243, 0x5, 0x32, 0x1a, 0x2, 0x243, 0x291, 0x3, - 0x2, 0x2, 0x2, 0x244, 0x245, 0x9, 0x3, 0x2, 0x2, 0x245, 0x246, 0x7, - 0x67, 0x2, 0x2, 0x246, 0x24a, 0x7, 0xb0, 0x2, 0x2, 0x247, 0x248, 0x7, - 0x4d, 0x2, 0x2, 0x248, 0x249, 0x7, 0x72, 0x2, 0x2, 0x249, 0x24b, 0x7, - 0x38, 0x2, 0x2, 0x24a, 0x247, 0x3, 0x2, 0x2, 0x2, 0x24a, 0x24b, 0x3, - 0x2, 0x2, 0x2, 0x24b, 0x24c, 0x3, 0x2, 0x2, 0x2, 0x24c, 0x24e, 0x5, - 0xc0, 0x61, 0x2, 0x24d, 0x24f, 0x5, 0x2e, 0x18, 0x2, 0x24e, 0x24d, 0x3, - 0x2, 0x2, 0x2, 0x24e, 0x24f, 0x3, 0x2, 0x2, 0x2, 0x24f, 0x251, 0x3, - 0x2, 0x2, 0x2, 0x250, 0x252, 0x5, 0x2c, 0x17, 0x2, 0x251, 0x250, 0x3, - 0x2, 0x2, 0x2, 0x251, 0x252, 0x3, 0x2, 0x2, 0x2, 0x252, 0x254, 0x3, - 0x2, 0x2, 0x2, 0x253, 0x255, 0x5, 0x34, 0x1b, 0x2, 0x254, 0x253, 0x3, - 0x2, 0x2, 0x2, 0x254, 0x255, 0x3, 0x2, 0x2, 0x2, 0x255, 0x25b, 0x3, - 0x2, 0x2, 0x2, 0x256, 0x25c, 0x5, 0x30, 0x19, 0x2, 0x257, 0x259, 0x5, - 0x36, 0x1c, 0x2, 0x258, 0x25a, 0x7, 0x7d, 0x2, 0x2, 0x259, 0x258, 0x3, - 0x2, 0x2, 0x2, 0x259, 0x25a, 0x3, 0x2, 0x2, 0x2, 0x25a, 0x25c, 0x3, - 0x2, 0x2, 0x2, 0x25b, 0x256, 0x3, 0x2, 0x2, 0x2, 0x25b, 0x257, 0x3, - 0x2, 0x2, 0x2, 0x25c, 0x25d, 0x3, 0x2, 0x2, 0x2, 0x25d, 0x25e, 0x5, - 0x32, 0x1a, 0x2, 0x25e, 0x291, 0x3, 0x2, 0x2, 0x2, 0x25f, 0x261, 0x9, - 0x3, 0x2, 0x2, 0x260, 0x262, 0x7, 0x9c, 0x2, 0x2, 0x261, 0x260, 0x3, - 0x2, 0x2, 0x2, 0x261, 0x262, 0x3, 0x2, 0x2, 0x2, 0x262, 0x263, 0x3, - 0x2, 0x2, 0x2, 0x263, 0x267, 0x7, 0x9a, 0x2, 0x2, 0x264, 0x265, 0x7, - 0x4d, 0x2, 0x2, 0x265, 0x266, 0x7, 0x72, 0x2, 0x2, 0x266, 0x268, 0x7, - 0x38, 0x2, 0x2, 0x267, 0x264, 0x3, 0x2, 0x2, 0x2, 0x267, 0x268, 0x3, - 0x2, 0x2, 0x2, 0x268, 0x269, 0x3, 0x2, 0x2, 0x2, 0x269, 0x26b, 0x5, - 0xc0, 0x61, 0x2, 0x26a, 0x26c, 0x5, 0x2e, 0x18, 0x2, 0x26b, 0x26a, 0x3, - 0x2, 0x2, 0x2, 0x26b, 0x26c, 0x3, 0x2, 0x2, 0x2, 0x26c, 0x26e, 0x3, - 0x2, 0x2, 0x2, 0x26d, 0x26f, 0x5, 0x2c, 0x17, 0x2, 0x26e, 0x26d, 0x3, - 0x2, 0x2, 0x2, 0x26e, 0x26f, 0x3, 0x2, 0x2, 0x2, 0x26f, 0x271, 0x3, - 0x2, 0x2, 0x2, 0x270, 0x272, 0x5, 0x34, 0x1b, 0x2, 0x271, 0x270, 0x3, - 0x2, 0x2, 0x2, 0x271, 0x272, 0x3, 0x2, 0x2, 0x2, 0x272, 0x274, 0x3, - 0x2, 0x2, 0x2, 0x273, 0x275, 0x5, 0x36, 0x1c, 0x2, 0x274, 0x273, 0x3, - 0x2, 0x2, 0x2, 0x274, 0x275, 0x3, 0x2, 0x2, 0x2, 0x275, 0x277, 0x3, - 0x2, 0x2, 0x2, 0x276, 0x278, 0x5, 0x32, 0x1a, 0x2, 0x277, 0x276, 0x3, - 0x2, 0x2, 0x2, 0x277, 0x278, 0x3, 0x2, 0x2, 0x2, 0x278, 0x291, 0x3, - 0x2, 0x2, 0x2, 0x279, 0x27c, 0x9, 0x3, 0x2, 0x2, 0x27a, 0x27b, 0x7, - 0x78, 0x2, 0x2, 0x27b, 0x27d, 0x7, 0x86, 0x2, 0x2, 0x27c, 0x27a, 0x3, - 0x2, 0x2, 0x2, 0x27c, 0x27d, 0x3, 0x2, 0x2, 0x2, 0x27d, 0x27e, 0x3, - 0x2, 0x2, 0x2, 0x27e, 0x282, 0x7, 0xb0, 0x2, 0x2, 0x27f, 0x280, 0x7, - 0x4d, 0x2, 0x2, 0x280, 0x281, 0x7, 0x72, 0x2, 0x2, 0x281, 0x283, 0x7, - 0x38, 0x2, 0x2, 0x282, 0x27f, 0x3, 0x2, 0x2, 0x2, 0x282, 0x283, 0x3, - 0x2, 0x2, 0x2, 0x283, 0x284, 0x3, 0x2, 0x2, 0x2, 0x284, 0x286, 0x5, - 0xc0, 0x61, 0x2, 0x285, 0x287, 0x5, 0x2e, 0x18, 0x2, 0x286, 0x285, 0x3, - 0x2, 0x2, 0x2, 0x286, 0x287, 0x3, 0x2, 0x2, 0x2, 0x287, 0x289, 0x3, - 0x2, 0x2, 0x2, 0x288, 0x28a, 0x5, 0x2c, 0x17, 0x2, 0x289, 0x288, 0x3, - 0x2, 0x2, 0x2, 0x289, 0x28a, 0x3, 0x2, 0x2, 0x2, 0x28a, 0x28c, 0x3, - 0x2, 0x2, 0x2, 0x28b, 0x28d, 0x5, 0x34, 0x1b, 0x2, 0x28c, 0x28b, 0x3, + 0x2, 0x2, 0x2, 0x214, 0x2a1, 0x3, 0x2, 0x2, 0x2, 0x215, 0x21d, 0x7, + 0x11, 0x2, 0x2, 0x216, 0x219, 0x7, 0x1f, 0x2, 0x2, 0x217, 0x218, 0x7, + 0x78, 0x2, 0x2, 0x218, 0x21a, 0x7, 0x86, 0x2, 0x2, 0x219, 0x217, 0x3, + 0x2, 0x2, 0x2, 0x219, 0x21a, 0x3, 0x2, 0x2, 0x2, 0x21a, 0x21d, 0x3, + 0x2, 0x2, 0x2, 0x21b, 0x21d, 0x7, 0x86, 0x2, 0x2, 0x21c, 0x215, 0x3, + 0x2, 0x2, 0x2, 0x21c, 0x216, 0x3, 0x2, 0x2, 0x2, 0x21c, 0x21b, 0x3, + 0x2, 0x2, 0x2, 0x21d, 0x21e, 0x3, 0x2, 0x2, 0x2, 0x21e, 0x222, 0x7, + 0x2f, 0x2, 0x2, 0x21f, 0x220, 0x7, 0x4d, 0x2, 0x2, 0x220, 0x221, 0x7, + 0x72, 0x2, 0x2, 0x221, 0x223, 0x7, 0x38, 0x2, 0x2, 0x222, 0x21f, 0x3, + 0x2, 0x2, 0x2, 0x222, 0x223, 0x3, 0x2, 0x2, 0x2, 0x223, 0x224, 0x3, + 0x2, 0x2, 0x2, 0x224, 0x226, 0x5, 0xc0, 0x61, 0x2, 0x225, 0x227, 0x5, + 0x2e, 0x18, 0x2, 0x226, 0x225, 0x3, 0x2, 0x2, 0x2, 0x226, 0x227, 0x3, + 0x2, 0x2, 0x2, 0x227, 0x229, 0x3, 0x2, 0x2, 0x2, 0x228, 0x22a, 0x5, + 0x2c, 0x17, 0x2, 0x229, 0x228, 0x3, 0x2, 0x2, 0x2, 0x229, 0x22a, 0x3, + 0x2, 0x2, 0x2, 0x22a, 0x22b, 0x3, 0x2, 0x2, 0x2, 0x22b, 0x22c, 0x5, + 0x18, 0xd, 0x2, 0x22c, 0x22d, 0x5, 0x1c, 0xf, 0x2, 0x22d, 0x2a1, 0x3, + 0x2, 0x2, 0x2, 0x22e, 0x22f, 0x9, 0x3, 0x2, 0x2, 0x22f, 0x230, 0x7, + 0x63, 0x2, 0x2, 0x230, 0x234, 0x7, 0xb0, 0x2, 0x2, 0x231, 0x232, 0x7, + 0x4d, 0x2, 0x2, 0x232, 0x233, 0x7, 0x72, 0x2, 0x2, 0x233, 0x235, 0x7, + 0x38, 0x2, 0x2, 0x234, 0x231, 0x3, 0x2, 0x2, 0x2, 0x234, 0x235, 0x3, + 0x2, 0x2, 0x2, 0x235, 0x236, 0x3, 0x2, 0x2, 0x2, 0x236, 0x238, 0x5, + 0xc0, 0x61, 0x2, 0x237, 0x239, 0x5, 0x2e, 0x18, 0x2, 0x238, 0x237, 0x3, + 0x2, 0x2, 0x2, 0x238, 0x239, 0x3, 0x2, 0x2, 0x2, 0x239, 0x23b, 0x3, + 0x2, 0x2, 0x2, 0x23a, 0x23c, 0x5, 0x2c, 0x17, 0x2, 0x23b, 0x23a, 0x3, + 0x2, 0x2, 0x2, 0x23b, 0x23c, 0x3, 0x2, 0x2, 0x2, 0x23c, 0x242, 0x3, + 0x2, 0x2, 0x2, 0x23d, 0x23e, 0x7, 0xb6, 0x2, 0x2, 0x23e, 0x240, 0x7, + 0xa0, 0x2, 0x2, 0x23f, 0x241, 0x7, 0xbd, 0x2, 0x2, 0x240, 0x23f, 0x3, + 0x2, 0x2, 0x2, 0x240, 0x241, 0x3, 0x2, 0x2, 0x2, 0x241, 0x243, 0x3, + 0x2, 0x2, 0x2, 0x242, 0x23d, 0x3, 0x2, 0x2, 0x2, 0x242, 0x243, 0x3, + 0x2, 0x2, 0x2, 0x243, 0x245, 0x3, 0x2, 0x2, 0x2, 0x244, 0x246, 0x5, + 0x30, 0x19, 0x2, 0x245, 0x244, 0x3, 0x2, 0x2, 0x2, 0x245, 0x246, 0x3, + 0x2, 0x2, 0x2, 0x246, 0x248, 0x3, 0x2, 0x2, 0x2, 0x247, 0x249, 0x5, + 0x34, 0x1b, 0x2, 0x248, 0x247, 0x3, 0x2, 0x2, 0x2, 0x248, 0x249, 0x3, + 0x2, 0x2, 0x2, 0x249, 0x24a, 0x3, 0x2, 0x2, 0x2, 0x24a, 0x24b, 0x5, + 0x32, 0x1a, 0x2, 0x24b, 0x2a1, 0x3, 0x2, 0x2, 0x2, 0x24c, 0x24d, 0x9, + 0x3, 0x2, 0x2, 0x24d, 0x24e, 0x7, 0x67, 0x2, 0x2, 0x24e, 0x252, 0x7, + 0xb0, 0x2, 0x2, 0x24f, 0x250, 0x7, 0x4d, 0x2, 0x2, 0x250, 0x251, 0x7, + 0x72, 0x2, 0x2, 0x251, 0x253, 0x7, 0x38, 0x2, 0x2, 0x252, 0x24f, 0x3, + 0x2, 0x2, 0x2, 0x252, 0x253, 0x3, 0x2, 0x2, 0x2, 0x253, 0x254, 0x3, + 0x2, 0x2, 0x2, 0x254, 0x256, 0x5, 0xc0, 0x61, 0x2, 0x255, 0x257, 0x5, + 0x2e, 0x18, 0x2, 0x256, 0x255, 0x3, 0x2, 0x2, 0x2, 0x256, 0x257, 0x3, + 0x2, 0x2, 0x2, 0x257, 0x259, 0x3, 0x2, 0x2, 0x2, 0x258, 0x25a, 0x5, + 0x2c, 0x17, 0x2, 0x259, 0x258, 0x3, 0x2, 0x2, 0x2, 0x259, 0x25a, 0x3, + 0x2, 0x2, 0x2, 0x25a, 0x25c, 0x3, 0x2, 0x2, 0x2, 0x25b, 0x25d, 0x5, + 0x34, 0x1b, 0x2, 0x25c, 0x25b, 0x3, 0x2, 0x2, 0x2, 0x25c, 0x25d, 0x3, + 0x2, 0x2, 0x2, 0x25d, 0x263, 0x3, 0x2, 0x2, 0x2, 0x25e, 0x264, 0x5, + 0x30, 0x19, 0x2, 0x25f, 0x261, 0x5, 0x36, 0x1c, 0x2, 0x260, 0x262, 0x7, + 0x7d, 0x2, 0x2, 0x261, 0x260, 0x3, 0x2, 0x2, 0x2, 0x261, 0x262, 0x3, + 0x2, 0x2, 0x2, 0x262, 0x264, 0x3, 0x2, 0x2, 0x2, 0x263, 0x25e, 0x3, + 0x2, 0x2, 0x2, 0x263, 0x25f, 0x3, 0x2, 0x2, 0x2, 0x264, 0x265, 0x3, + 0x2, 0x2, 0x2, 0x265, 0x266, 0x5, 0x32, 0x1a, 0x2, 0x266, 0x2a1, 0x3, + 0x2, 0x2, 0x2, 0x267, 0x26f, 0x7, 0x11, 0x2, 0x2, 0x268, 0x26b, 0x7, + 0x1f, 0x2, 0x2, 0x269, 0x26a, 0x7, 0x78, 0x2, 0x2, 0x26a, 0x26c, 0x7, + 0x86, 0x2, 0x2, 0x26b, 0x269, 0x3, 0x2, 0x2, 0x2, 0x26b, 0x26c, 0x3, + 0x2, 0x2, 0x2, 0x26c, 0x26f, 0x3, 0x2, 0x2, 0x2, 0x26d, 0x26f, 0x7, + 0x86, 0x2, 0x2, 0x26e, 0x267, 0x3, 0x2, 0x2, 0x2, 0x26e, 0x268, 0x3, + 0x2, 0x2, 0x2, 0x26e, 0x26d, 0x3, 0x2, 0x2, 0x2, 0x26f, 0x271, 0x3, + 0x2, 0x2, 0x2, 0x270, 0x272, 0x7, 0x9c, 0x2, 0x2, 0x271, 0x270, 0x3, + 0x2, 0x2, 0x2, 0x271, 0x272, 0x3, 0x2, 0x2, 0x2, 0x272, 0x273, 0x3, + 0x2, 0x2, 0x2, 0x273, 0x277, 0x7, 0x9a, 0x2, 0x2, 0x274, 0x275, 0x7, + 0x4d, 0x2, 0x2, 0x275, 0x276, 0x7, 0x72, 0x2, 0x2, 0x276, 0x278, 0x7, + 0x38, 0x2, 0x2, 0x277, 0x274, 0x3, 0x2, 0x2, 0x2, 0x277, 0x278, 0x3, + 0x2, 0x2, 0x2, 0x278, 0x279, 0x3, 0x2, 0x2, 0x2, 0x279, 0x27b, 0x5, + 0xc0, 0x61, 0x2, 0x27a, 0x27c, 0x5, 0x2e, 0x18, 0x2, 0x27b, 0x27a, 0x3, + 0x2, 0x2, 0x2, 0x27b, 0x27c, 0x3, 0x2, 0x2, 0x2, 0x27c, 0x27e, 0x3, + 0x2, 0x2, 0x2, 0x27d, 0x27f, 0x5, 0x2c, 0x17, 0x2, 0x27e, 0x27d, 0x3, + 0x2, 0x2, 0x2, 0x27e, 0x27f, 0x3, 0x2, 0x2, 0x2, 0x27f, 0x281, 0x3, + 0x2, 0x2, 0x2, 0x280, 0x282, 0x5, 0x34, 0x1b, 0x2, 0x281, 0x280, 0x3, + 0x2, 0x2, 0x2, 0x281, 0x282, 0x3, 0x2, 0x2, 0x2, 0x282, 0x284, 0x3, + 0x2, 0x2, 0x2, 0x283, 0x285, 0x5, 0x36, 0x1c, 0x2, 0x284, 0x283, 0x3, + 0x2, 0x2, 0x2, 0x284, 0x285, 0x3, 0x2, 0x2, 0x2, 0x285, 0x287, 0x3, + 0x2, 0x2, 0x2, 0x286, 0x288, 0x5, 0x32, 0x1a, 0x2, 0x287, 0x286, 0x3, + 0x2, 0x2, 0x2, 0x287, 0x288, 0x3, 0x2, 0x2, 0x2, 0x288, 0x2a1, 0x3, + 0x2, 0x2, 0x2, 0x289, 0x28c, 0x9, 0x3, 0x2, 0x2, 0x28a, 0x28b, 0x7, + 0x78, 0x2, 0x2, 0x28b, 0x28d, 0x7, 0x86, 0x2, 0x2, 0x28c, 0x28a, 0x3, 0x2, 0x2, 0x2, 0x28c, 0x28d, 0x3, 0x2, 0x2, 0x2, 0x28d, 0x28e, 0x3, - 0x2, 0x2, 0x2, 0x28e, 0x28f, 0x5, 0x32, 0x1a, 0x2, 0x28f, 0x291, 0x3, - 0x2, 0x2, 0x2, 0x290, 0x207, 0x3, 0x2, 0x2, 0x2, 0x290, 0x215, 0x3, - 0x2, 0x2, 0x2, 0x290, 0x226, 0x3, 0x2, 0x2, 0x2, 0x290, 0x244, 0x3, - 0x2, 0x2, 0x2, 0x290, 0x25f, 0x3, 0x2, 0x2, 0x2, 0x290, 0x279, 0x3, - 0x2, 0x2, 0x2, 0x291, 0x17, 0x3, 0x2, 0x2, 0x2, 0x292, 0x293, 0x7, 0xd0, - 0x2, 0x2, 0x293, 0x298, 0x5, 0x1a, 0xe, 0x2, 0x294, 0x295, 0x7, 0xc5, - 0x2, 0x2, 0x295, 0x297, 0x5, 0x1a, 0xe, 0x2, 0x296, 0x294, 0x3, 0x2, - 0x2, 0x2, 0x297, 0x29a, 0x3, 0x2, 0x2, 0x2, 0x298, 0x296, 0x3, 0x2, - 0x2, 0x2, 0x298, 0x299, 0x3, 0x2, 0x2, 0x2, 0x299, 0x29b, 0x3, 0x2, - 0x2, 0x2, 0x29a, 0x298, 0x3, 0x2, 0x2, 0x2, 0x29b, 0x29c, 0x7, 0xda, - 0x2, 0x2, 0x29c, 0x19, 0x3, 0x2, 0x2, 0x2, 0x29d, 0x29e, 0x5, 0xd6, - 0x6c, 0x2, 0x29e, 0x2b4, 0x5, 0xaa, 0x56, 0x2, 0x29f, 0x2a0, 0x6, 0xe, - 0x2, 0x3, 0x2a0, 0x2a1, 0x7, 0x27, 0x2, 0x2, 0x2a1, 0x2a2, 0x5, 0xcc, - 0x67, 0x2, 0x2a2, 0x2a3, 0x8, 0xe, 0x1, 0x2, 0x2a3, 0x2b3, 0x3, 0x2, - 0x2, 0x2, 0x2a4, 0x2a5, 0x6, 0xe, 0x3, 0x3, 0x2a5, 0x2a6, 0x7, 0x3a, - 0x2, 0x2, 0x2a6, 0x2a7, 0x5, 0xb0, 0x59, 0x2, 0x2a7, 0x2a8, 0x8, 0xe, - 0x1, 0x2, 0x2a8, 0x2b3, 0x3, 0x2, 0x2, 0x2, 0x2a9, 0x2aa, 0x6, 0xe, - 0x4, 0x3, 0x2aa, 0x2ab, 0x7, 0x4a, 0x2, 0x2, 0x2ab, 0x2b3, 0x8, 0xe, - 0x1, 0x2, 0x2ac, 0x2ad, 0x6, 0xe, 0x5, 0x3, 0x2ad, 0x2ae, 0x7, 0x52, - 0x2, 0x2, 0x2ae, 0x2b3, 0x8, 0xe, 0x1, 0x2, 0x2af, 0x2b0, 0x6, 0xe, - 0x6, 0x3, 0x2b0, 0x2b1, 0x7, 0x58, 0x2, 0x2, 0x2b1, 0x2b3, 0x8, 0xe, - 0x1, 0x2, 0x2b2, 0x29f, 0x3, 0x2, 0x2, 0x2, 0x2b2, 0x2a4, 0x3, 0x2, - 0x2, 0x2, 0x2b2, 0x2a9, 0x3, 0x2, 0x2, 0x2, 0x2b2, 0x2ac, 0x3, 0x2, - 0x2, 0x2, 0x2b2, 0x2af, 0x3, 0x2, 0x2, 0x2, 0x2b3, 0x2b6, 0x3, 0x2, - 0x2, 0x2, 0x2b4, 0x2b2, 0x3, 0x2, 0x2, 0x2, 0x2b4, 0x2b5, 0x3, 0x2, - 0x2, 0x2, 0x2b5, 0x1b, 0x3, 0x2, 0x2, 0x2, 0x2b6, 0x2b4, 0x3, 0x2, 0x2, - 0x2, 0x2b7, 0x2b9, 0x5, 0x1e, 0x10, 0x2, 0x2b8, 0x2b7, 0x3, 0x2, 0x2, - 0x2, 0x2b8, 0x2b9, 0x3, 0x2, 0x2, 0x2, 0x2b9, 0x2d0, 0x3, 0x2, 0x2, - 0x2, 0x2ba, 0x2bb, 0x6, 0xf, 0x7, 0x3, 0x2bb, 0x2bc, 0x5, 0x22, 0x12, - 0x2, 0x2bc, 0x2bd, 0x8, 0xf, 0x1, 0x2, 0x2bd, 0x2cf, 0x3, 0x2, 0x2, - 0x2, 0x2be, 0x2bf, 0x6, 0xf, 0x8, 0x3, 0x2bf, 0x2c0, 0x5, 0x24, 0x13, - 0x2, 0x2c0, 0x2c1, 0x8, 0xf, 0x1, 0x2, 0x2c1, 0x2cf, 0x3, 0x2, 0x2, - 0x2, 0x2c2, 0x2c3, 0x6, 0xf, 0x9, 0x3, 0x2c3, 0x2c4, 0x5, 0x26, 0x14, - 0x2, 0x2c4, 0x2c5, 0x8, 0xf, 0x1, 0x2, 0x2c5, 0x2cf, 0x3, 0x2, 0x2, - 0x2, 0x2c6, 0x2c7, 0x6, 0xf, 0xa, 0x3, 0x2c7, 0x2c8, 0x5, 0x28, 0x15, - 0x2, 0x2c8, 0x2c9, 0x8, 0xf, 0x1, 0x2, 0x2c9, 0x2cf, 0x3, 0x2, 0x2, - 0x2, 0x2ca, 0x2cb, 0x6, 0xf, 0xb, 0x3, 0x2cb, 0x2cc, 0x5, 0x2a, 0x16, - 0x2, 0x2cc, 0x2cd, 0x8, 0xf, 0x1, 0x2, 0x2cd, 0x2cf, 0x3, 0x2, 0x2, - 0x2, 0x2ce, 0x2ba, 0x3, 0x2, 0x2, 0x2, 0x2ce, 0x2be, 0x3, 0x2, 0x2, - 0x2, 0x2ce, 0x2c2, 0x3, 0x2, 0x2, 0x2, 0x2ce, 0x2c6, 0x3, 0x2, 0x2, - 0x2, 0x2ce, 0x2ca, 0x3, 0x2, 0x2, 0x2, 0x2cf, 0x2d2, 0x3, 0x2, 0x2, - 0x2, 0x2d0, 0x2ce, 0x3, 0x2, 0x2, 0x2, 0x2d0, 0x2d1, 0x3, 0x2, 0x2, - 0x2, 0x2d1, 0x1d, 0x3, 0x2, 0x2, 0x2, 0x2d2, 0x2d0, 0x3, 0x2, 0x2, 0x2, - 0x2d3, 0x2d4, 0x7, 0x7f, 0x2, 0x2, 0x2d4, 0x2d5, 0x7, 0x5a, 0x2, 0x2, - 0x2d5, 0x2d6, 0x5, 0xac, 0x57, 0x2, 0x2d6, 0x1f, 0x3, 0x2, 0x2, 0x2, - 0x2d7, 0x2de, 0x5, 0xd6, 0x6c, 0x2, 0x2d8, 0x2db, 0x5, 0xd6, 0x6c, 0x2, - 0x2d9, 0x2da, 0x7, 0xd0, 0x2, 0x2, 0x2da, 0x2dc, 0x7, 0xda, 0x2, 0x2, - 0x2db, 0x2d9, 0x3, 0x2, 0x2, 0x2, 0x2db, 0x2dc, 0x3, 0x2, 0x2, 0x2, - 0x2dc, 0x2df, 0x3, 0x2, 0x2, 0x2, 0x2dd, 0x2df, 0x5, 0xcc, 0x67, 0x2, - 0x2de, 0x2d8, 0x3, 0x2, 0x2, 0x2, 0x2de, 0x2dd, 0x3, 0x2, 0x2, 0x2, - 0x2df, 0x21, 0x3, 0x2, 0x2, 0x2, 0x2e0, 0x2e1, 0x7, 0x93, 0x2, 0x2, - 0x2e1, 0x2e2, 0x7, 0xd0, 0x2, 0x2, 0x2e2, 0x2e3, 0x5, 0xd6, 0x6c, 0x2, - 0x2e3, 0x2e7, 0x7, 0xd0, 0x2, 0x2, 0x2e4, 0x2e6, 0x5, 0x20, 0x11, 0x2, - 0x2e5, 0x2e4, 0x3, 0x2, 0x2, 0x2, 0x2e6, 0x2e9, 0x3, 0x2, 0x2, 0x2, - 0x2e7, 0x2e5, 0x3, 0x2, 0x2, 0x2, 0x2e7, 0x2e8, 0x3, 0x2, 0x2, 0x2, - 0x2e8, 0x2ea, 0x3, 0x2, 0x2, 0x2, 0x2e9, 0x2e7, 0x3, 0x2, 0x2, 0x2, - 0x2ea, 0x2eb, 0x7, 0xda, 0x2, 0x2, 0x2eb, 0x2ec, 0x7, 0xda, 0x2, 0x2, - 0x2ec, 0x23, 0x3, 0x2, 0x2, 0x2, 0x2ed, 0x2ee, 0x7, 0x60, 0x2, 0x2, - 0x2ee, 0x2f8, 0x7, 0xd0, 0x2, 0x2, 0x2ef, 0x2f9, 0x7, 0xbd, 0x2, 0x2, - 0x2f0, 0x2f1, 0x7, 0x6a, 0x2, 0x2, 0x2f1, 0x2f2, 0x7, 0xbd, 0x2, 0x2, - 0x2f2, 0x2f3, 0x7, 0x68, 0x2, 0x2, 0x2f3, 0x2f9, 0x7, 0xbd, 0x2, 0x2, - 0x2f4, 0x2f5, 0x7, 0x68, 0x2, 0x2, 0x2f5, 0x2f6, 0x7, 0xbd, 0x2, 0x2, - 0x2f6, 0x2f7, 0x7, 0x6a, 0x2, 0x2, 0x2f7, 0x2f9, 0x7, 0xbd, 0x2, 0x2, - 0x2f8, 0x2ef, 0x3, 0x2, 0x2, 0x2, 0x2f8, 0x2f0, 0x3, 0x2, 0x2, 0x2, - 0x2f8, 0x2f4, 0x3, 0x2, 0x2, 0x2, 0x2f9, 0x2fa, 0x3, 0x2, 0x2, 0x2, - 0x2fa, 0x2fb, 0x7, 0xda, 0x2, 0x2, 0x2fb, 0x25, 0x3, 0x2, 0x2, 0x2, - 0x2fc, 0x2fd, 0x7, 0x5d, 0x2, 0x2, 0x2fd, 0x2fe, 0x7, 0xd0, 0x2, 0x2, - 0x2fe, 0x2ff, 0x5, 0xd6, 0x6c, 0x2, 0x2ff, 0x303, 0x7, 0xd0, 0x2, 0x2, - 0x300, 0x302, 0x5, 0x20, 0x11, 0x2, 0x301, 0x300, 0x3, 0x2, 0x2, 0x2, - 0x302, 0x305, 0x3, 0x2, 0x2, 0x2, 0x303, 0x301, 0x3, 0x2, 0x2, 0x2, - 0x303, 0x304, 0x3, 0x2, 0x2, 0x2, 0x304, 0x306, 0x3, 0x2, 0x2, 0x2, - 0x305, 0x303, 0x3, 0x2, 0x2, 0x2, 0x306, 0x307, 0x7, 0xda, 0x2, 0x2, - 0x307, 0x308, 0x7, 0xda, 0x2, 0x2, 0x308, 0x27, 0x3, 0x2, 0x2, 0x2, - 0x309, 0x30a, 0x7, 0x82, 0x2, 0x2, 0x30a, 0x315, 0x7, 0xd0, 0x2, 0x2, - 0x30b, 0x30c, 0x7, 0x6a, 0x2, 0x2, 0x30c, 0x30d, 0x5, 0xd6, 0x6c, 0x2, - 0x30d, 0x30e, 0x7, 0x68, 0x2, 0x2, 0x30e, 0x30f, 0x5, 0xd6, 0x6c, 0x2, - 0x30f, 0x316, 0x3, 0x2, 0x2, 0x2, 0x310, 0x311, 0x7, 0x68, 0x2, 0x2, - 0x311, 0x312, 0x5, 0xd6, 0x6c, 0x2, 0x312, 0x313, 0x7, 0x6a, 0x2, 0x2, - 0x313, 0x314, 0x5, 0xd6, 0x6c, 0x2, 0x314, 0x316, 0x3, 0x2, 0x2, 0x2, - 0x315, 0x30b, 0x3, 0x2, 0x2, 0x2, 0x315, 0x310, 0x3, 0x2, 0x2, 0x2, - 0x316, 0x317, 0x3, 0x2, 0x2, 0x2, 0x317, 0x318, 0x7, 0xda, 0x2, 0x2, - 0x318, 0x29, 0x3, 0x2, 0x2, 0x2, 0x319, 0x31a, 0x7, 0x91, 0x2, 0x2, - 0x31a, 0x31b, 0x7, 0xd0, 0x2, 0x2, 0x31b, 0x31c, 0x5, 0x9a, 0x4e, 0x2, - 0x31c, 0x31d, 0x7, 0xda, 0x2, 0x2, 0x31d, 0x2b, 0x3, 0x2, 0x2, 0x2, - 0x31e, 0x31f, 0x7, 0x76, 0x2, 0x2, 0x31f, 0x322, 0x7, 0x19, 0x2, 0x2, - 0x320, 0x323, 0x5, 0xd6, 0x6c, 0x2, 0x321, 0x323, 0x7, 0xbf, 0x2, 0x2, - 0x322, 0x320, 0x3, 0x2, 0x2, 0x2, 0x322, 0x321, 0x3, 0x2, 0x2, 0x2, - 0x323, 0x2d, 0x3, 0x2, 0x2, 0x2, 0x324, 0x325, 0x7, 0xae, 0x2, 0x2, - 0x325, 0x326, 0x7, 0xbf, 0x2, 0x2, 0x326, 0x2f, 0x3, 0x2, 0x2, 0x2, - 0x327, 0x328, 0x7, 0xa2, 0x2, 0x2, 0x328, 0x329, 0x5, 0xc0, 0x61, 0x2, - 0x329, 0x31, 0x3, 0x2, 0x2, 0x2, 0x32a, 0x32b, 0x7, 0xc, 0x2, 0x2, 0x32b, - 0x32c, 0x5, 0x68, 0x35, 0x2, 0x32c, 0x33, 0x3, 0x2, 0x2, 0x2, 0x32d, - 0x32e, 0x7, 0xd0, 0x2, 0x2, 0x32e, 0x333, 0x5, 0x42, 0x22, 0x2, 0x32f, - 0x330, 0x7, 0xc5, 0x2, 0x2, 0x330, 0x332, 0x5, 0x42, 0x22, 0x2, 0x331, - 0x32f, 0x3, 0x2, 0x2, 0x2, 0x332, 0x335, 0x3, 0x2, 0x2, 0x2, 0x333, - 0x331, 0x3, 0x2, 0x2, 0x2, 0x333, 0x334, 0x3, 0x2, 0x2, 0x2, 0x334, - 0x336, 0x3, 0x2, 0x2, 0x2, 0x335, 0x333, 0x3, 0x2, 0x2, 0x2, 0x336, - 0x337, 0x7, 0xda, 0x2, 0x2, 0x337, 0x33d, 0x3, 0x2, 0x2, 0x2, 0x338, - 0x339, 0x7, 0xc, 0x2, 0x2, 0x339, 0x33d, 0x5, 0xc0, 0x61, 0x2, 0x33a, - 0x33b, 0x7, 0xc, 0x2, 0x2, 0x33b, 0x33d, 0x5, 0xbe, 0x60, 0x2, 0x33c, - 0x32d, 0x3, 0x2, 0x2, 0x2, 0x33c, 0x338, 0x3, 0x2, 0x2, 0x2, 0x33c, - 0x33a, 0x3, 0x2, 0x2, 0x2, 0x33d, 0x35, 0x3, 0x2, 0x2, 0x2, 0x33e, 0x359, - 0x5, 0x40, 0x21, 0x2, 0x33f, 0x340, 0x6, 0x1c, 0xc, 0x3, 0x340, 0x341, - 0x5, 0x7e, 0x40, 0x2, 0x341, 0x342, 0x8, 0x1c, 0x1, 0x2, 0x342, 0x358, - 0x3, 0x2, 0x2, 0x2, 0x343, 0x344, 0x6, 0x1c, 0xd, 0x3, 0x344, 0x345, - 0x5, 0x38, 0x1d, 0x2, 0x345, 0x346, 0x8, 0x1c, 0x1, 0x2, 0x346, 0x358, - 0x3, 0x2, 0x2, 0x2, 0x347, 0x348, 0x6, 0x1c, 0xe, 0x3, 0x348, 0x349, - 0x5, 0x3a, 0x1e, 0x2, 0x349, 0x34a, 0x8, 0x1c, 0x1, 0x2, 0x34a, 0x358, - 0x3, 0x2, 0x2, 0x2, 0x34b, 0x34c, 0x6, 0x1c, 0xf, 0x3, 0x34c, 0x34d, - 0x5, 0x3c, 0x1f, 0x2, 0x34d, 0x34e, 0x8, 0x1c, 0x1, 0x2, 0x34e, 0x358, - 0x3, 0x2, 0x2, 0x2, 0x34f, 0x350, 0x6, 0x1c, 0x10, 0x3, 0x350, 0x351, - 0x5, 0x3e, 0x20, 0x2, 0x351, 0x352, 0x8, 0x1c, 0x1, 0x2, 0x352, 0x358, - 0x3, 0x2, 0x2, 0x2, 0x353, 0x354, 0x6, 0x1c, 0x11, 0x3, 0x354, 0x355, - 0x5, 0x86, 0x44, 0x2, 0x355, 0x356, 0x8, 0x1c, 0x1, 0x2, 0x356, 0x358, - 0x3, 0x2, 0x2, 0x2, 0x357, 0x33f, 0x3, 0x2, 0x2, 0x2, 0x357, 0x343, - 0x3, 0x2, 0x2, 0x2, 0x357, 0x347, 0x3, 0x2, 0x2, 0x2, 0x357, 0x34b, - 0x3, 0x2, 0x2, 0x2, 0x357, 0x34f, 0x3, 0x2, 0x2, 0x2, 0x357, 0x353, - 0x3, 0x2, 0x2, 0x2, 0x358, 0x35b, 0x3, 0x2, 0x2, 0x2, 0x359, 0x357, - 0x3, 0x2, 0x2, 0x2, 0x359, 0x35a, 0x3, 0x2, 0x2, 0x2, 0x35a, 0x37, 0x3, - 0x2, 0x2, 0x2, 0x35b, 0x359, 0x3, 0x2, 0x2, 0x2, 0x35c, 0x35d, 0x7, - 0x7c, 0x2, 0x2, 0x35d, 0x35e, 0x7, 0x14, 0x2, 0x2, 0x35e, 0x35f, 0x5, - 0xb0, 0x59, 0x2, 0x35f, 0x39, 0x3, 0x2, 0x2, 0x2, 0x360, 0x361, 0x7, - 0x7f, 0x2, 0x2, 0x361, 0x362, 0x7, 0x5a, 0x2, 0x2, 0x362, 0x363, 0x5, - 0xb0, 0x59, 0x2, 0x363, 0x3b, 0x3, 0x2, 0x2, 0x2, 0x364, 0x365, 0x7, - 0x8b, 0x2, 0x2, 0x365, 0x366, 0x7, 0x14, 0x2, 0x2, 0x366, 0x367, 0x5, - 0xb0, 0x59, 0x2, 0x367, 0x3d, 0x3, 0x2, 0x2, 0x2, 0x368, 0x369, 0x7, - 0xa8, 0x2, 0x2, 0x369, 0x36e, 0x5, 0x50, 0x29, 0x2, 0x36a, 0x36b, 0x7, - 0xc5, 0x2, 0x2, 0x36b, 0x36d, 0x5, 0x50, 0x29, 0x2, 0x36c, 0x36a, 0x3, - 0x2, 0x2, 0x2, 0x36d, 0x370, 0x3, 0x2, 0x2, 0x2, 0x36e, 0x36c, 0x3, - 0x2, 0x2, 0x2, 0x36e, 0x36f, 0x3, 0x2, 0x2, 0x2, 0x36f, 0x3f, 0x3, 0x2, - 0x2, 0x2, 0x370, 0x36e, 0x3, 0x2, 0x2, 0x2, 0x371, 0x373, 0x7, 0x36, - 0x2, 0x2, 0x372, 0x374, 0x7, 0xca, 0x2, 0x2, 0x373, 0x372, 0x3, 0x2, - 0x2, 0x2, 0x373, 0x374, 0x3, 0x2, 0x2, 0x2, 0x374, 0x375, 0x3, 0x2, - 0x2, 0x2, 0x375, 0x37b, 0x5, 0xd8, 0x6d, 0x2, 0x376, 0x378, 0x7, 0xd0, - 0x2, 0x2, 0x377, 0x379, 0x5, 0xac, 0x57, 0x2, 0x378, 0x377, 0x3, 0x2, - 0x2, 0x2, 0x378, 0x379, 0x3, 0x2, 0x2, 0x2, 0x379, 0x37a, 0x3, 0x2, - 0x2, 0x2, 0x37a, 0x37c, 0x7, 0xda, 0x2, 0x2, 0x37b, 0x376, 0x3, 0x2, - 0x2, 0x2, 0x37b, 0x37c, 0x3, 0x2, 0x2, 0x2, 0x37c, 0x41, 0x3, 0x2, 0x2, - 0x2, 0x37d, 0x388, 0x5, 0x44, 0x23, 0x2, 0x37e, 0x37f, 0x7, 0x1e, 0x2, - 0x2, 0x37f, 0x380, 0x5, 0xd6, 0x6c, 0x2, 0x380, 0x381, 0x7, 0x17, 0x2, - 0x2, 0x381, 0x382, 0x5, 0xb0, 0x59, 0x2, 0x382, 0x388, 0x3, 0x2, 0x2, - 0x2, 0x383, 0x384, 0x7, 0x50, 0x2, 0x2, 0x384, 0x388, 0x5, 0x48, 0x25, - 0x2, 0x385, 0x386, 0x7, 0x80, 0x2, 0x2, 0x386, 0x388, 0x5, 0x4a, 0x26, - 0x2, 0x387, 0x37d, 0x3, 0x2, 0x2, 0x2, 0x387, 0x37e, 0x3, 0x2, 0x2, - 0x2, 0x387, 0x383, 0x3, 0x2, 0x2, 0x2, 0x387, 0x385, 0x3, 0x2, 0x2, - 0x2, 0x388, 0x43, 0x3, 0x2, 0x2, 0x2, 0x389, 0x38a, 0x5, 0xba, 0x5e, - 0x2, 0x38a, 0x38c, 0x5, 0xaa, 0x56, 0x2, 0x38b, 0x38d, 0x5, 0x46, 0x24, - 0x2, 0x38c, 0x38b, 0x3, 0x2, 0x2, 0x2, 0x38c, 0x38d, 0x3, 0x2, 0x2, - 0x2, 0x38d, 0x390, 0x3, 0x2, 0x2, 0x2, 0x38e, 0x38f, 0x7, 0x1d, 0x2, - 0x2, 0x38f, 0x391, 0x7, 0xbf, 0x2, 0x2, 0x390, 0x38e, 0x3, 0x2, 0x2, - 0x2, 0x390, 0x391, 0x3, 0x2, 0x2, 0x2, 0x391, 0x393, 0x3, 0x2, 0x2, - 0x2, 0x392, 0x394, 0x5, 0x4c, 0x27, 0x2, 0x393, 0x392, 0x3, 0x2, 0x2, - 0x2, 0x393, 0x394, 0x3, 0x2, 0x2, 0x2, 0x394, 0x397, 0x3, 0x2, 0x2, - 0x2, 0x395, 0x396, 0x7, 0xa8, 0x2, 0x2, 0x396, 0x398, 0x5, 0xb0, 0x59, - 0x2, 0x397, 0x395, 0x3, 0x2, 0x2, 0x2, 0x397, 0x398, 0x3, 0x2, 0x2, - 0x2, 0x398, 0x3aa, 0x3, 0x2, 0x2, 0x2, 0x399, 0x39b, 0x5, 0xba, 0x5e, - 0x2, 0x39a, 0x39c, 0x5, 0xaa, 0x56, 0x2, 0x39b, 0x39a, 0x3, 0x2, 0x2, - 0x2, 0x39b, 0x39c, 0x3, 0x2, 0x2, 0x2, 0x39c, 0x39d, 0x3, 0x2, 0x2, - 0x2, 0x39d, 0x3a0, 0x5, 0x46, 0x24, 0x2, 0x39e, 0x39f, 0x7, 0x1d, 0x2, + 0x2, 0x2, 0x2, 0x28e, 0x292, 0x7, 0xb0, 0x2, 0x2, 0x28f, 0x290, 0x7, + 0x4d, 0x2, 0x2, 0x290, 0x291, 0x7, 0x72, 0x2, 0x2, 0x291, 0x293, 0x7, + 0x38, 0x2, 0x2, 0x292, 0x28f, 0x3, 0x2, 0x2, 0x2, 0x292, 0x293, 0x3, + 0x2, 0x2, 0x2, 0x293, 0x294, 0x3, 0x2, 0x2, 0x2, 0x294, 0x296, 0x5, + 0xc0, 0x61, 0x2, 0x295, 0x297, 0x5, 0x2e, 0x18, 0x2, 0x296, 0x295, 0x3, + 0x2, 0x2, 0x2, 0x296, 0x297, 0x3, 0x2, 0x2, 0x2, 0x297, 0x299, 0x3, + 0x2, 0x2, 0x2, 0x298, 0x29a, 0x5, 0x2c, 0x17, 0x2, 0x299, 0x298, 0x3, + 0x2, 0x2, 0x2, 0x299, 0x29a, 0x3, 0x2, 0x2, 0x2, 0x29a, 0x29c, 0x3, + 0x2, 0x2, 0x2, 0x29b, 0x29d, 0x5, 0x34, 0x1b, 0x2, 0x29c, 0x29b, 0x3, + 0x2, 0x2, 0x2, 0x29c, 0x29d, 0x3, 0x2, 0x2, 0x2, 0x29d, 0x29e, 0x3, + 0x2, 0x2, 0x2, 0x29e, 0x29f, 0x5, 0x32, 0x1a, 0x2, 0x29f, 0x2a1, 0x3, + 0x2, 0x2, 0x2, 0x2a0, 0x207, 0x3, 0x2, 0x2, 0x2, 0x2a0, 0x21c, 0x3, + 0x2, 0x2, 0x2, 0x2a0, 0x22e, 0x3, 0x2, 0x2, 0x2, 0x2a0, 0x24c, 0x3, + 0x2, 0x2, 0x2, 0x2a0, 0x26e, 0x3, 0x2, 0x2, 0x2, 0x2a0, 0x289, 0x3, + 0x2, 0x2, 0x2, 0x2a1, 0x17, 0x3, 0x2, 0x2, 0x2, 0x2a2, 0x2a3, 0x7, 0xd0, + 0x2, 0x2, 0x2a3, 0x2a8, 0x5, 0x1a, 0xe, 0x2, 0x2a4, 0x2a5, 0x7, 0xc5, + 0x2, 0x2, 0x2a5, 0x2a7, 0x5, 0x1a, 0xe, 0x2, 0x2a6, 0x2a4, 0x3, 0x2, + 0x2, 0x2, 0x2a7, 0x2aa, 0x3, 0x2, 0x2, 0x2, 0x2a8, 0x2a6, 0x3, 0x2, + 0x2, 0x2, 0x2a8, 0x2a9, 0x3, 0x2, 0x2, 0x2, 0x2a9, 0x2ab, 0x3, 0x2, + 0x2, 0x2, 0x2aa, 0x2a8, 0x3, 0x2, 0x2, 0x2, 0x2ab, 0x2ac, 0x7, 0xda, + 0x2, 0x2, 0x2ac, 0x19, 0x3, 0x2, 0x2, 0x2, 0x2ad, 0x2ae, 0x5, 0xd6, + 0x6c, 0x2, 0x2ae, 0x2c4, 0x5, 0xaa, 0x56, 0x2, 0x2af, 0x2b0, 0x6, 0xe, + 0x2, 0x3, 0x2b0, 0x2b1, 0x7, 0x27, 0x2, 0x2, 0x2b1, 0x2b2, 0x5, 0xcc, + 0x67, 0x2, 0x2b2, 0x2b3, 0x8, 0xe, 0x1, 0x2, 0x2b3, 0x2c3, 0x3, 0x2, + 0x2, 0x2, 0x2b4, 0x2b5, 0x6, 0xe, 0x3, 0x3, 0x2b5, 0x2b6, 0x7, 0x3a, + 0x2, 0x2, 0x2b6, 0x2b7, 0x5, 0xb0, 0x59, 0x2, 0x2b7, 0x2b8, 0x8, 0xe, + 0x1, 0x2, 0x2b8, 0x2c3, 0x3, 0x2, 0x2, 0x2, 0x2b9, 0x2ba, 0x6, 0xe, + 0x4, 0x3, 0x2ba, 0x2bb, 0x7, 0x4a, 0x2, 0x2, 0x2bb, 0x2c3, 0x8, 0xe, + 0x1, 0x2, 0x2bc, 0x2bd, 0x6, 0xe, 0x5, 0x3, 0x2bd, 0x2be, 0x7, 0x52, + 0x2, 0x2, 0x2be, 0x2c3, 0x8, 0xe, 0x1, 0x2, 0x2bf, 0x2c0, 0x6, 0xe, + 0x6, 0x3, 0x2c0, 0x2c1, 0x7, 0x58, 0x2, 0x2, 0x2c1, 0x2c3, 0x8, 0xe, + 0x1, 0x2, 0x2c2, 0x2af, 0x3, 0x2, 0x2, 0x2, 0x2c2, 0x2b4, 0x3, 0x2, + 0x2, 0x2, 0x2c2, 0x2b9, 0x3, 0x2, 0x2, 0x2, 0x2c2, 0x2bc, 0x3, 0x2, + 0x2, 0x2, 0x2c2, 0x2bf, 0x3, 0x2, 0x2, 0x2, 0x2c3, 0x2c6, 0x3, 0x2, + 0x2, 0x2, 0x2c4, 0x2c2, 0x3, 0x2, 0x2, 0x2, 0x2c4, 0x2c5, 0x3, 0x2, + 0x2, 0x2, 0x2c5, 0x1b, 0x3, 0x2, 0x2, 0x2, 0x2c6, 0x2c4, 0x3, 0x2, 0x2, + 0x2, 0x2c7, 0x2c9, 0x5, 0x1e, 0x10, 0x2, 0x2c8, 0x2c7, 0x3, 0x2, 0x2, + 0x2, 0x2c8, 0x2c9, 0x3, 0x2, 0x2, 0x2, 0x2c9, 0x2e0, 0x3, 0x2, 0x2, + 0x2, 0x2ca, 0x2cb, 0x6, 0xf, 0x7, 0x3, 0x2cb, 0x2cc, 0x5, 0x22, 0x12, + 0x2, 0x2cc, 0x2cd, 0x8, 0xf, 0x1, 0x2, 0x2cd, 0x2df, 0x3, 0x2, 0x2, + 0x2, 0x2ce, 0x2cf, 0x6, 0xf, 0x8, 0x3, 0x2cf, 0x2d0, 0x5, 0x24, 0x13, + 0x2, 0x2d0, 0x2d1, 0x8, 0xf, 0x1, 0x2, 0x2d1, 0x2df, 0x3, 0x2, 0x2, + 0x2, 0x2d2, 0x2d3, 0x6, 0xf, 0x9, 0x3, 0x2d3, 0x2d4, 0x5, 0x26, 0x14, + 0x2, 0x2d4, 0x2d5, 0x8, 0xf, 0x1, 0x2, 0x2d5, 0x2df, 0x3, 0x2, 0x2, + 0x2, 0x2d6, 0x2d7, 0x6, 0xf, 0xa, 0x3, 0x2d7, 0x2d8, 0x5, 0x28, 0x15, + 0x2, 0x2d8, 0x2d9, 0x8, 0xf, 0x1, 0x2, 0x2d9, 0x2df, 0x3, 0x2, 0x2, + 0x2, 0x2da, 0x2db, 0x6, 0xf, 0xb, 0x3, 0x2db, 0x2dc, 0x5, 0x2a, 0x16, + 0x2, 0x2dc, 0x2dd, 0x8, 0xf, 0x1, 0x2, 0x2dd, 0x2df, 0x3, 0x2, 0x2, + 0x2, 0x2de, 0x2ca, 0x3, 0x2, 0x2, 0x2, 0x2de, 0x2ce, 0x3, 0x2, 0x2, + 0x2, 0x2de, 0x2d2, 0x3, 0x2, 0x2, 0x2, 0x2de, 0x2d6, 0x3, 0x2, 0x2, + 0x2, 0x2de, 0x2da, 0x3, 0x2, 0x2, 0x2, 0x2df, 0x2e2, 0x3, 0x2, 0x2, + 0x2, 0x2e0, 0x2de, 0x3, 0x2, 0x2, 0x2, 0x2e0, 0x2e1, 0x3, 0x2, 0x2, + 0x2, 0x2e1, 0x1d, 0x3, 0x2, 0x2, 0x2, 0x2e2, 0x2e0, 0x3, 0x2, 0x2, 0x2, + 0x2e3, 0x2e4, 0x7, 0x7f, 0x2, 0x2, 0x2e4, 0x2e5, 0x7, 0x5a, 0x2, 0x2, + 0x2e5, 0x2e6, 0x5, 0xac, 0x57, 0x2, 0x2e6, 0x1f, 0x3, 0x2, 0x2, 0x2, + 0x2e7, 0x2ee, 0x5, 0xd6, 0x6c, 0x2, 0x2e8, 0x2eb, 0x5, 0xd6, 0x6c, 0x2, + 0x2e9, 0x2ea, 0x7, 0xd0, 0x2, 0x2, 0x2ea, 0x2ec, 0x7, 0xda, 0x2, 0x2, + 0x2eb, 0x2e9, 0x3, 0x2, 0x2, 0x2, 0x2eb, 0x2ec, 0x3, 0x2, 0x2, 0x2, + 0x2ec, 0x2ef, 0x3, 0x2, 0x2, 0x2, 0x2ed, 0x2ef, 0x5, 0xcc, 0x67, 0x2, + 0x2ee, 0x2e8, 0x3, 0x2, 0x2, 0x2, 0x2ee, 0x2ed, 0x3, 0x2, 0x2, 0x2, + 0x2ef, 0x21, 0x3, 0x2, 0x2, 0x2, 0x2f0, 0x2f1, 0x7, 0x93, 0x2, 0x2, + 0x2f1, 0x2f2, 0x7, 0xd0, 0x2, 0x2, 0x2f2, 0x2f3, 0x5, 0xd6, 0x6c, 0x2, + 0x2f3, 0x2f7, 0x7, 0xd0, 0x2, 0x2, 0x2f4, 0x2f6, 0x5, 0x20, 0x11, 0x2, + 0x2f5, 0x2f4, 0x3, 0x2, 0x2, 0x2, 0x2f6, 0x2f9, 0x3, 0x2, 0x2, 0x2, + 0x2f7, 0x2f5, 0x3, 0x2, 0x2, 0x2, 0x2f7, 0x2f8, 0x3, 0x2, 0x2, 0x2, + 0x2f8, 0x2fa, 0x3, 0x2, 0x2, 0x2, 0x2f9, 0x2f7, 0x3, 0x2, 0x2, 0x2, + 0x2fa, 0x2fb, 0x7, 0xda, 0x2, 0x2, 0x2fb, 0x2fc, 0x7, 0xda, 0x2, 0x2, + 0x2fc, 0x23, 0x3, 0x2, 0x2, 0x2, 0x2fd, 0x2fe, 0x7, 0x60, 0x2, 0x2, + 0x2fe, 0x308, 0x7, 0xd0, 0x2, 0x2, 0x2ff, 0x309, 0x7, 0xbd, 0x2, 0x2, + 0x300, 0x301, 0x7, 0x6a, 0x2, 0x2, 0x301, 0x302, 0x7, 0xbd, 0x2, 0x2, + 0x302, 0x303, 0x7, 0x68, 0x2, 0x2, 0x303, 0x309, 0x7, 0xbd, 0x2, 0x2, + 0x304, 0x305, 0x7, 0x68, 0x2, 0x2, 0x305, 0x306, 0x7, 0xbd, 0x2, 0x2, + 0x306, 0x307, 0x7, 0x6a, 0x2, 0x2, 0x307, 0x309, 0x7, 0xbd, 0x2, 0x2, + 0x308, 0x2ff, 0x3, 0x2, 0x2, 0x2, 0x308, 0x300, 0x3, 0x2, 0x2, 0x2, + 0x308, 0x304, 0x3, 0x2, 0x2, 0x2, 0x309, 0x30a, 0x3, 0x2, 0x2, 0x2, + 0x30a, 0x30b, 0x7, 0xda, 0x2, 0x2, 0x30b, 0x25, 0x3, 0x2, 0x2, 0x2, + 0x30c, 0x30d, 0x7, 0x5d, 0x2, 0x2, 0x30d, 0x30e, 0x7, 0xd0, 0x2, 0x2, + 0x30e, 0x30f, 0x5, 0xd6, 0x6c, 0x2, 0x30f, 0x313, 0x7, 0xd0, 0x2, 0x2, + 0x310, 0x312, 0x5, 0x20, 0x11, 0x2, 0x311, 0x310, 0x3, 0x2, 0x2, 0x2, + 0x312, 0x315, 0x3, 0x2, 0x2, 0x2, 0x313, 0x311, 0x3, 0x2, 0x2, 0x2, + 0x313, 0x314, 0x3, 0x2, 0x2, 0x2, 0x314, 0x316, 0x3, 0x2, 0x2, 0x2, + 0x315, 0x313, 0x3, 0x2, 0x2, 0x2, 0x316, 0x317, 0x7, 0xda, 0x2, 0x2, + 0x317, 0x318, 0x7, 0xda, 0x2, 0x2, 0x318, 0x27, 0x3, 0x2, 0x2, 0x2, + 0x319, 0x31a, 0x7, 0x82, 0x2, 0x2, 0x31a, 0x325, 0x7, 0xd0, 0x2, 0x2, + 0x31b, 0x31c, 0x7, 0x6a, 0x2, 0x2, 0x31c, 0x31d, 0x5, 0xd6, 0x6c, 0x2, + 0x31d, 0x31e, 0x7, 0x68, 0x2, 0x2, 0x31e, 0x31f, 0x5, 0xd6, 0x6c, 0x2, + 0x31f, 0x326, 0x3, 0x2, 0x2, 0x2, 0x320, 0x321, 0x7, 0x68, 0x2, 0x2, + 0x321, 0x322, 0x5, 0xd6, 0x6c, 0x2, 0x322, 0x323, 0x7, 0x6a, 0x2, 0x2, + 0x323, 0x324, 0x5, 0xd6, 0x6c, 0x2, 0x324, 0x326, 0x3, 0x2, 0x2, 0x2, + 0x325, 0x31b, 0x3, 0x2, 0x2, 0x2, 0x325, 0x320, 0x3, 0x2, 0x2, 0x2, + 0x326, 0x327, 0x3, 0x2, 0x2, 0x2, 0x327, 0x328, 0x7, 0xda, 0x2, 0x2, + 0x328, 0x29, 0x3, 0x2, 0x2, 0x2, 0x329, 0x32a, 0x7, 0x91, 0x2, 0x2, + 0x32a, 0x32b, 0x7, 0xd0, 0x2, 0x2, 0x32b, 0x32c, 0x5, 0x9a, 0x4e, 0x2, + 0x32c, 0x32d, 0x7, 0xda, 0x2, 0x2, 0x32d, 0x2b, 0x3, 0x2, 0x2, 0x2, + 0x32e, 0x32f, 0x7, 0x76, 0x2, 0x2, 0x32f, 0x332, 0x7, 0x19, 0x2, 0x2, + 0x330, 0x333, 0x5, 0xd6, 0x6c, 0x2, 0x331, 0x333, 0x7, 0xbf, 0x2, 0x2, + 0x332, 0x330, 0x3, 0x2, 0x2, 0x2, 0x332, 0x331, 0x3, 0x2, 0x2, 0x2, + 0x333, 0x2d, 0x3, 0x2, 0x2, 0x2, 0x334, 0x335, 0x7, 0xae, 0x2, 0x2, + 0x335, 0x336, 0x7, 0xbf, 0x2, 0x2, 0x336, 0x2f, 0x3, 0x2, 0x2, 0x2, + 0x337, 0x338, 0x7, 0xa2, 0x2, 0x2, 0x338, 0x339, 0x5, 0xc0, 0x61, 0x2, + 0x339, 0x31, 0x3, 0x2, 0x2, 0x2, 0x33a, 0x33b, 0x7, 0xc, 0x2, 0x2, 0x33b, + 0x33c, 0x5, 0x68, 0x35, 0x2, 0x33c, 0x33, 0x3, 0x2, 0x2, 0x2, 0x33d, + 0x33e, 0x7, 0xd0, 0x2, 0x2, 0x33e, 0x343, 0x5, 0x42, 0x22, 0x2, 0x33f, + 0x340, 0x7, 0xc5, 0x2, 0x2, 0x340, 0x342, 0x5, 0x42, 0x22, 0x2, 0x341, + 0x33f, 0x3, 0x2, 0x2, 0x2, 0x342, 0x345, 0x3, 0x2, 0x2, 0x2, 0x343, + 0x341, 0x3, 0x2, 0x2, 0x2, 0x343, 0x344, 0x3, 0x2, 0x2, 0x2, 0x344, + 0x346, 0x3, 0x2, 0x2, 0x2, 0x345, 0x343, 0x3, 0x2, 0x2, 0x2, 0x346, + 0x347, 0x7, 0xda, 0x2, 0x2, 0x347, 0x34d, 0x3, 0x2, 0x2, 0x2, 0x348, + 0x349, 0x7, 0xc, 0x2, 0x2, 0x349, 0x34d, 0x5, 0xc0, 0x61, 0x2, 0x34a, + 0x34b, 0x7, 0xc, 0x2, 0x2, 0x34b, 0x34d, 0x5, 0xbe, 0x60, 0x2, 0x34c, + 0x33d, 0x3, 0x2, 0x2, 0x2, 0x34c, 0x348, 0x3, 0x2, 0x2, 0x2, 0x34c, + 0x34a, 0x3, 0x2, 0x2, 0x2, 0x34d, 0x35, 0x3, 0x2, 0x2, 0x2, 0x34e, 0x369, + 0x5, 0x40, 0x21, 0x2, 0x34f, 0x350, 0x6, 0x1c, 0xc, 0x3, 0x350, 0x351, + 0x5, 0x7e, 0x40, 0x2, 0x351, 0x352, 0x8, 0x1c, 0x1, 0x2, 0x352, 0x368, + 0x3, 0x2, 0x2, 0x2, 0x353, 0x354, 0x6, 0x1c, 0xd, 0x3, 0x354, 0x355, + 0x5, 0x38, 0x1d, 0x2, 0x355, 0x356, 0x8, 0x1c, 0x1, 0x2, 0x356, 0x368, + 0x3, 0x2, 0x2, 0x2, 0x357, 0x358, 0x6, 0x1c, 0xe, 0x3, 0x358, 0x359, + 0x5, 0x3a, 0x1e, 0x2, 0x359, 0x35a, 0x8, 0x1c, 0x1, 0x2, 0x35a, 0x368, + 0x3, 0x2, 0x2, 0x2, 0x35b, 0x35c, 0x6, 0x1c, 0xf, 0x3, 0x35c, 0x35d, + 0x5, 0x3c, 0x1f, 0x2, 0x35d, 0x35e, 0x8, 0x1c, 0x1, 0x2, 0x35e, 0x368, + 0x3, 0x2, 0x2, 0x2, 0x35f, 0x360, 0x6, 0x1c, 0x10, 0x3, 0x360, 0x361, + 0x5, 0x3e, 0x20, 0x2, 0x361, 0x362, 0x8, 0x1c, 0x1, 0x2, 0x362, 0x368, + 0x3, 0x2, 0x2, 0x2, 0x363, 0x364, 0x6, 0x1c, 0x11, 0x3, 0x364, 0x365, + 0x5, 0x86, 0x44, 0x2, 0x365, 0x366, 0x8, 0x1c, 0x1, 0x2, 0x366, 0x368, + 0x3, 0x2, 0x2, 0x2, 0x367, 0x34f, 0x3, 0x2, 0x2, 0x2, 0x367, 0x353, + 0x3, 0x2, 0x2, 0x2, 0x367, 0x357, 0x3, 0x2, 0x2, 0x2, 0x367, 0x35b, + 0x3, 0x2, 0x2, 0x2, 0x367, 0x35f, 0x3, 0x2, 0x2, 0x2, 0x367, 0x363, + 0x3, 0x2, 0x2, 0x2, 0x368, 0x36b, 0x3, 0x2, 0x2, 0x2, 0x369, 0x367, + 0x3, 0x2, 0x2, 0x2, 0x369, 0x36a, 0x3, 0x2, 0x2, 0x2, 0x36a, 0x37, 0x3, + 0x2, 0x2, 0x2, 0x36b, 0x369, 0x3, 0x2, 0x2, 0x2, 0x36c, 0x36d, 0x7, + 0x7c, 0x2, 0x2, 0x36d, 0x36e, 0x7, 0x14, 0x2, 0x2, 0x36e, 0x36f, 0x5, + 0xb0, 0x59, 0x2, 0x36f, 0x39, 0x3, 0x2, 0x2, 0x2, 0x370, 0x371, 0x7, + 0x7f, 0x2, 0x2, 0x371, 0x372, 0x7, 0x5a, 0x2, 0x2, 0x372, 0x373, 0x5, + 0xb0, 0x59, 0x2, 0x373, 0x3b, 0x3, 0x2, 0x2, 0x2, 0x374, 0x375, 0x7, + 0x8b, 0x2, 0x2, 0x375, 0x376, 0x7, 0x14, 0x2, 0x2, 0x376, 0x377, 0x5, + 0xb0, 0x59, 0x2, 0x377, 0x3d, 0x3, 0x2, 0x2, 0x2, 0x378, 0x379, 0x7, + 0xa8, 0x2, 0x2, 0x379, 0x37e, 0x5, 0x50, 0x29, 0x2, 0x37a, 0x37b, 0x7, + 0xc5, 0x2, 0x2, 0x37b, 0x37d, 0x5, 0x50, 0x29, 0x2, 0x37c, 0x37a, 0x3, + 0x2, 0x2, 0x2, 0x37d, 0x380, 0x3, 0x2, 0x2, 0x2, 0x37e, 0x37c, 0x3, + 0x2, 0x2, 0x2, 0x37e, 0x37f, 0x3, 0x2, 0x2, 0x2, 0x37f, 0x3f, 0x3, 0x2, + 0x2, 0x2, 0x380, 0x37e, 0x3, 0x2, 0x2, 0x2, 0x381, 0x383, 0x7, 0x36, + 0x2, 0x2, 0x382, 0x384, 0x7, 0xca, 0x2, 0x2, 0x383, 0x382, 0x3, 0x2, + 0x2, 0x2, 0x383, 0x384, 0x3, 0x2, 0x2, 0x2, 0x384, 0x385, 0x3, 0x2, + 0x2, 0x2, 0x385, 0x38b, 0x5, 0xd8, 0x6d, 0x2, 0x386, 0x388, 0x7, 0xd0, + 0x2, 0x2, 0x387, 0x389, 0x5, 0xac, 0x57, 0x2, 0x388, 0x387, 0x3, 0x2, + 0x2, 0x2, 0x388, 0x389, 0x3, 0x2, 0x2, 0x2, 0x389, 0x38a, 0x3, 0x2, + 0x2, 0x2, 0x38a, 0x38c, 0x7, 0xda, 0x2, 0x2, 0x38b, 0x386, 0x3, 0x2, + 0x2, 0x2, 0x38b, 0x38c, 0x3, 0x2, 0x2, 0x2, 0x38c, 0x41, 0x3, 0x2, 0x2, + 0x2, 0x38d, 0x398, 0x5, 0x44, 0x23, 0x2, 0x38e, 0x38f, 0x7, 0x1e, 0x2, + 0x2, 0x38f, 0x390, 0x5, 0xd6, 0x6c, 0x2, 0x390, 0x391, 0x7, 0x17, 0x2, + 0x2, 0x391, 0x392, 0x5, 0xb0, 0x59, 0x2, 0x392, 0x398, 0x3, 0x2, 0x2, + 0x2, 0x393, 0x394, 0x7, 0x50, 0x2, 0x2, 0x394, 0x398, 0x5, 0x48, 0x25, + 0x2, 0x395, 0x396, 0x7, 0x80, 0x2, 0x2, 0x396, 0x398, 0x5, 0x4a, 0x26, + 0x2, 0x397, 0x38d, 0x3, 0x2, 0x2, 0x2, 0x397, 0x38e, 0x3, 0x2, 0x2, + 0x2, 0x397, 0x393, 0x3, 0x2, 0x2, 0x2, 0x397, 0x395, 0x3, 0x2, 0x2, + 0x2, 0x398, 0x43, 0x3, 0x2, 0x2, 0x2, 0x399, 0x39a, 0x5, 0xba, 0x5e, + 0x2, 0x39a, 0x39c, 0x5, 0xaa, 0x56, 0x2, 0x39b, 0x39d, 0x5, 0x46, 0x24, + 0x2, 0x39c, 0x39b, 0x3, 0x2, 0x2, 0x2, 0x39c, 0x39d, 0x3, 0x2, 0x2, + 0x2, 0x39d, 0x3a0, 0x3, 0x2, 0x2, 0x2, 0x39e, 0x39f, 0x7, 0x1d, 0x2, 0x2, 0x39f, 0x3a1, 0x7, 0xbf, 0x2, 0x2, 0x3a0, 0x39e, 0x3, 0x2, 0x2, 0x2, 0x3a0, 0x3a1, 0x3, 0x2, 0x2, 0x2, 0x3a1, 0x3a3, 0x3, 0x2, 0x2, 0x2, 0x3a2, 0x3a4, 0x5, 0x4c, 0x27, 0x2, 0x3a3, 0x3a2, 0x3, 0x2, 0x2, 0x2, 0x3a3, 0x3a4, 0x3, 0x2, 0x2, 0x2, 0x3a4, 0x3a7, 0x3, 0x2, 0x2, 0x2, 0x3a5, 0x3a6, 0x7, 0xa8, 0x2, 0x2, 0x3a6, 0x3a8, 0x5, 0xb0, 0x59, 0x2, 0x3a7, 0x3a5, 0x3, 0x2, 0x2, 0x2, 0x3a7, 0x3a8, 0x3, 0x2, 0x2, - 0x2, 0x3a8, 0x3aa, 0x3, 0x2, 0x2, 0x2, 0x3a9, 0x389, 0x3, 0x2, 0x2, - 0x2, 0x3a9, 0x399, 0x3, 0x2, 0x2, 0x2, 0x3aa, 0x45, 0x3, 0x2, 0x2, 0x2, - 0x3ab, 0x3ac, 0x9, 0x4, 0x2, 0x2, 0x3ac, 0x3ad, 0x5, 0xb0, 0x59, 0x2, - 0x3ad, 0x47, 0x3, 0x2, 0x2, 0x2, 0x3ae, 0x3af, 0x5, 0xba, 0x5e, 0x2, - 0x3af, 0x3b0, 0x5, 0xb0, 0x59, 0x2, 0x3b0, 0x3b1, 0x7, 0xa9, 0x2, 0x2, - 0x3b1, 0x3b2, 0x5, 0xaa, 0x56, 0x2, 0x3b2, 0x3b3, 0x7, 0x47, 0x2, 0x2, - 0x3b3, 0x3b4, 0x7, 0xbd, 0x2, 0x2, 0x3b4, 0x49, 0x3, 0x2, 0x2, 0x2, - 0x3b5, 0x3b6, 0x5, 0xba, 0x5e, 0x2, 0x3b6, 0x3b7, 0x5, 0x66, 0x34, 0x2, - 0x3b7, 0x4b, 0x3, 0x2, 0x2, 0x2, 0x3b8, 0x3b9, 0x7, 0x1a, 0x2, 0x2, - 0x3b9, 0x3ba, 0x7, 0xd0, 0x2, 0x2, 0x3ba, 0x3bf, 0x5, 0x4e, 0x28, 0x2, - 0x3bb, 0x3bc, 0x7, 0xc5, 0x2, 0x2, 0x3bc, 0x3be, 0x5, 0x4e, 0x28, 0x2, - 0x3bd, 0x3bb, 0x3, 0x2, 0x2, 0x2, 0x3be, 0x3c1, 0x3, 0x2, 0x2, 0x2, - 0x3bf, 0x3bd, 0x3, 0x2, 0x2, 0x2, 0x3bf, 0x3c0, 0x3, 0x2, 0x2, 0x2, - 0x3c0, 0x3c2, 0x3, 0x2, 0x2, 0x2, 0x3c1, 0x3bf, 0x3, 0x2, 0x2, 0x2, - 0x3c2, 0x3c3, 0x7, 0xda, 0x2, 0x2, 0x3c3, 0x4d, 0x3, 0x2, 0x2, 0x2, - 0x3c4, 0x3ca, 0x5, 0xd6, 0x6c, 0x2, 0x3c5, 0x3c7, 0x7, 0xd0, 0x2, 0x2, - 0x3c6, 0x3c8, 0x5, 0xac, 0x57, 0x2, 0x3c7, 0x3c6, 0x3, 0x2, 0x2, 0x2, - 0x3c7, 0x3c8, 0x3, 0x2, 0x2, 0x2, 0x3c8, 0x3c9, 0x3, 0x2, 0x2, 0x2, - 0x3c9, 0x3cb, 0x7, 0xda, 0x2, 0x2, 0x3ca, 0x3c5, 0x3, 0x2, 0x2, 0x2, - 0x3ca, 0x3cb, 0x3, 0x2, 0x2, 0x2, 0x3cb, 0x4f, 0x3, 0x2, 0x2, 0x2, 0x3cc, - 0x3d4, 0x5, 0xb0, 0x59, 0x2, 0x3cd, 0x3d5, 0x7, 0x29, 0x2, 0x2, 0x3ce, - 0x3cf, 0x7, 0xa2, 0x2, 0x2, 0x3cf, 0x3d0, 0x7, 0x30, 0x2, 0x2, 0x3d0, - 0x3d5, 0x7, 0xbf, 0x2, 0x2, 0x3d1, 0x3d2, 0x7, 0xa2, 0x2, 0x2, 0x3d2, - 0x3d3, 0x7, 0xb1, 0x2, 0x2, 0x3d3, 0x3d5, 0x7, 0xbf, 0x2, 0x2, 0x3d4, - 0x3cd, 0x3, 0x2, 0x2, 0x2, 0x3d4, 0x3ce, 0x3, 0x2, 0x2, 0x2, 0x3d4, - 0x3d1, 0x3, 0x2, 0x2, 0x2, 0x3d4, 0x3d5, 0x3, 0x2, 0x2, 0x2, 0x3d5, - 0x51, 0x3, 0x2, 0x2, 0x2, 0x3d6, 0x3d8, 0x9, 0x5, 0x2, 0x2, 0x3d7, 0x3d9, - 0x7, 0x9a, 0x2, 0x2, 0x3d8, 0x3d7, 0x3, 0x2, 0x2, 0x2, 0x3d8, 0x3d9, - 0x3, 0x2, 0x2, 0x2, 0x3d9, 0x3da, 0x3, 0x2, 0x2, 0x2, 0x3da, 0x3db, - 0x5, 0xbc, 0x5f, 0x2, 0x3db, 0x53, 0x3, 0x2, 0x2, 0x2, 0x3dc, 0x3dd, - 0x9, 0x6, 0x2, 0x2, 0x3dd, 0x3e0, 0x7, 0x22, 0x2, 0x2, 0x3de, 0x3df, - 0x7, 0x4d, 0x2, 0x2, 0x3df, 0x3e1, 0x7, 0x38, 0x2, 0x2, 0x3e0, 0x3de, - 0x3, 0x2, 0x2, 0x2, 0x3e0, 0x3e1, 0x3, 0x2, 0x2, 0x2, 0x3e1, 0x3e2, - 0x3, 0x2, 0x2, 0x2, 0x3e2, 0x3e4, 0x5, 0xc6, 0x64, 0x2, 0x3e3, 0x3e5, - 0x5, 0x2c, 0x17, 0x2, 0x3e4, 0x3e3, 0x3, 0x2, 0x2, 0x2, 0x3e4, 0x3e5, - 0x3, 0x2, 0x2, 0x2, 0x3e5, 0x3fc, 0x3, 0x2, 0x2, 0x2, 0x3e6, 0x3ed, - 0x9, 0x6, 0x2, 0x2, 0x3e7, 0x3ee, 0x7, 0x2f, 0x2, 0x2, 0x3e8, 0x3ea, - 0x7, 0x9c, 0x2, 0x2, 0x3e9, 0x3e8, 0x3, 0x2, 0x2, 0x2, 0x3e9, 0x3ea, - 0x3, 0x2, 0x2, 0x2, 0x3ea, 0x3eb, 0x3, 0x2, 0x2, 0x2, 0x3eb, 0x3ee, - 0x7, 0x9a, 0x2, 0x2, 0x3ec, 0x3ee, 0x7, 0xb0, 0x2, 0x2, 0x3ed, 0x3e7, - 0x3, 0x2, 0x2, 0x2, 0x3ed, 0x3e9, 0x3, 0x2, 0x2, 0x2, 0x3ed, 0x3ec, - 0x3, 0x2, 0x2, 0x2, 0x3ee, 0x3f1, 0x3, 0x2, 0x2, 0x2, 0x3ef, 0x3f0, - 0x7, 0x4d, 0x2, 0x2, 0x3f0, 0x3f2, 0x7, 0x38, 0x2, 0x2, 0x3f1, 0x3ef, - 0x3, 0x2, 0x2, 0x2, 0x3f1, 0x3f2, 0x3, 0x2, 0x2, 0x2, 0x3f2, 0x3f3, - 0x3, 0x2, 0x2, 0x2, 0x3f3, 0x3f5, 0x5, 0xc0, 0x61, 0x2, 0x3f4, 0x3f6, - 0x5, 0x2c, 0x17, 0x2, 0x3f5, 0x3f4, 0x3, 0x2, 0x2, 0x2, 0x3f5, 0x3f6, - 0x3, 0x2, 0x2, 0x2, 0x3f6, 0x3f9, 0x3, 0x2, 0x2, 0x2, 0x3f7, 0x3f8, - 0x7, 0x71, 0x2, 0x2, 0x3f8, 0x3fa, 0x7, 0x28, 0x2, 0x2, 0x3f9, 0x3f7, - 0x3, 0x2, 0x2, 0x2, 0x3f9, 0x3fa, 0x3, 0x2, 0x2, 0x2, 0x3fa, 0x3fc, - 0x3, 0x2, 0x2, 0x2, 0x3fb, 0x3dc, 0x3, 0x2, 0x2, 0x2, 0x3fb, 0x3e6, - 0x3, 0x2, 0x2, 0x2, 0x3fc, 0x55, 0x3, 0x2, 0x2, 0x2, 0x3fd, 0x3fe, 0x7, - 0x38, 0x2, 0x2, 0x3fe, 0x3ff, 0x7, 0x22, 0x2, 0x2, 0x3ff, 0x40b, 0x5, - 0xc6, 0x64, 0x2, 0x400, 0x407, 0x7, 0x38, 0x2, 0x2, 0x401, 0x408, 0x7, - 0x2f, 0x2, 0x2, 0x402, 0x404, 0x7, 0x9c, 0x2, 0x2, 0x403, 0x402, 0x3, - 0x2, 0x2, 0x2, 0x403, 0x404, 0x3, 0x2, 0x2, 0x2, 0x404, 0x405, 0x3, - 0x2, 0x2, 0x2, 0x405, 0x408, 0x7, 0x9a, 0x2, 0x2, 0x406, 0x408, 0x7, - 0xb0, 0x2, 0x2, 0x407, 0x401, 0x3, 0x2, 0x2, 0x2, 0x407, 0x403, 0x3, - 0x2, 0x2, 0x2, 0x407, 0x406, 0x3, 0x2, 0x2, 0x2, 0x407, 0x408, 0x3, - 0x2, 0x2, 0x2, 0x408, 0x409, 0x3, 0x2, 0x2, 0x2, 0x409, 0x40b, 0x5, - 0xc0, 0x61, 0x2, 0x40a, 0x3fd, 0x3, 0x2, 0x2, 0x2, 0x40a, 0x400, 0x3, - 0x2, 0x2, 0x2, 0x40b, 0x57, 0x3, 0x2, 0x2, 0x2, 0x40c, 0x40d, 0x7, 0x39, - 0x2, 0x2, 0x40d, 0x40e, 0x7, 0xf, 0x2, 0x2, 0x40e, 0x413, 0x5, 0x4, - 0x3, 0x2, 0x40f, 0x410, 0x7, 0x39, 0x2, 0x2, 0x410, 0x411, 0x7, 0x98, - 0x2, 0x2, 0x411, 0x413, 0x5, 0x4, 0x3, 0x2, 0x412, 0x40c, 0x3, 0x2, - 0x2, 0x2, 0x412, 0x40f, 0x3, 0x2, 0x2, 0x2, 0x413, 0x59, 0x3, 0x2, 0x2, - 0x2, 0x414, 0x415, 0x7, 0x54, 0x2, 0x2, 0x415, 0x417, 0x7, 0x56, 0x2, - 0x2, 0x416, 0x418, 0x7, 0x9a, 0x2, 0x2, 0x417, 0x416, 0x3, 0x2, 0x2, - 0x2, 0x417, 0x418, 0x3, 0x2, 0x2, 0x2, 0x418, 0x41c, 0x3, 0x2, 0x2, - 0x2, 0x419, 0x41d, 0x5, 0xc0, 0x61, 0x2, 0x41a, 0x41b, 0x7, 0x45, 0x2, - 0x2, 0x41b, 0x41d, 0x5, 0xbe, 0x60, 0x2, 0x41c, 0x419, 0x3, 0x2, 0x2, - 0x2, 0x41c, 0x41a, 0x3, 0x2, 0x2, 0x2, 0x41d, 0x41f, 0x3, 0x2, 0x2, - 0x2, 0x41e, 0x420, 0x5, 0x5c, 0x2f, 0x2, 0x41f, 0x41e, 0x3, 0x2, 0x2, - 0x2, 0x41f, 0x420, 0x3, 0x2, 0x2, 0x2, 0x420, 0x421, 0x3, 0x2, 0x2, - 0x2, 0x421, 0x422, 0x5, 0x5e, 0x30, 0x2, 0x422, 0x5b, 0x3, 0x2, 0x2, - 0x2, 0x423, 0x424, 0x7, 0xd0, 0x2, 0x2, 0x424, 0x429, 0x5, 0xba, 0x5e, - 0x2, 0x425, 0x426, 0x7, 0xc5, 0x2, 0x2, 0x426, 0x428, 0x5, 0xba, 0x5e, - 0x2, 0x427, 0x425, 0x3, 0x2, 0x2, 0x2, 0x428, 0x42b, 0x3, 0x2, 0x2, - 0x2, 0x429, 0x427, 0x3, 0x2, 0x2, 0x2, 0x429, 0x42a, 0x3, 0x2, 0x2, - 0x2, 0x42a, 0x42c, 0x3, 0x2, 0x2, 0x2, 0x42b, 0x429, 0x3, 0x2, 0x2, - 0x2, 0x42c, 0x42d, 0x7, 0xda, 0x2, 0x2, 0x42d, 0x5d, 0x3, 0x2, 0x2, - 0x2, 0x42e, 0x42f, 0x7, 0x41, 0x2, 0x2, 0x42f, 0x438, 0x5, 0xd6, 0x6c, - 0x2, 0x430, 0x438, 0x7, 0xaf, 0x2, 0x2, 0x431, 0x433, 0x5, 0x68, 0x35, - 0x2, 0x432, 0x434, 0x7, 0xdb, 0x2, 0x2, 0x433, 0x432, 0x3, 0x2, 0x2, - 0x2, 0x433, 0x434, 0x3, 0x2, 0x2, 0x2, 0x434, 0x435, 0x3, 0x2, 0x2, - 0x2, 0x435, 0x436, 0x7, 0x2, 0x2, 0x3, 0x436, 0x438, 0x3, 0x2, 0x2, - 0x2, 0x437, 0x42e, 0x3, 0x2, 0x2, 0x2, 0x437, 0x430, 0x3, 0x2, 0x2, - 0x2, 0x437, 0x431, 0x3, 0x2, 0x2, 0x2, 0x438, 0x5f, 0x3, 0x2, 0x2, 0x2, - 0x439, 0x43a, 0x7, 0x5b, 0x2, 0x2, 0x43a, 0x43c, 0x7, 0x6f, 0x2, 0x2, - 0x43b, 0x43d, 0x5, 0x2c, 0x17, 0x2, 0x43c, 0x43b, 0x3, 0x2, 0x2, 0x2, - 0x43c, 0x43d, 0x3, 0x2, 0x2, 0x2, 0x43d, 0x43e, 0x3, 0x2, 0x2, 0x2, - 0x43e, 0x440, 0x5, 0x78, 0x3d, 0x2, 0x43f, 0x441, 0x9, 0x7, 0x2, 0x2, - 0x440, 0x43f, 0x3, 0x2, 0x2, 0x2, 0x440, 0x441, 0x3, 0x2, 0x2, 0x2, - 0x441, 0x61, 0x3, 0x2, 0x2, 0x2, 0x442, 0x443, 0x7, 0x77, 0x2, 0x2, - 0x443, 0x444, 0x7, 0x9a, 0x2, 0x2, 0x444, 0x446, 0x5, 0xc0, 0x61, 0x2, - 0x445, 0x447, 0x5, 0x2c, 0x17, 0x2, 0x446, 0x445, 0x3, 0x2, 0x2, 0x2, - 0x446, 0x447, 0x3, 0x2, 0x2, 0x2, 0x447, 0x449, 0x3, 0x2, 0x2, 0x2, - 0x448, 0x44a, 0x5, 0x10, 0x9, 0x2, 0x449, 0x448, 0x3, 0x2, 0x2, 0x2, - 0x449, 0x44a, 0x3, 0x2, 0x2, 0x2, 0x44a, 0x44c, 0x3, 0x2, 0x2, 0x2, - 0x44b, 0x44d, 0x7, 0x3d, 0x2, 0x2, 0x44c, 0x44b, 0x3, 0x2, 0x2, 0x2, - 0x44c, 0x44d, 0x3, 0x2, 0x2, 0x2, 0x44d, 0x44f, 0x3, 0x2, 0x2, 0x2, - 0x44e, 0x450, 0x7, 0x26, 0x2, 0x2, 0x44f, 0x44e, 0x3, 0x2, 0x2, 0x2, - 0x44f, 0x450, 0x3, 0x2, 0x2, 0x2, 0x450, 0x63, 0x3, 0x2, 0x2, 0x2, 0x451, - 0x452, 0x7, 0x85, 0x2, 0x2, 0x452, 0x453, 0x7, 0x9a, 0x2, 0x2, 0x453, - 0x454, 0x5, 0xc0, 0x61, 0x2, 0x454, 0x455, 0x7, 0xa2, 0x2, 0x2, 0x455, - 0x45d, 0x5, 0xc0, 0x61, 0x2, 0x456, 0x457, 0x7, 0xc5, 0x2, 0x2, 0x457, - 0x458, 0x5, 0xc0, 0x61, 0x2, 0x458, 0x459, 0x7, 0xa2, 0x2, 0x2, 0x459, - 0x45a, 0x5, 0xc0, 0x61, 0x2, 0x45a, 0x45c, 0x3, 0x2, 0x2, 0x2, 0x45b, - 0x456, 0x3, 0x2, 0x2, 0x2, 0x45c, 0x45f, 0x3, 0x2, 0x2, 0x2, 0x45d, - 0x45b, 0x3, 0x2, 0x2, 0x2, 0x45d, 0x45e, 0x3, 0x2, 0x2, 0x2, 0x45e, - 0x461, 0x3, 0x2, 0x2, 0x2, 0x45f, 0x45d, 0x3, 0x2, 0x2, 0x2, 0x460, - 0x462, 0x5, 0x2c, 0x17, 0x2, 0x461, 0x460, 0x3, 0x2, 0x2, 0x2, 0x461, - 0x462, 0x3, 0x2, 0x2, 0x2, 0x462, 0x65, 0x3, 0x2, 0x2, 0x2, 0x463, 0x465, - 0x7, 0xd0, 0x2, 0x2, 0x464, 0x466, 0x5, 0x6e, 0x38, 0x2, 0x465, 0x464, - 0x3, 0x2, 0x2, 0x2, 0x465, 0x466, 0x3, 0x2, 0x2, 0x2, 0x466, 0x467, - 0x3, 0x2, 0x2, 0x2, 0x467, 0x468, 0x7, 0x8d, 0x2, 0x2, 0x468, 0x46a, - 0x5, 0xac, 0x57, 0x2, 0x469, 0x46b, 0x5, 0x7a, 0x3e, 0x2, 0x46a, 0x469, - 0x3, 0x2, 0x2, 0x2, 0x46a, 0x46b, 0x3, 0x2, 0x2, 0x2, 0x46b, 0x46d, - 0x3, 0x2, 0x2, 0x2, 0x46c, 0x46e, 0x5, 0x80, 0x41, 0x2, 0x46d, 0x46c, - 0x3, 0x2, 0x2, 0x2, 0x46d, 0x46e, 0x3, 0x2, 0x2, 0x2, 0x46e, 0x46f, - 0x3, 0x2, 0x2, 0x2, 0x46f, 0x470, 0x7, 0xda, 0x2, 0x2, 0x470, 0x67, - 0x3, 0x2, 0x2, 0x2, 0x471, 0x477, 0x5, 0x6a, 0x36, 0x2, 0x472, 0x473, - 0x7, 0xaa, 0x2, 0x2, 0x473, 0x474, 0x7, 0x6, 0x2, 0x2, 0x474, 0x476, - 0x5, 0x6a, 0x36, 0x2, 0x475, 0x472, 0x3, 0x2, 0x2, 0x2, 0x476, 0x479, - 0x3, 0x2, 0x2, 0x2, 0x477, 0x475, 0x3, 0x2, 0x2, 0x2, 0x477, 0x478, - 0x3, 0x2, 0x2, 0x2, 0x478, 0x69, 0x3, 0x2, 0x2, 0x2, 0x479, 0x477, 0x3, - 0x2, 0x2, 0x2, 0x47a, 0x480, 0x5, 0x6c, 0x37, 0x2, 0x47b, 0x47c, 0x7, - 0xd0, 0x2, 0x2, 0x47c, 0x47d, 0x5, 0x68, 0x35, 0x2, 0x47d, 0x47e, 0x7, - 0xda, 0x2, 0x2, 0x47e, 0x480, 0x3, 0x2, 0x2, 0x2, 0x47f, 0x47a, 0x3, - 0x2, 0x2, 0x2, 0x47f, 0x47b, 0x3, 0x2, 0x2, 0x2, 0x480, 0x6b, 0x3, 0x2, - 0x2, 0x2, 0x481, 0x483, 0x5, 0x6e, 0x38, 0x2, 0x482, 0x481, 0x3, 0x2, - 0x2, 0x2, 0x482, 0x483, 0x3, 0x2, 0x2, 0x2, 0x483, 0x484, 0x3, 0x2, - 0x2, 0x2, 0x484, 0x486, 0x7, 0x8d, 0x2, 0x2, 0x485, 0x487, 0x7, 0x31, - 0x2, 0x2, 0x486, 0x485, 0x3, 0x2, 0x2, 0x2, 0x486, 0x487, 0x3, 0x2, - 0x2, 0x2, 0x487, 0x489, 0x3, 0x2, 0x2, 0x2, 0x488, 0x48a, 0x5, 0x70, - 0x39, 0x2, 0x489, 0x488, 0x3, 0x2, 0x2, 0x2, 0x489, 0x48a, 0x3, 0x2, - 0x2, 0x2, 0x48a, 0x48b, 0x3, 0x2, 0x2, 0x2, 0x48b, 0x48d, 0x5, 0xac, - 0x57, 0x2, 0x48c, 0x48e, 0x5, 0x72, 0x3a, 0x2, 0x48d, 0x48c, 0x3, 0x2, - 0x2, 0x2, 0x48d, 0x48e, 0x3, 0x2, 0x2, 0x2, 0x48e, 0x490, 0x3, 0x2, - 0x2, 0x2, 0x48f, 0x491, 0x5, 0x74, 0x3b, 0x2, 0x490, 0x48f, 0x3, 0x2, - 0x2, 0x2, 0x490, 0x491, 0x3, 0x2, 0x2, 0x2, 0x491, 0x493, 0x3, 0x2, - 0x2, 0x2, 0x492, 0x494, 0x5, 0x76, 0x3c, 0x2, 0x493, 0x492, 0x3, 0x2, - 0x2, 0x2, 0x493, 0x494, 0x3, 0x2, 0x2, 0x2, 0x494, 0x496, 0x3, 0x2, - 0x2, 0x2, 0x495, 0x497, 0x5, 0x78, 0x3d, 0x2, 0x496, 0x495, 0x3, 0x2, - 0x2, 0x2, 0x496, 0x497, 0x3, 0x2, 0x2, 0x2, 0x497, 0x499, 0x3, 0x2, - 0x2, 0x2, 0x498, 0x49a, 0x5, 0x7a, 0x3e, 0x2, 0x499, 0x498, 0x3, 0x2, - 0x2, 0x2, 0x499, 0x49a, 0x3, 0x2, 0x2, 0x2, 0x49a, 0x49d, 0x3, 0x2, - 0x2, 0x2, 0x49b, 0x49c, 0x7, 0xb6, 0x2, 0x2, 0x49c, 0x49e, 0x9, 0x8, - 0x2, 0x2, 0x49d, 0x49b, 0x3, 0x2, 0x2, 0x2, 0x49d, 0x49e, 0x3, 0x2, - 0x2, 0x2, 0x49e, 0x4a1, 0x3, 0x2, 0x2, 0x2, 0x49f, 0x4a0, 0x7, 0xb6, - 0x2, 0x2, 0x4a0, 0x4a2, 0x7, 0xa4, 0x2, 0x2, 0x4a1, 0x49f, 0x3, 0x2, - 0x2, 0x2, 0x4a1, 0x4a2, 0x3, 0x2, 0x2, 0x2, 0x4a2, 0x4a4, 0x3, 0x2, - 0x2, 0x2, 0x4a3, 0x4a5, 0x5, 0x7c, 0x3f, 0x2, 0x4a4, 0x4a3, 0x3, 0x2, - 0x2, 0x2, 0x4a4, 0x4a5, 0x3, 0x2, 0x2, 0x2, 0x4a5, 0x4a7, 0x3, 0x2, - 0x2, 0x2, 0x4a6, 0x4a8, 0x5, 0x7e, 0x40, 0x2, 0x4a7, 0x4a6, 0x3, 0x2, - 0x2, 0x2, 0x4a7, 0x4a8, 0x3, 0x2, 0x2, 0x2, 0x4a8, 0x4aa, 0x3, 0x2, - 0x2, 0x2, 0x4a9, 0x4ab, 0x5, 0x82, 0x42, 0x2, 0x4aa, 0x4a9, 0x3, 0x2, - 0x2, 0x2, 0x4aa, 0x4ab, 0x3, 0x2, 0x2, 0x2, 0x4ab, 0x4ad, 0x3, 0x2, - 0x2, 0x2, 0x4ac, 0x4ae, 0x5, 0x84, 0x43, 0x2, 0x4ad, 0x4ac, 0x3, 0x2, - 0x2, 0x2, 0x4ad, 0x4ae, 0x3, 0x2, 0x2, 0x2, 0x4ae, 0x4b0, 0x3, 0x2, - 0x2, 0x2, 0x4af, 0x4b1, 0x5, 0x86, 0x44, 0x2, 0x4b0, 0x4af, 0x3, 0x2, - 0x2, 0x2, 0x4b0, 0x4b1, 0x3, 0x2, 0x2, 0x2, 0x4b1, 0x6d, 0x3, 0x2, 0x2, - 0x2, 0x4b2, 0x4b3, 0x7, 0xb6, 0x2, 0x2, 0x4b3, 0x4b4, 0x5, 0xac, 0x57, - 0x2, 0x4b4, 0x6f, 0x3, 0x2, 0x2, 0x2, 0x4b5, 0x4b6, 0x7, 0xa3, 0x2, - 0x2, 0x4b6, 0x4b9, 0x7, 0xbd, 0x2, 0x2, 0x4b7, 0x4b8, 0x7, 0xb6, 0x2, - 0x2, 0x4b8, 0x4ba, 0x7, 0x9f, 0x2, 0x2, 0x4b9, 0x4b7, 0x3, 0x2, 0x2, - 0x2, 0x4b9, 0x4ba, 0x3, 0x2, 0x2, 0x2, 0x4ba, 0x71, 0x3, 0x2, 0x2, 0x2, - 0x4bb, 0x4bc, 0x7, 0x43, 0x2, 0x2, 0x4bc, 0x4bd, 0x5, 0x88, 0x45, 0x2, - 0x4bd, 0x73, 0x3, 0x2, 0x2, 0x2, 0x4be, 0x4c0, 0x9, 0x9, 0x2, 0x2, 0x4bf, - 0x4be, 0x3, 0x2, 0x2, 0x2, 0x4bf, 0x4c0, 0x3, 0x2, 0x2, 0x2, 0x4c0, - 0x4c1, 0x3, 0x2, 0x2, 0x2, 0x4c1, 0x4c2, 0x7, 0xb, 0x2, 0x2, 0x4c2, - 0x4c3, 0x7, 0x59, 0x2, 0x2, 0x4c3, 0x4c4, 0x5, 0xac, 0x57, 0x2, 0x4c4, - 0x75, 0x3, 0x2, 0x2, 0x2, 0x4c5, 0x4c6, 0x7, 0x7e, 0x2, 0x2, 0x4c6, - 0x4c7, 0x5, 0xb0, 0x59, 0x2, 0x4c7, 0x77, 0x3, 0x2, 0x2, 0x2, 0x4c8, - 0x4c9, 0x7, 0xb5, 0x2, 0x2, 0x4c9, 0x4ca, 0x5, 0xb0, 0x59, 0x2, 0x4ca, - 0x79, 0x3, 0x2, 0x2, 0x2, 0x4cb, 0x4cc, 0x7, 0x48, 0x2, 0x2, 0x4cc, - 0x4d3, 0x7, 0x14, 0x2, 0x2, 0x4cd, 0x4ce, 0x9, 0x8, 0x2, 0x2, 0x4ce, - 0x4cf, 0x7, 0xd0, 0x2, 0x2, 0x4cf, 0x4d0, 0x5, 0xac, 0x57, 0x2, 0x4d0, - 0x4d1, 0x7, 0xda, 0x2, 0x2, 0x4d1, 0x4d4, 0x3, 0x2, 0x2, 0x2, 0x4d2, - 0x4d4, 0x5, 0xac, 0x57, 0x2, 0x4d3, 0x4cd, 0x3, 0x2, 0x2, 0x2, 0x4d3, - 0x4d2, 0x3, 0x2, 0x2, 0x2, 0x4d4, 0x7b, 0x3, 0x2, 0x2, 0x2, 0x4d5, 0x4d6, - 0x7, 0x49, 0x2, 0x2, 0x4d6, 0x4d7, 0x5, 0xb0, 0x59, 0x2, 0x4d7, 0x7d, - 0x3, 0x2, 0x2, 0x2, 0x4d8, 0x4d9, 0x7, 0x79, 0x2, 0x2, 0x4d9, 0x4da, - 0x7, 0x14, 0x2, 0x2, 0x4da, 0x4db, 0x5, 0x94, 0x4b, 0x2, 0x4db, 0x7f, - 0x3, 0x2, 0x2, 0x2, 0x4dc, 0x4dd, 0x7, 0x79, 0x2, 0x2, 0x4dd, 0x4de, - 0x7, 0x14, 0x2, 0x2, 0x4de, 0x4df, 0x5, 0xac, 0x57, 0x2, 0x4df, 0x81, - 0x3, 0x2, 0x2, 0x2, 0x4e0, 0x4e1, 0x7, 0x62, 0x2, 0x2, 0x4e1, 0x4e2, - 0x5, 0x92, 0x4a, 0x2, 0x4e2, 0x4e3, 0x7, 0x14, 0x2, 0x2, 0x4e3, 0x4e4, - 0x5, 0xac, 0x57, 0x2, 0x4e4, 0x83, 0x3, 0x2, 0x2, 0x2, 0x4e5, 0x4e6, - 0x7, 0x62, 0x2, 0x2, 0x4e6, 0x4e9, 0x5, 0x92, 0x4a, 0x2, 0x4e7, 0x4e8, - 0x7, 0xb6, 0x2, 0x2, 0x4e8, 0x4ea, 0x7, 0x9f, 0x2, 0x2, 0x4e9, 0x4e7, - 0x3, 0x2, 0x2, 0x2, 0x4e9, 0x4ea, 0x3, 0x2, 0x2, 0x2, 0x4ea, 0x85, 0x3, - 0x2, 0x2, 0x2, 0x4eb, 0x4ec, 0x7, 0x91, 0x2, 0x2, 0x4ec, 0x4ed, 0x5, - 0x9a, 0x4e, 0x2, 0x4ed, 0x87, 0x3, 0x2, 0x2, 0x2, 0x4ee, 0x4ef, 0x8, - 0x45, 0x1, 0x2, 0x4ef, 0x4f1, 0x5, 0xbc, 0x5f, 0x2, 0x4f0, 0x4f2, 0x7, - 0x3d, 0x2, 0x2, 0x4f1, 0x4f0, 0x3, 0x2, 0x2, 0x2, 0x4f1, 0x4f2, 0x3, - 0x2, 0x2, 0x2, 0x4f2, 0x4f4, 0x3, 0x2, 0x2, 0x2, 0x4f3, 0x4f5, 0x5, - 0x90, 0x49, 0x2, 0x4f4, 0x4f3, 0x3, 0x2, 0x2, 0x2, 0x4f4, 0x4f5, 0x3, - 0x2, 0x2, 0x2, 0x4f5, 0x4fb, 0x3, 0x2, 0x2, 0x2, 0x4f6, 0x4f7, 0x7, - 0xd0, 0x2, 0x2, 0x4f7, 0x4f8, 0x5, 0x88, 0x45, 0x2, 0x4f8, 0x4f9, 0x7, - 0xda, 0x2, 0x2, 0x4f9, 0x4fb, 0x3, 0x2, 0x2, 0x2, 0x4fa, 0x4ee, 0x3, - 0x2, 0x2, 0x2, 0x4fa, 0x4f6, 0x3, 0x2, 0x2, 0x2, 0x4fb, 0x50d, 0x3, - 0x2, 0x2, 0x2, 0x4fc, 0x4fd, 0xc, 0x5, 0x2, 0x2, 0x4fd, 0x4fe, 0x5, - 0x8c, 0x47, 0x2, 0x4fe, 0x4ff, 0x5, 0x88, 0x45, 0x6, 0x4ff, 0x50c, 0x3, - 0x2, 0x2, 0x2, 0x500, 0x502, 0xc, 0x6, 0x2, 0x2, 0x501, 0x503, 0x9, - 0xa, 0x2, 0x2, 0x502, 0x501, 0x3, 0x2, 0x2, 0x2, 0x502, 0x503, 0x3, - 0x2, 0x2, 0x2, 0x503, 0x505, 0x3, 0x2, 0x2, 0x2, 0x504, 0x506, 0x5, - 0x8a, 0x46, 0x2, 0x505, 0x504, 0x3, 0x2, 0x2, 0x2, 0x505, 0x506, 0x3, - 0x2, 0x2, 0x2, 0x506, 0x507, 0x3, 0x2, 0x2, 0x2, 0x507, 0x508, 0x7, - 0x59, 0x2, 0x2, 0x508, 0x509, 0x5, 0x88, 0x45, 0x2, 0x509, 0x50a, 0x5, - 0x8e, 0x48, 0x2, 0x50a, 0x50c, 0x3, 0x2, 0x2, 0x2, 0x50b, 0x4fc, 0x3, - 0x2, 0x2, 0x2, 0x50b, 0x500, 0x3, 0x2, 0x2, 0x2, 0x50c, 0x50f, 0x3, - 0x2, 0x2, 0x2, 0x50d, 0x50b, 0x3, 0x2, 0x2, 0x2, 0x50d, 0x50e, 0x3, - 0x2, 0x2, 0x2, 0x50e, 0x89, 0x3, 0x2, 0x2, 0x2, 0x50f, 0x50d, 0x3, 0x2, - 0x2, 0x2, 0x510, 0x512, 0x9, 0xb, 0x2, 0x2, 0x511, 0x510, 0x3, 0x2, - 0x2, 0x2, 0x511, 0x512, 0x3, 0x2, 0x2, 0x2, 0x512, 0x513, 0x3, 0x2, - 0x2, 0x2, 0x513, 0x51a, 0x7, 0x53, 0x2, 0x2, 0x514, 0x516, 0x7, 0x53, - 0x2, 0x2, 0x515, 0x517, 0x9, 0xb, 0x2, 0x2, 0x516, 0x515, 0x3, 0x2, - 0x2, 0x2, 0x516, 0x517, 0x3, 0x2, 0x2, 0x2, 0x517, 0x51a, 0x3, 0x2, - 0x2, 0x2, 0x518, 0x51a, 0x9, 0xb, 0x2, 0x2, 0x519, 0x511, 0x3, 0x2, - 0x2, 0x2, 0x519, 0x514, 0x3, 0x2, 0x2, 0x2, 0x519, 0x518, 0x3, 0x2, - 0x2, 0x2, 0x51a, 0x53c, 0x3, 0x2, 0x2, 0x2, 0x51b, 0x51d, 0x9, 0xc, - 0x2, 0x2, 0x51c, 0x51b, 0x3, 0x2, 0x2, 0x2, 0x51c, 0x51d, 0x3, 0x2, - 0x2, 0x2, 0x51d, 0x51e, 0x3, 0x2, 0x2, 0x2, 0x51e, 0x520, 0x9, 0xd, - 0x2, 0x2, 0x51f, 0x521, 0x7, 0x7a, 0x2, 0x2, 0x520, 0x51f, 0x3, 0x2, - 0x2, 0x2, 0x520, 0x521, 0x3, 0x2, 0x2, 0x2, 0x521, 0x52a, 0x3, 0x2, - 0x2, 0x2, 0x522, 0x524, 0x9, 0xd, 0x2, 0x2, 0x523, 0x525, 0x7, 0x7a, - 0x2, 0x2, 0x524, 0x523, 0x3, 0x2, 0x2, 0x2, 0x524, 0x525, 0x3, 0x2, - 0x2, 0x2, 0x525, 0x527, 0x3, 0x2, 0x2, 0x2, 0x526, 0x528, 0x9, 0xc, - 0x2, 0x2, 0x527, 0x526, 0x3, 0x2, 0x2, 0x2, 0x527, 0x528, 0x3, 0x2, - 0x2, 0x2, 0x528, 0x52a, 0x3, 0x2, 0x2, 0x2, 0x529, 0x51c, 0x3, 0x2, - 0x2, 0x2, 0x529, 0x522, 0x3, 0x2, 0x2, 0x2, 0x52a, 0x53c, 0x3, 0x2, - 0x2, 0x2, 0x52b, 0x52d, 0x9, 0xe, 0x2, 0x2, 0x52c, 0x52b, 0x3, 0x2, - 0x2, 0x2, 0x52c, 0x52d, 0x3, 0x2, 0x2, 0x2, 0x52d, 0x52e, 0x3, 0x2, - 0x2, 0x2, 0x52e, 0x530, 0x7, 0x44, 0x2, 0x2, 0x52f, 0x531, 0x7, 0x7a, - 0x2, 0x2, 0x530, 0x52f, 0x3, 0x2, 0x2, 0x2, 0x530, 0x531, 0x3, 0x2, - 0x2, 0x2, 0x531, 0x53a, 0x3, 0x2, 0x2, 0x2, 0x532, 0x534, 0x7, 0x44, - 0x2, 0x2, 0x533, 0x535, 0x7, 0x7a, 0x2, 0x2, 0x534, 0x533, 0x3, 0x2, - 0x2, 0x2, 0x534, 0x535, 0x3, 0x2, 0x2, 0x2, 0x535, 0x537, 0x3, 0x2, - 0x2, 0x2, 0x536, 0x538, 0x9, 0xe, 0x2, 0x2, 0x537, 0x536, 0x3, 0x2, - 0x2, 0x2, 0x537, 0x538, 0x3, 0x2, 0x2, 0x2, 0x538, 0x53a, 0x3, 0x2, - 0x2, 0x2, 0x539, 0x52c, 0x3, 0x2, 0x2, 0x2, 0x539, 0x532, 0x3, 0x2, - 0x2, 0x2, 0x53a, 0x53c, 0x3, 0x2, 0x2, 0x2, 0x53b, 0x519, 0x3, 0x2, - 0x2, 0x2, 0x53b, 0x529, 0x3, 0x2, 0x2, 0x2, 0x53b, 0x539, 0x3, 0x2, - 0x2, 0x2, 0x53c, 0x8b, 0x3, 0x2, 0x2, 0x2, 0x53d, 0x53f, 0x9, 0xa, 0x2, - 0x2, 0x53e, 0x53d, 0x3, 0x2, 0x2, 0x2, 0x53e, 0x53f, 0x3, 0x2, 0x2, - 0x2, 0x53f, 0x540, 0x3, 0x2, 0x2, 0x2, 0x540, 0x541, 0x7, 0x20, 0x2, - 0x2, 0x541, 0x544, 0x7, 0x59, 0x2, 0x2, 0x542, 0x544, 0x7, 0xc5, 0x2, - 0x2, 0x543, 0x53e, 0x3, 0x2, 0x2, 0x2, 0x543, 0x542, 0x3, 0x2, 0x2, - 0x2, 0x544, 0x8d, 0x3, 0x2, 0x2, 0x2, 0x545, 0x546, 0x7, 0x76, 0x2, - 0x2, 0x546, 0x54f, 0x5, 0xac, 0x57, 0x2, 0x547, 0x548, 0x7, 0xad, 0x2, - 0x2, 0x548, 0x549, 0x7, 0xd0, 0x2, 0x2, 0x549, 0x54a, 0x5, 0xac, 0x57, - 0x2, 0x54a, 0x54b, 0x7, 0xda, 0x2, 0x2, 0x54b, 0x54f, 0x3, 0x2, 0x2, - 0x2, 0x54c, 0x54d, 0x7, 0xad, 0x2, 0x2, 0x54d, 0x54f, 0x5, 0xac, 0x57, - 0x2, 0x54e, 0x545, 0x3, 0x2, 0x2, 0x2, 0x54e, 0x547, 0x3, 0x2, 0x2, - 0x2, 0x54e, 0x54c, 0x3, 0x2, 0x2, 0x2, 0x54f, 0x8f, 0x3, 0x2, 0x2, 0x2, - 0x550, 0x551, 0x7, 0x8b, 0x2, 0x2, 0x551, 0x554, 0x5, 0x98, 0x4d, 0x2, - 0x552, 0x553, 0x7, 0x75, 0x2, 0x2, 0x553, 0x555, 0x5, 0x98, 0x4d, 0x2, - 0x554, 0x552, 0x3, 0x2, 0x2, 0x2, 0x554, 0x555, 0x3, 0x2, 0x2, 0x2, - 0x555, 0x91, 0x3, 0x2, 0x2, 0x2, 0x556, 0x559, 0x5, 0xb0, 0x59, 0x2, - 0x557, 0x558, 0x9, 0xf, 0x2, 0x2, 0x558, 0x55a, 0x5, 0xb0, 0x59, 0x2, - 0x559, 0x557, 0x3, 0x2, 0x2, 0x2, 0x559, 0x55a, 0x3, 0x2, 0x2, 0x2, - 0x55a, 0x93, 0x3, 0x2, 0x2, 0x2, 0x55b, 0x560, 0x5, 0x96, 0x4c, 0x2, - 0x55c, 0x55d, 0x7, 0xc5, 0x2, 0x2, 0x55d, 0x55f, 0x5, 0x96, 0x4c, 0x2, - 0x55e, 0x55c, 0x3, 0x2, 0x2, 0x2, 0x55f, 0x562, 0x3, 0x2, 0x2, 0x2, - 0x560, 0x55e, 0x3, 0x2, 0x2, 0x2, 0x560, 0x561, 0x3, 0x2, 0x2, 0x2, - 0x561, 0x95, 0x3, 0x2, 0x2, 0x2, 0x562, 0x560, 0x3, 0x2, 0x2, 0x2, 0x563, - 0x565, 0x5, 0xb0, 0x59, 0x2, 0x564, 0x566, 0x9, 0x10, 0x2, 0x2, 0x565, - 0x564, 0x3, 0x2, 0x2, 0x2, 0x565, 0x566, 0x3, 0x2, 0x2, 0x2, 0x566, - 0x569, 0x3, 0x2, 0x2, 0x2, 0x567, 0x568, 0x7, 0x74, 0x2, 0x2, 0x568, - 0x56a, 0x9, 0x11, 0x2, 0x2, 0x569, 0x567, 0x3, 0x2, 0x2, 0x2, 0x569, - 0x56a, 0x3, 0x2, 0x2, 0x2, 0x56a, 0x56d, 0x3, 0x2, 0x2, 0x2, 0x56b, - 0x56c, 0x7, 0x1b, 0x2, 0x2, 0x56c, 0x56e, 0x7, 0xbf, 0x2, 0x2, 0x56d, - 0x56b, 0x3, 0x2, 0x2, 0x2, 0x56d, 0x56e, 0x3, 0x2, 0x2, 0x2, 0x56e, - 0x97, 0x3, 0x2, 0x2, 0x2, 0x56f, 0x572, 0x5, 0xca, 0x66, 0x2, 0x570, - 0x571, 0x7, 0xdc, 0x2, 0x2, 0x571, 0x573, 0x5, 0xca, 0x66, 0x2, 0x572, - 0x570, 0x3, 0x2, 0x2, 0x2, 0x572, 0x573, 0x3, 0x2, 0x2, 0x2, 0x573, - 0x99, 0x3, 0x2, 0x2, 0x2, 0x574, 0x579, 0x5, 0x9c, 0x4f, 0x2, 0x575, - 0x576, 0x7, 0xc5, 0x2, 0x2, 0x576, 0x578, 0x5, 0x9c, 0x4f, 0x2, 0x577, - 0x575, 0x3, 0x2, 0x2, 0x2, 0x578, 0x57b, 0x3, 0x2, 0x2, 0x2, 0x579, - 0x577, 0x3, 0x2, 0x2, 0x2, 0x579, 0x57a, 0x3, 0x2, 0x2, 0x2, 0x57a, - 0x9b, 0x3, 0x2, 0x2, 0x2, 0x57b, 0x579, 0x3, 0x2, 0x2, 0x2, 0x57c, 0x57d, - 0x5, 0xd6, 0x6c, 0x2, 0x57d, 0x57e, 0x7, 0xca, 0x2, 0x2, 0x57e, 0x57f, - 0x5, 0xcc, 0x67, 0x2, 0x57f, 0x9d, 0x3, 0x2, 0x2, 0x2, 0x580, 0x581, - 0x7, 0x90, 0x2, 0x2, 0x581, 0x582, 0x5, 0x9a, 0x4e, 0x2, 0x582, 0x9f, - 0x3, 0x2, 0x2, 0x2, 0x583, 0x584, 0x7, 0x92, 0x2, 0x2, 0x584, 0x585, - 0x7, 0x1f, 0x2, 0x2, 0x585, 0x586, 0x7, 0x22, 0x2, 0x2, 0x586, 0x5ae, - 0x5, 0xc6, 0x64, 0x2, 0x587, 0x588, 0x7, 0x92, 0x2, 0x2, 0x588, 0x589, - 0x7, 0x1f, 0x2, 0x2, 0x589, 0x58a, 0x7, 0x2f, 0x2, 0x2, 0x58a, 0x5ae, - 0x5, 0xc0, 0x61, 0x2, 0x58b, 0x58c, 0x7, 0x92, 0x2, 0x2, 0x58c, 0x58e, - 0x7, 0x1f, 0x2, 0x2, 0x58d, 0x58f, 0x7, 0x9c, 0x2, 0x2, 0x58e, 0x58d, - 0x3, 0x2, 0x2, 0x2, 0x58e, 0x58f, 0x3, 0x2, 0x2, 0x2, 0x58f, 0x591, - 0x3, 0x2, 0x2, 0x2, 0x590, 0x592, 0x7, 0x9a, 0x2, 0x2, 0x591, 0x590, - 0x3, 0x2, 0x2, 0x2, 0x591, 0x592, 0x3, 0x2, 0x2, 0x2, 0x592, 0x593, - 0x3, 0x2, 0x2, 0x2, 0x593, 0x5ae, 0x5, 0xc0, 0x61, 0x2, 0x594, 0x595, - 0x7, 0x92, 0x2, 0x2, 0x595, 0x5ae, 0x7, 0x23, 0x2, 0x2, 0x596, 0x597, - 0x7, 0x92, 0x2, 0x2, 0x597, 0x59a, 0x7, 0x2e, 0x2, 0x2, 0x598, 0x599, - 0x7, 0x43, 0x2, 0x2, 0x599, 0x59b, 0x5, 0xc6, 0x64, 0x2, 0x59a, 0x598, - 0x3, 0x2, 0x2, 0x2, 0x59a, 0x59b, 0x3, 0x2, 0x2, 0x2, 0x59b, 0x5ae, - 0x3, 0x2, 0x2, 0x2, 0x59c, 0x59e, 0x7, 0x92, 0x2, 0x2, 0x59d, 0x59f, - 0x7, 0x9c, 0x2, 0x2, 0x59e, 0x59d, 0x3, 0x2, 0x2, 0x2, 0x59e, 0x59f, - 0x3, 0x2, 0x2, 0x2, 0x59f, 0x5a0, 0x3, 0x2, 0x2, 0x2, 0x5a0, 0x5a3, - 0x7, 0x9b, 0x2, 0x2, 0x5a1, 0x5a2, 0x9, 0x12, 0x2, 0x2, 0x5a2, 0x5a4, - 0x5, 0xc6, 0x64, 0x2, 0x5a3, 0x5a1, 0x3, 0x2, 0x2, 0x2, 0x5a3, 0x5a4, - 0x3, 0x2, 0x2, 0x2, 0x5a4, 0x5a8, 0x3, 0x2, 0x2, 0x2, 0x5a5, 0x5a6, - 0x7, 0x61, 0x2, 0x2, 0x5a6, 0x5a9, 0x7, 0xbf, 0x2, 0x2, 0x5a7, 0x5a9, - 0x5, 0x78, 0x3d, 0x2, 0x5a8, 0x5a5, 0x3, 0x2, 0x2, 0x2, 0x5a8, 0x5a7, - 0x3, 0x2, 0x2, 0x2, 0x5a8, 0x5a9, 0x3, 0x2, 0x2, 0x2, 0x5a9, 0x5ab, - 0x3, 0x2, 0x2, 0x2, 0x5aa, 0x5ac, 0x5, 0x84, 0x43, 0x2, 0x5ab, 0x5aa, - 0x3, 0x2, 0x2, 0x2, 0x5ab, 0x5ac, 0x3, 0x2, 0x2, 0x2, 0x5ac, 0x5ae, - 0x3, 0x2, 0x2, 0x2, 0x5ad, 0x583, 0x3, 0x2, 0x2, 0x2, 0x5ad, 0x587, - 0x3, 0x2, 0x2, 0x2, 0x5ad, 0x58b, 0x3, 0x2, 0x2, 0x2, 0x5ad, 0x594, - 0x3, 0x2, 0x2, 0x2, 0x5ad, 0x596, 0x3, 0x2, 0x2, 0x2, 0x5ad, 0x59c, - 0x3, 0x2, 0x2, 0x2, 0x5ae, 0xa1, 0x3, 0x2, 0x2, 0x2, 0x5af, 0x5b0, 0x7, - 0x99, 0x2, 0x2, 0x5b0, 0x5b1, 0x7, 0x3f, 0x2, 0x2, 0x5b1, 0x5b2, 0x7, - 0x32, 0x2, 0x2, 0x5b2, 0x5d2, 0x5, 0xc0, 0x61, 0x2, 0x5b3, 0x5b4, 0x7, - 0x99, 0x2, 0x2, 0x5b4, 0x5b5, 0x7, 0x3f, 0x2, 0x2, 0x5b5, 0x5d2, 0x7, - 0x65, 0x2, 0x2, 0x5b6, 0x5b7, 0x7, 0x99, 0x2, 0x2, 0x5b7, 0x5b8, 0x7, - 0x83, 0x2, 0x2, 0x5b8, 0x5d2, 0x7, 0x2e, 0x2, 0x2, 0x5b9, 0x5ba, 0x7, - 0x99, 0x2, 0x2, 0x5ba, 0x5bb, 0x7, 0x83, 0x2, 0x2, 0x5bb, 0x5bc, 0x7, - 0x2f, 0x2, 0x2, 0x5bc, 0x5d2, 0x5, 0xc0, 0x61, 0x2, 0x5bd, 0x5be, 0x7, - 0x99, 0x2, 0x2, 0x5be, 0x5c6, 0x9, 0x13, 0x2, 0x2, 0x5bf, 0x5c0, 0x7, - 0x32, 0x2, 0x2, 0x5c0, 0x5c7, 0x7, 0x8f, 0x2, 0x2, 0x5c1, 0x5c7, 0x7, - 0x3c, 0x2, 0x2, 0x5c2, 0x5c4, 0x7, 0xa8, 0x2, 0x2, 0x5c3, 0x5c2, 0x3, - 0x2, 0x2, 0x2, 0x5c3, 0x5c4, 0x3, 0x2, 0x2, 0x2, 0x5c4, 0x5c5, 0x3, - 0x2, 0x2, 0x2, 0x5c5, 0x5c7, 0x7, 0x69, 0x2, 0x2, 0x5c6, 0x5bf, 0x3, - 0x2, 0x2, 0x2, 0x5c6, 0x5c1, 0x3, 0x2, 0x2, 0x2, 0x5c6, 0x5c3, 0x3, - 0x2, 0x2, 0x2, 0x5c7, 0x5c8, 0x3, 0x2, 0x2, 0x2, 0x5c8, 0x5d2, 0x5, - 0xc0, 0x61, 0x2, 0x5c9, 0x5ca, 0x7, 0x99, 0x2, 0x2, 0x5ca, 0x5cb, 0x9, - 0x13, 0x2, 0x2, 0x5cb, 0x5cc, 0x7, 0x88, 0x2, 0x2, 0x5cc, 0x5d2, 0x7, - 0x8f, 0x2, 0x2, 0x5cd, 0x5ce, 0x7, 0x99, 0x2, 0x2, 0x5ce, 0x5cf, 0x7, - 0x97, 0x2, 0x2, 0x5cf, 0x5d0, 0x7, 0x87, 0x2, 0x2, 0x5d0, 0x5d2, 0x5, - 0xc0, 0x61, 0x2, 0x5d1, 0x5af, 0x3, 0x2, 0x2, 0x2, 0x5d1, 0x5b3, 0x3, - 0x2, 0x2, 0x2, 0x5d1, 0x5b6, 0x3, 0x2, 0x2, 0x2, 0x5d1, 0x5b9, 0x3, - 0x2, 0x2, 0x2, 0x5d1, 0x5bd, 0x3, 0x2, 0x2, 0x2, 0x5d1, 0x5c9, 0x3, - 0x2, 0x2, 0x2, 0x5d1, 0x5cd, 0x3, 0x2, 0x2, 0x2, 0x5d2, 0xa3, 0x3, 0x2, - 0x2, 0x2, 0x5d3, 0x5d5, 0x7, 0xa7, 0x2, 0x2, 0x5d4, 0x5d6, 0x7, 0x9c, - 0x2, 0x2, 0x5d5, 0x5d4, 0x3, 0x2, 0x2, 0x2, 0x5d5, 0x5d6, 0x3, 0x2, - 0x2, 0x2, 0x5d6, 0x5d8, 0x3, 0x2, 0x2, 0x2, 0x5d7, 0x5d9, 0x7, 0x9a, - 0x2, 0x2, 0x5d8, 0x5d7, 0x3, 0x2, 0x2, 0x2, 0x5d8, 0x5d9, 0x3, 0x2, - 0x2, 0x2, 0x5d9, 0x5dc, 0x3, 0x2, 0x2, 0x2, 0x5da, 0x5db, 0x7, 0x4d, - 0x2, 0x2, 0x5db, 0x5dd, 0x7, 0x38, 0x2, 0x2, 0x5dc, 0x5da, 0x3, 0x2, - 0x2, 0x2, 0x5dc, 0x5dd, 0x3, 0x2, 0x2, 0x2, 0x5dd, 0x5de, 0x3, 0x2, - 0x2, 0x2, 0x5de, 0x5e0, 0x5, 0xc0, 0x61, 0x2, 0x5df, 0x5e1, 0x5, 0x2c, - 0x17, 0x2, 0x5e0, 0x5df, 0x3, 0x2, 0x2, 0x2, 0x5e0, 0x5e1, 0x3, 0x2, - 0x2, 0x2, 0x5e1, 0xa5, 0x3, 0x2, 0x2, 0x2, 0x5e2, 0x5e3, 0x7, 0xac, - 0x2, 0x2, 0x5e3, 0x5e4, 0x5, 0xc6, 0x64, 0x2, 0x5e4, 0xa7, 0x3, 0x2, - 0x2, 0x2, 0x5e5, 0x5e6, 0x7, 0xb2, 0x2, 0x2, 0x5e6, 0x5e8, 0x5, 0xc0, - 0x61, 0x2, 0x5e7, 0x5e9, 0x7, 0x37, 0x2, 0x2, 0x5e8, 0x5e7, 0x3, 0x2, - 0x2, 0x2, 0x5e8, 0x5e9, 0x3, 0x2, 0x2, 0x2, 0x5e9, 0x5ec, 0x3, 0x2, - 0x2, 0x2, 0x5ea, 0x5eb, 0x7, 0x62, 0x2, 0x2, 0x5eb, 0x5ed, 0x7, 0xbd, - 0x2, 0x2, 0x5ec, 0x5ea, 0x3, 0x2, 0x2, 0x2, 0x5ec, 0x5ed, 0x3, 0x2, - 0x2, 0x2, 0x5ed, 0xa9, 0x3, 0x2, 0x2, 0x2, 0x5ee, 0x61e, 0x5, 0xd6, - 0x6c, 0x2, 0x5ef, 0x5f0, 0x5, 0xd6, 0x6c, 0x2, 0x5f0, 0x5f1, 0x7, 0xd0, - 0x2, 0x2, 0x5f1, 0x5f2, 0x5, 0xd6, 0x6c, 0x2, 0x5f2, 0x5f9, 0x5, 0xaa, - 0x56, 0x2, 0x5f3, 0x5f4, 0x7, 0xc5, 0x2, 0x2, 0x5f4, 0x5f5, 0x5, 0xd6, - 0x6c, 0x2, 0x5f5, 0x5f6, 0x5, 0xaa, 0x56, 0x2, 0x5f6, 0x5f8, 0x3, 0x2, - 0x2, 0x2, 0x5f7, 0x5f3, 0x3, 0x2, 0x2, 0x2, 0x5f8, 0x5fb, 0x3, 0x2, - 0x2, 0x2, 0x5f9, 0x5f7, 0x3, 0x2, 0x2, 0x2, 0x5f9, 0x5fa, 0x3, 0x2, - 0x2, 0x2, 0x5fa, 0x5fc, 0x3, 0x2, 0x2, 0x2, 0x5fb, 0x5f9, 0x3, 0x2, - 0x2, 0x2, 0x5fc, 0x5fd, 0x7, 0xda, 0x2, 0x2, 0x5fd, 0x61e, 0x3, 0x2, - 0x2, 0x2, 0x5fe, 0x5ff, 0x5, 0xd6, 0x6c, 0x2, 0x5ff, 0x600, 0x7, 0xd0, - 0x2, 0x2, 0x600, 0x605, 0x5, 0xda, 0x6e, 0x2, 0x601, 0x602, 0x7, 0xc5, - 0x2, 0x2, 0x602, 0x604, 0x5, 0xda, 0x6e, 0x2, 0x603, 0x601, 0x3, 0x2, - 0x2, 0x2, 0x604, 0x607, 0x3, 0x2, 0x2, 0x2, 0x605, 0x603, 0x3, 0x2, - 0x2, 0x2, 0x605, 0x606, 0x3, 0x2, 0x2, 0x2, 0x606, 0x608, 0x3, 0x2, - 0x2, 0x2, 0x607, 0x605, 0x3, 0x2, 0x2, 0x2, 0x608, 0x609, 0x7, 0xda, - 0x2, 0x2, 0x609, 0x61e, 0x3, 0x2, 0x2, 0x2, 0x60a, 0x60b, 0x5, 0xd6, - 0x6c, 0x2, 0x60b, 0x60c, 0x7, 0xd0, 0x2, 0x2, 0x60c, 0x611, 0x5, 0xaa, - 0x56, 0x2, 0x60d, 0x60e, 0x7, 0xc5, 0x2, 0x2, 0x60e, 0x610, 0x5, 0xaa, - 0x56, 0x2, 0x60f, 0x60d, 0x3, 0x2, 0x2, 0x2, 0x610, 0x613, 0x3, 0x2, - 0x2, 0x2, 0x611, 0x60f, 0x3, 0x2, 0x2, 0x2, 0x611, 0x612, 0x3, 0x2, - 0x2, 0x2, 0x612, 0x614, 0x3, 0x2, 0x2, 0x2, 0x613, 0x611, 0x3, 0x2, - 0x2, 0x2, 0x614, 0x615, 0x7, 0xda, 0x2, 0x2, 0x615, 0x61e, 0x3, 0x2, - 0x2, 0x2, 0x616, 0x617, 0x5, 0xd6, 0x6c, 0x2, 0x617, 0x619, 0x7, 0xd0, - 0x2, 0x2, 0x618, 0x61a, 0x5, 0xac, 0x57, 0x2, 0x619, 0x618, 0x3, 0x2, - 0x2, 0x2, 0x619, 0x61a, 0x3, 0x2, 0x2, 0x2, 0x61a, 0x61b, 0x3, 0x2, - 0x2, 0x2, 0x61b, 0x61c, 0x7, 0xda, 0x2, 0x2, 0x61c, 0x61e, 0x3, 0x2, - 0x2, 0x2, 0x61d, 0x5ee, 0x3, 0x2, 0x2, 0x2, 0x61d, 0x5ef, 0x3, 0x2, - 0x2, 0x2, 0x61d, 0x5fe, 0x3, 0x2, 0x2, 0x2, 0x61d, 0x60a, 0x3, 0x2, - 0x2, 0x2, 0x61d, 0x616, 0x3, 0x2, 0x2, 0x2, 0x61e, 0xab, 0x3, 0x2, 0x2, - 0x2, 0x61f, 0x624, 0x5, 0xae, 0x58, 0x2, 0x620, 0x621, 0x7, 0xc5, 0x2, - 0x2, 0x621, 0x623, 0x5, 0xae, 0x58, 0x2, 0x622, 0x620, 0x3, 0x2, 0x2, - 0x2, 0x623, 0x626, 0x3, 0x2, 0x2, 0x2, 0x624, 0x622, 0x3, 0x2, 0x2, - 0x2, 0x624, 0x625, 0x3, 0x2, 0x2, 0x2, 0x625, 0xad, 0x3, 0x2, 0x2, 0x2, - 0x626, 0x624, 0x3, 0x2, 0x2, 0x2, 0x627, 0x628, 0x5, 0xc0, 0x61, 0x2, - 0x628, 0x629, 0x7, 0xc8, 0x2, 0x2, 0x629, 0x62b, 0x3, 0x2, 0x2, 0x2, - 0x62a, 0x627, 0x3, 0x2, 0x2, 0x2, 0x62a, 0x62b, 0x3, 0x2, 0x2, 0x2, - 0x62b, 0x62c, 0x3, 0x2, 0x2, 0x2, 0x62c, 0x633, 0x7, 0xc1, 0x2, 0x2, - 0x62d, 0x62e, 0x7, 0xd0, 0x2, 0x2, 0x62e, 0x62f, 0x5, 0x68, 0x35, 0x2, - 0x62f, 0x630, 0x7, 0xda, 0x2, 0x2, 0x630, 0x633, 0x3, 0x2, 0x2, 0x2, - 0x631, 0x633, 0x5, 0xb0, 0x59, 0x2, 0x632, 0x62a, 0x3, 0x2, 0x2, 0x2, - 0x632, 0x62d, 0x3, 0x2, 0x2, 0x2, 0x632, 0x631, 0x3, 0x2, 0x2, 0x2, - 0x633, 0xaf, 0x3, 0x2, 0x2, 0x2, 0x634, 0x635, 0x8, 0x59, 0x1, 0x2, - 0x635, 0x637, 0x7, 0x15, 0x2, 0x2, 0x636, 0x638, 0x5, 0xb0, 0x59, 0x2, - 0x637, 0x636, 0x3, 0x2, 0x2, 0x2, 0x637, 0x638, 0x3, 0x2, 0x2, 0x2, - 0x638, 0x63e, 0x3, 0x2, 0x2, 0x2, 0x639, 0x63a, 0x7, 0xb4, 0x2, 0x2, - 0x63a, 0x63b, 0x5, 0xb0, 0x59, 0x2, 0x63b, 0x63c, 0x7, 0x9e, 0x2, 0x2, - 0x63c, 0x63d, 0x5, 0xb0, 0x59, 0x2, 0x63d, 0x63f, 0x3, 0x2, 0x2, 0x2, - 0x63e, 0x639, 0x3, 0x2, 0x2, 0x2, 0x63f, 0x640, 0x3, 0x2, 0x2, 0x2, - 0x640, 0x63e, 0x3, 0x2, 0x2, 0x2, 0x640, 0x641, 0x3, 0x2, 0x2, 0x2, - 0x641, 0x644, 0x3, 0x2, 0x2, 0x2, 0x642, 0x643, 0x7, 0x34, 0x2, 0x2, - 0x643, 0x645, 0x5, 0xb0, 0x59, 0x2, 0x644, 0x642, 0x3, 0x2, 0x2, 0x2, - 0x644, 0x645, 0x3, 0x2, 0x2, 0x2, 0x645, 0x646, 0x3, 0x2, 0x2, 0x2, - 0x646, 0x647, 0x7, 0x35, 0x2, 0x2, 0x647, 0x6a0, 0x3, 0x2, 0x2, 0x2, - 0x648, 0x649, 0x7, 0x16, 0x2, 0x2, 0x649, 0x64a, 0x7, 0xd0, 0x2, 0x2, - 0x64a, 0x64b, 0x5, 0xb0, 0x59, 0x2, 0x64b, 0x64c, 0x7, 0xc, 0x2, 0x2, - 0x64c, 0x64d, 0x5, 0xaa, 0x56, 0x2, 0x64d, 0x64e, 0x7, 0xda, 0x2, 0x2, - 0x64e, 0x6a0, 0x3, 0x2, 0x2, 0x2, 0x64f, 0x650, 0x7, 0x24, 0x2, 0x2, - 0x650, 0x6a0, 0x7, 0xbf, 0x2, 0x2, 0x651, 0x652, 0x7, 0x3b, 0x2, 0x2, - 0x652, 0x653, 0x7, 0xd0, 0x2, 0x2, 0x653, 0x654, 0x5, 0xce, 0x68, 0x2, - 0x654, 0x655, 0x7, 0x43, 0x2, 0x2, 0x655, 0x656, 0x5, 0xb0, 0x59, 0x2, - 0x656, 0x657, 0x7, 0xda, 0x2, 0x2, 0x657, 0x6a0, 0x3, 0x2, 0x2, 0x2, - 0x658, 0x659, 0x7, 0x55, 0x2, 0x2, 0x659, 0x65a, 0x5, 0xb0, 0x59, 0x2, - 0x65a, 0x65b, 0x5, 0xce, 0x68, 0x2, 0x65b, 0x6a0, 0x3, 0x2, 0x2, 0x2, - 0x65c, 0x65d, 0x7, 0x96, 0x2, 0x2, 0x65d, 0x65e, 0x7, 0xd0, 0x2, 0x2, - 0x65e, 0x65f, 0x5, 0xb0, 0x59, 0x2, 0x65f, 0x660, 0x7, 0x43, 0x2, 0x2, - 0x660, 0x663, 0x5, 0xb0, 0x59, 0x2, 0x661, 0x662, 0x7, 0x40, 0x2, 0x2, - 0x662, 0x664, 0x5, 0xb0, 0x59, 0x2, 0x663, 0x661, 0x3, 0x2, 0x2, 0x2, - 0x663, 0x664, 0x3, 0x2, 0x2, 0x2, 0x664, 0x665, 0x3, 0x2, 0x2, 0x2, - 0x665, 0x666, 0x7, 0xda, 0x2, 0x2, 0x666, 0x6a0, 0x3, 0x2, 0x2, 0x2, - 0x667, 0x668, 0x7, 0xa1, 0x2, 0x2, 0x668, 0x6a0, 0x7, 0xbf, 0x2, 0x2, - 0x669, 0x66a, 0x7, 0xa6, 0x2, 0x2, 0x66a, 0x66b, 0x7, 0xd0, 0x2, 0x2, - 0x66b, 0x66c, 0x9, 0x14, 0x2, 0x2, 0x66c, 0x66d, 0x7, 0xbf, 0x2, 0x2, - 0x66d, 0x66e, 0x7, 0x43, 0x2, 0x2, 0x66e, 0x66f, 0x5, 0xb0, 0x59, 0x2, - 0x66f, 0x670, 0x7, 0xda, 0x2, 0x2, 0x670, 0x6a0, 0x3, 0x2, 0x2, 0x2, - 0x671, 0x677, 0x5, 0xd6, 0x6c, 0x2, 0x672, 0x674, 0x7, 0xd0, 0x2, 0x2, - 0x673, 0x675, 0x5, 0xac, 0x57, 0x2, 0x674, 0x673, 0x3, 0x2, 0x2, 0x2, - 0x674, 0x675, 0x3, 0x2, 0x2, 0x2, 0x675, 0x676, 0x3, 0x2, 0x2, 0x2, - 0x676, 0x678, 0x7, 0xda, 0x2, 0x2, 0x677, 0x672, 0x3, 0x2, 0x2, 0x2, - 0x677, 0x678, 0x3, 0x2, 0x2, 0x2, 0x678, 0x679, 0x3, 0x2, 0x2, 0x2, - 0x679, 0x67b, 0x7, 0xd0, 0x2, 0x2, 0x67a, 0x67c, 0x7, 0x31, 0x2, 0x2, - 0x67b, 0x67a, 0x3, 0x2, 0x2, 0x2, 0x67b, 0x67c, 0x3, 0x2, 0x2, 0x2, - 0x67c, 0x67e, 0x3, 0x2, 0x2, 0x2, 0x67d, 0x67f, 0x5, 0xb2, 0x5a, 0x2, - 0x67e, 0x67d, 0x3, 0x2, 0x2, 0x2, 0x67e, 0x67f, 0x3, 0x2, 0x2, 0x2, - 0x67f, 0x680, 0x3, 0x2, 0x2, 0x2, 0x680, 0x681, 0x7, 0xda, 0x2, 0x2, - 0x681, 0x6a0, 0x3, 0x2, 0x2, 0x2, 0x682, 0x6a0, 0x5, 0xcc, 0x67, 0x2, - 0x683, 0x684, 0x7, 0xc7, 0x2, 0x2, 0x684, 0x6a0, 0x5, 0xb0, 0x59, 0x13, - 0x685, 0x686, 0x7, 0x72, 0x2, 0x2, 0x686, 0x6a0, 0x5, 0xb0, 0x59, 0xe, - 0x687, 0x688, 0x5, 0xc0, 0x61, 0x2, 0x688, 0x689, 0x7, 0xc8, 0x2, 0x2, - 0x689, 0x68b, 0x3, 0x2, 0x2, 0x2, 0x68a, 0x687, 0x3, 0x2, 0x2, 0x2, - 0x68a, 0x68b, 0x3, 0x2, 0x2, 0x2, 0x68b, 0x68c, 0x3, 0x2, 0x2, 0x2, - 0x68c, 0x6a0, 0x7, 0xc1, 0x2, 0x2, 0x68d, 0x68e, 0x7, 0xd0, 0x2, 0x2, - 0x68e, 0x68f, 0x5, 0x68, 0x35, 0x2, 0x68f, 0x690, 0x7, 0xda, 0x2, 0x2, - 0x690, 0x6a0, 0x3, 0x2, 0x2, 0x2, 0x691, 0x692, 0x7, 0xd0, 0x2, 0x2, - 0x692, 0x693, 0x5, 0xb0, 0x59, 0x2, 0x693, 0x694, 0x7, 0xda, 0x2, 0x2, - 0x694, 0x6a0, 0x3, 0x2, 0x2, 0x2, 0x695, 0x696, 0x7, 0xd0, 0x2, 0x2, - 0x696, 0x697, 0x5, 0xac, 0x57, 0x2, 0x697, 0x698, 0x7, 0xda, 0x2, 0x2, - 0x698, 0x6a0, 0x3, 0x2, 0x2, 0x2, 0x699, 0x69b, 0x7, 0xce, 0x2, 0x2, - 0x69a, 0x69c, 0x5, 0xac, 0x57, 0x2, 0x69b, 0x69a, 0x3, 0x2, 0x2, 0x2, - 0x69b, 0x69c, 0x3, 0x2, 0x2, 0x2, 0x69c, 0x69d, 0x3, 0x2, 0x2, 0x2, - 0x69d, 0x6a0, 0x7, 0xd9, 0x2, 0x2, 0x69e, 0x6a0, 0x5, 0xb8, 0x5d, 0x2, - 0x69f, 0x634, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x648, 0x3, 0x2, 0x2, 0x2, - 0x69f, 0x64f, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x651, 0x3, 0x2, 0x2, 0x2, - 0x69f, 0x658, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x65c, 0x3, 0x2, 0x2, 0x2, - 0x69f, 0x667, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x669, 0x3, 0x2, 0x2, 0x2, - 0x69f, 0x671, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x682, 0x3, 0x2, 0x2, 0x2, - 0x69f, 0x683, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x685, 0x3, 0x2, 0x2, 0x2, - 0x69f, 0x68a, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x68d, 0x3, 0x2, 0x2, 0x2, - 0x69f, 0x691, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x695, 0x3, 0x2, 0x2, 0x2, - 0x69f, 0x699, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x69e, 0x3, 0x2, 0x2, 0x2, - 0x6a0, 0x6e8, 0x3, 0x2, 0x2, 0x2, 0x6a1, 0x6a2, 0xc, 0x12, 0x2, 0x2, - 0x6a2, 0x6a3, 0x9, 0x15, 0x2, 0x2, 0x6a3, 0x6e7, 0x5, 0xb0, 0x59, 0x13, - 0x6a4, 0x6a5, 0xc, 0x11, 0x2, 0x2, 0x6a5, 0x6a6, 0x9, 0x16, 0x2, 0x2, - 0x6a6, 0x6e7, 0x5, 0xb0, 0x59, 0x12, 0x6a7, 0x6ba, 0xc, 0x10, 0x2, 0x2, - 0x6a8, 0x6bb, 0x7, 0xc9, 0x2, 0x2, 0x6a9, 0x6bb, 0x7, 0xca, 0x2, 0x2, - 0x6aa, 0x6bb, 0x7, 0xd2, 0x2, 0x2, 0x6ab, 0x6bb, 0x7, 0xcf, 0x2, 0x2, - 0x6ac, 0x6bb, 0x7, 0xcb, 0x2, 0x2, 0x6ad, 0x6bb, 0x7, 0xd1, 0x2, 0x2, - 0x6ae, 0x6bb, 0x7, 0xcc, 0x2, 0x2, 0x6af, 0x6b1, 0x7, 0x46, 0x2, 0x2, - 0x6b0, 0x6af, 0x3, 0x2, 0x2, 0x2, 0x6b0, 0x6b1, 0x3, 0x2, 0x2, 0x2, - 0x6b1, 0x6b3, 0x3, 0x2, 0x2, 0x2, 0x6b2, 0x6b4, 0x7, 0x72, 0x2, 0x2, - 0x6b3, 0x6b2, 0x3, 0x2, 0x2, 0x2, 0x6b3, 0x6b4, 0x3, 0x2, 0x2, 0x2, - 0x6b4, 0x6b5, 0x3, 0x2, 0x2, 0x2, 0x6b5, 0x6bb, 0x7, 0x4f, 0x2, 0x2, - 0x6b6, 0x6b8, 0x7, 0x72, 0x2, 0x2, 0x6b7, 0x6b6, 0x3, 0x2, 0x2, 0x2, - 0x6b7, 0x6b8, 0x3, 0x2, 0x2, 0x2, 0x6b8, 0x6b9, 0x3, 0x2, 0x2, 0x2, - 0x6b9, 0x6bb, 0x9, 0x17, 0x2, 0x2, 0x6ba, 0x6a8, 0x3, 0x2, 0x2, 0x2, - 0x6ba, 0x6a9, 0x3, 0x2, 0x2, 0x2, 0x6ba, 0x6aa, 0x3, 0x2, 0x2, 0x2, - 0x6ba, 0x6ab, 0x3, 0x2, 0x2, 0x2, 0x6ba, 0x6ac, 0x3, 0x2, 0x2, 0x2, - 0x6ba, 0x6ad, 0x3, 0x2, 0x2, 0x2, 0x6ba, 0x6ae, 0x3, 0x2, 0x2, 0x2, - 0x6ba, 0x6b0, 0x3, 0x2, 0x2, 0x2, 0x6ba, 0x6b7, 0x3, 0x2, 0x2, 0x2, - 0x6bb, 0x6bc, 0x3, 0x2, 0x2, 0x2, 0x6bc, 0x6e7, 0x5, 0xb0, 0x59, 0x11, - 0x6bd, 0x6be, 0xc, 0xd, 0x2, 0x2, 0x6be, 0x6bf, 0x7, 0x8, 0x2, 0x2, - 0x6bf, 0x6e7, 0x5, 0xb0, 0x59, 0xe, 0x6c0, 0x6c1, 0xc, 0xc, 0x2, 0x2, - 0x6c1, 0x6c2, 0x7, 0x78, 0x2, 0x2, 0x6c2, 0x6e7, 0x5, 0xb0, 0x59, 0xd, - 0x6c3, 0x6c5, 0xc, 0xb, 0x2, 0x2, 0x6c4, 0x6c6, 0x7, 0x72, 0x2, 0x2, - 0x6c5, 0x6c4, 0x3, 0x2, 0x2, 0x2, 0x6c5, 0x6c6, 0x3, 0x2, 0x2, 0x2, - 0x6c6, 0x6c7, 0x3, 0x2, 0x2, 0x2, 0x6c7, 0x6c8, 0x7, 0x12, 0x2, 0x2, - 0x6c8, 0x6c9, 0x5, 0xb0, 0x59, 0x2, 0x6c9, 0x6ca, 0x7, 0x8, 0x2, 0x2, - 0x6ca, 0x6cb, 0x5, 0xb0, 0x59, 0xc, 0x6cb, 0x6e7, 0x3, 0x2, 0x2, 0x2, - 0x6cc, 0x6cd, 0xc, 0xa, 0x2, 0x2, 0x6cd, 0x6ce, 0x7, 0xd5, 0x2, 0x2, - 0x6ce, 0x6cf, 0x5, 0xb0, 0x59, 0x2, 0x6cf, 0x6d0, 0x7, 0xc4, 0x2, 0x2, - 0x6d0, 0x6d1, 0x5, 0xb0, 0x59, 0xa, 0x6d1, 0x6e7, 0x3, 0x2, 0x2, 0x2, - 0x6d2, 0x6d3, 0xc, 0x15, 0x2, 0x2, 0x6d3, 0x6d4, 0x7, 0xce, 0x2, 0x2, - 0x6d4, 0x6d5, 0x5, 0xb0, 0x59, 0x2, 0x6d5, 0x6d6, 0x7, 0xd9, 0x2, 0x2, - 0x6d6, 0x6e7, 0x3, 0x2, 0x2, 0x2, 0x6d7, 0x6d8, 0xc, 0x14, 0x2, 0x2, - 0x6d8, 0x6d9, 0x7, 0xc8, 0x2, 0x2, 0x6d9, 0x6e7, 0x7, 0xbd, 0x2, 0x2, - 0x6da, 0x6db, 0xc, 0xf, 0x2, 0x2, 0x6db, 0x6dd, 0x7, 0x57, 0x2, 0x2, - 0x6dc, 0x6de, 0x7, 0x72, 0x2, 0x2, 0x6dd, 0x6dc, 0x3, 0x2, 0x2, 0x2, - 0x6dd, 0x6de, 0x3, 0x2, 0x2, 0x2, 0x6de, 0x6df, 0x3, 0x2, 0x2, 0x2, - 0x6df, 0x6e7, 0x7, 0x73, 0x2, 0x2, 0x6e0, 0x6e4, 0xc, 0x9, 0x2, 0x2, - 0x6e1, 0x6e5, 0x5, 0xd4, 0x6b, 0x2, 0x6e2, 0x6e3, 0x7, 0xc, 0x2, 0x2, - 0x6e3, 0x6e5, 0x5, 0xd6, 0x6c, 0x2, 0x6e4, 0x6e1, 0x3, 0x2, 0x2, 0x2, - 0x6e4, 0x6e2, 0x3, 0x2, 0x2, 0x2, 0x6e5, 0x6e7, 0x3, 0x2, 0x2, 0x2, - 0x6e6, 0x6a1, 0x3, 0x2, 0x2, 0x2, 0x6e6, 0x6a4, 0x3, 0x2, 0x2, 0x2, - 0x6e6, 0x6a7, 0x3, 0x2, 0x2, 0x2, 0x6e6, 0x6bd, 0x3, 0x2, 0x2, 0x2, - 0x6e6, 0x6c0, 0x3, 0x2, 0x2, 0x2, 0x6e6, 0x6c3, 0x3, 0x2, 0x2, 0x2, - 0x6e6, 0x6cc, 0x3, 0x2, 0x2, 0x2, 0x6e6, 0x6d2, 0x3, 0x2, 0x2, 0x2, - 0x6e6, 0x6d7, 0x3, 0x2, 0x2, 0x2, 0x6e6, 0x6da, 0x3, 0x2, 0x2, 0x2, - 0x6e6, 0x6e0, 0x3, 0x2, 0x2, 0x2, 0x6e7, 0x6ea, 0x3, 0x2, 0x2, 0x2, - 0x6e8, 0x6e6, 0x3, 0x2, 0x2, 0x2, 0x6e8, 0x6e9, 0x3, 0x2, 0x2, 0x2, - 0x6e9, 0xb1, 0x3, 0x2, 0x2, 0x2, 0x6ea, 0x6e8, 0x3, 0x2, 0x2, 0x2, 0x6eb, - 0x6f0, 0x5, 0xb4, 0x5b, 0x2, 0x6ec, 0x6ed, 0x7, 0xc5, 0x2, 0x2, 0x6ed, - 0x6ef, 0x5, 0xb4, 0x5b, 0x2, 0x6ee, 0x6ec, 0x3, 0x2, 0x2, 0x2, 0x6ef, - 0x6f2, 0x3, 0x2, 0x2, 0x2, 0x6f0, 0x6ee, 0x3, 0x2, 0x2, 0x2, 0x6f0, - 0x6f1, 0x3, 0x2, 0x2, 0x2, 0x6f1, 0xb3, 0x3, 0x2, 0x2, 0x2, 0x6f2, 0x6f0, - 0x3, 0x2, 0x2, 0x2, 0x6f3, 0x6f6, 0x5, 0xb6, 0x5c, 0x2, 0x6f4, 0x6f6, - 0x5, 0xb0, 0x59, 0x2, 0x6f5, 0x6f3, 0x3, 0x2, 0x2, 0x2, 0x6f5, 0x6f4, - 0x3, 0x2, 0x2, 0x2, 0x6f6, 0xb5, 0x3, 0x2, 0x2, 0x2, 0x6f7, 0x6f8, 0x7, - 0xd0, 0x2, 0x2, 0x6f8, 0x6fd, 0x5, 0xd6, 0x6c, 0x2, 0x6f9, 0x6fa, 0x7, - 0xc5, 0x2, 0x2, 0x6fa, 0x6fc, 0x5, 0xd6, 0x6c, 0x2, 0x6fb, 0x6f9, 0x3, - 0x2, 0x2, 0x2, 0x6fc, 0x6ff, 0x3, 0x2, 0x2, 0x2, 0x6fd, 0x6fb, 0x3, - 0x2, 0x2, 0x2, 0x6fd, 0x6fe, 0x3, 0x2, 0x2, 0x2, 0x6fe, 0x700, 0x3, - 0x2, 0x2, 0x2, 0x6ff, 0x6fd, 0x3, 0x2, 0x2, 0x2, 0x700, 0x701, 0x7, - 0xda, 0x2, 0x2, 0x701, 0x70b, 0x3, 0x2, 0x2, 0x2, 0x702, 0x707, 0x5, - 0xd6, 0x6c, 0x2, 0x703, 0x704, 0x7, 0xc5, 0x2, 0x2, 0x704, 0x706, 0x5, - 0xd6, 0x6c, 0x2, 0x705, 0x703, 0x3, 0x2, 0x2, 0x2, 0x706, 0x709, 0x3, - 0x2, 0x2, 0x2, 0x707, 0x705, 0x3, 0x2, 0x2, 0x2, 0x707, 0x708, 0x3, - 0x2, 0x2, 0x2, 0x708, 0x70b, 0x3, 0x2, 0x2, 0x2, 0x709, 0x707, 0x3, - 0x2, 0x2, 0x2, 0x70a, 0x6f7, 0x3, 0x2, 0x2, 0x2, 0x70a, 0x702, 0x3, - 0x2, 0x2, 0x2, 0x70b, 0x70c, 0x3, 0x2, 0x2, 0x2, 0x70c, 0x70d, 0x7, - 0xc0, 0x2, 0x2, 0x70d, 0x70e, 0x5, 0xb0, 0x59, 0x2, 0x70e, 0xb7, 0x3, - 0x2, 0x2, 0x2, 0x70f, 0x710, 0x5, 0xc0, 0x61, 0x2, 0x710, 0x711, 0x7, - 0xc8, 0x2, 0x2, 0x711, 0x713, 0x3, 0x2, 0x2, 0x2, 0x712, 0x70f, 0x3, - 0x2, 0x2, 0x2, 0x712, 0x713, 0x3, 0x2, 0x2, 0x2, 0x713, 0x714, 0x3, - 0x2, 0x2, 0x2, 0x714, 0x715, 0x5, 0xba, 0x5e, 0x2, 0x715, 0xb9, 0x3, - 0x2, 0x2, 0x2, 0x716, 0x719, 0x5, 0xd6, 0x6c, 0x2, 0x717, 0x718, 0x7, - 0xc8, 0x2, 0x2, 0x718, 0x71a, 0x5, 0xd6, 0x6c, 0x2, 0x719, 0x717, 0x3, - 0x2, 0x2, 0x2, 0x719, 0x71a, 0x3, 0x2, 0x2, 0x2, 0x71a, 0xbb, 0x3, 0x2, - 0x2, 0x2, 0x71b, 0x71c, 0x8, 0x5f, 0x1, 0x2, 0x71c, 0x723, 0x5, 0xc0, - 0x61, 0x2, 0x71d, 0x723, 0x5, 0xbe, 0x60, 0x2, 0x71e, 0x71f, 0x7, 0xd0, - 0x2, 0x2, 0x71f, 0x720, 0x5, 0x68, 0x35, 0x2, 0x720, 0x721, 0x7, 0xda, - 0x2, 0x2, 0x721, 0x723, 0x3, 0x2, 0x2, 0x2, 0x722, 0x71b, 0x3, 0x2, - 0x2, 0x2, 0x722, 0x71d, 0x3, 0x2, 0x2, 0x2, 0x722, 0x71e, 0x3, 0x2, - 0x2, 0x2, 0x723, 0x72c, 0x3, 0x2, 0x2, 0x2, 0x724, 0x728, 0xc, 0x3, - 0x2, 0x2, 0x725, 0x729, 0x5, 0xd4, 0x6b, 0x2, 0x726, 0x727, 0x7, 0xc, - 0x2, 0x2, 0x727, 0x729, 0x5, 0xd6, 0x6c, 0x2, 0x728, 0x725, 0x3, 0x2, - 0x2, 0x2, 0x728, 0x726, 0x3, 0x2, 0x2, 0x2, 0x729, 0x72b, 0x3, 0x2, - 0x2, 0x2, 0x72a, 0x724, 0x3, 0x2, 0x2, 0x2, 0x72b, 0x72e, 0x3, 0x2, - 0x2, 0x2, 0x72c, 0x72a, 0x3, 0x2, 0x2, 0x2, 0x72c, 0x72d, 0x3, 0x2, - 0x2, 0x2, 0x72d, 0xbd, 0x3, 0x2, 0x2, 0x2, 0x72e, 0x72c, 0x3, 0x2, 0x2, - 0x2, 0x72f, 0x730, 0x5, 0xd6, 0x6c, 0x2, 0x730, 0x732, 0x7, 0xd0, 0x2, - 0x2, 0x731, 0x733, 0x5, 0xc2, 0x62, 0x2, 0x732, 0x731, 0x3, 0x2, 0x2, - 0x2, 0x732, 0x733, 0x3, 0x2, 0x2, 0x2, 0x733, 0x734, 0x3, 0x2, 0x2, - 0x2, 0x734, 0x735, 0x7, 0xda, 0x2, 0x2, 0x735, 0xbf, 0x3, 0x2, 0x2, - 0x2, 0x736, 0x737, 0x5, 0xc6, 0x64, 0x2, 0x737, 0x738, 0x7, 0xc8, 0x2, - 0x2, 0x738, 0x73a, 0x3, 0x2, 0x2, 0x2, 0x739, 0x736, 0x3, 0x2, 0x2, - 0x2, 0x739, 0x73a, 0x3, 0x2, 0x2, 0x2, 0x73a, 0x73b, 0x3, 0x2, 0x2, - 0x2, 0x73b, 0x73c, 0x5, 0xd6, 0x6c, 0x2, 0x73c, 0xc1, 0x3, 0x2, 0x2, - 0x2, 0x73d, 0x742, 0x5, 0xc4, 0x63, 0x2, 0x73e, 0x73f, 0x7, 0xc5, 0x2, - 0x2, 0x73f, 0x741, 0x5, 0xc4, 0x63, 0x2, 0x740, 0x73e, 0x3, 0x2, 0x2, - 0x2, 0x741, 0x744, 0x3, 0x2, 0x2, 0x2, 0x742, 0x740, 0x3, 0x2, 0x2, - 0x2, 0x742, 0x743, 0x3, 0x2, 0x2, 0x2, 0x743, 0xc3, 0x3, 0x2, 0x2, 0x2, - 0x744, 0x742, 0x3, 0x2, 0x2, 0x2, 0x745, 0x749, 0x5, 0xba, 0x5e, 0x2, - 0x746, 0x749, 0x5, 0xbe, 0x60, 0x2, 0x747, 0x749, 0x5, 0xcc, 0x67, 0x2, - 0x748, 0x745, 0x3, 0x2, 0x2, 0x2, 0x748, 0x746, 0x3, 0x2, 0x2, 0x2, - 0x748, 0x747, 0x3, 0x2, 0x2, 0x2, 0x749, 0xc5, 0x3, 0x2, 0x2, 0x2, 0x74a, - 0x74b, 0x5, 0xd6, 0x6c, 0x2, 0x74b, 0xc7, 0x3, 0x2, 0x2, 0x2, 0x74c, - 0x755, 0x7, 0xbb, 0x2, 0x2, 0x74d, 0x74e, 0x7, 0xc8, 0x2, 0x2, 0x74e, - 0x755, 0x9, 0x18, 0x2, 0x2, 0x74f, 0x750, 0x7, 0xbd, 0x2, 0x2, 0x750, - 0x752, 0x7, 0xc8, 0x2, 0x2, 0x751, 0x753, 0x9, 0x18, 0x2, 0x2, 0x752, - 0x751, 0x3, 0x2, 0x2, 0x2, 0x752, 0x753, 0x3, 0x2, 0x2, 0x2, 0x753, - 0x755, 0x3, 0x2, 0x2, 0x2, 0x754, 0x74c, 0x3, 0x2, 0x2, 0x2, 0x754, - 0x74d, 0x3, 0x2, 0x2, 0x2, 0x754, 0x74f, 0x3, 0x2, 0x2, 0x2, 0x755, - 0xc9, 0x3, 0x2, 0x2, 0x2, 0x756, 0x758, 0x9, 0x19, 0x2, 0x2, 0x757, - 0x756, 0x3, 0x2, 0x2, 0x2, 0x757, 0x758, 0x3, 0x2, 0x2, 0x2, 0x758, - 0x75f, 0x3, 0x2, 0x2, 0x2, 0x759, 0x760, 0x5, 0xc8, 0x65, 0x2, 0x75a, - 0x760, 0x7, 0xbc, 0x2, 0x2, 0x75b, 0x760, 0x7, 0xbd, 0x2, 0x2, 0x75c, - 0x760, 0x7, 0xbe, 0x2, 0x2, 0x75d, 0x760, 0x7, 0x51, 0x2, 0x2, 0x75e, - 0x760, 0x7, 0x70, 0x2, 0x2, 0x75f, 0x759, 0x3, 0x2, 0x2, 0x2, 0x75f, - 0x75a, 0x3, 0x2, 0x2, 0x2, 0x75f, 0x75b, 0x3, 0x2, 0x2, 0x2, 0x75f, - 0x75c, 0x3, 0x2, 0x2, 0x2, 0x75f, 0x75d, 0x3, 0x2, 0x2, 0x2, 0x75f, - 0x75e, 0x3, 0x2, 0x2, 0x2, 0x760, 0xcb, 0x3, 0x2, 0x2, 0x2, 0x761, 0x765, - 0x5, 0xca, 0x66, 0x2, 0x762, 0x765, 0x7, 0xbf, 0x2, 0x2, 0x763, 0x765, - 0x7, 0x73, 0x2, 0x2, 0x764, 0x761, 0x3, 0x2, 0x2, 0x2, 0x764, 0x762, - 0x3, 0x2, 0x2, 0x2, 0x764, 0x763, 0x3, 0x2, 0x2, 0x2, 0x765, 0xcd, 0x3, - 0x2, 0x2, 0x2, 0x766, 0x767, 0x9, 0x1a, 0x2, 0x2, 0x767, 0xcf, 0x3, - 0x2, 0x2, 0x2, 0x768, 0x769, 0x9, 0x1b, 0x2, 0x2, 0x769, 0xd1, 0x3, - 0x2, 0x2, 0x2, 0x76a, 0x76b, 0x9, 0x1c, 0x2, 0x2, 0x76b, 0xd3, 0x3, - 0x2, 0x2, 0x2, 0x76c, 0x76f, 0x7, 0xba, 0x2, 0x2, 0x76d, 0x76f, 0x5, - 0xd2, 0x6a, 0x2, 0x76e, 0x76c, 0x3, 0x2, 0x2, 0x2, 0x76e, 0x76d, 0x3, - 0x2, 0x2, 0x2, 0x76f, 0xd5, 0x3, 0x2, 0x2, 0x2, 0x770, 0x774, 0x7, 0xba, - 0x2, 0x2, 0x771, 0x774, 0x5, 0xce, 0x68, 0x2, 0x772, 0x774, 0x5, 0xd0, - 0x69, 0x2, 0x773, 0x770, 0x3, 0x2, 0x2, 0x2, 0x773, 0x771, 0x3, 0x2, - 0x2, 0x2, 0x773, 0x772, 0x3, 0x2, 0x2, 0x2, 0x774, 0xd7, 0x3, 0x2, 0x2, - 0x2, 0x775, 0x778, 0x5, 0xd6, 0x6c, 0x2, 0x776, 0x778, 0x7, 0x73, 0x2, - 0x2, 0x777, 0x775, 0x3, 0x2, 0x2, 0x2, 0x777, 0x776, 0x3, 0x2, 0x2, - 0x2, 0x778, 0xd9, 0x3, 0x2, 0x2, 0x2, 0x779, 0x77a, 0x7, 0xbf, 0x2, - 0x2, 0x77a, 0x77b, 0x7, 0xca, 0x2, 0x2, 0x77b, 0x77c, 0x5, 0xca, 0x66, - 0x2, 0x77c, 0xdb, 0x3, 0x2, 0x2, 0x2, 0x103, 0xe0, 0xe4, 0xe7, 0xea, + 0x2, 0x3a8, 0x3ba, 0x3, 0x2, 0x2, 0x2, 0x3a9, 0x3ab, 0x5, 0xba, 0x5e, + 0x2, 0x3aa, 0x3ac, 0x5, 0xaa, 0x56, 0x2, 0x3ab, 0x3aa, 0x3, 0x2, 0x2, + 0x2, 0x3ab, 0x3ac, 0x3, 0x2, 0x2, 0x2, 0x3ac, 0x3ad, 0x3, 0x2, 0x2, + 0x2, 0x3ad, 0x3b0, 0x5, 0x46, 0x24, 0x2, 0x3ae, 0x3af, 0x7, 0x1d, 0x2, + 0x2, 0x3af, 0x3b1, 0x7, 0xbf, 0x2, 0x2, 0x3b0, 0x3ae, 0x3, 0x2, 0x2, + 0x2, 0x3b0, 0x3b1, 0x3, 0x2, 0x2, 0x2, 0x3b1, 0x3b3, 0x3, 0x2, 0x2, + 0x2, 0x3b2, 0x3b4, 0x5, 0x4c, 0x27, 0x2, 0x3b3, 0x3b2, 0x3, 0x2, 0x2, + 0x2, 0x3b3, 0x3b4, 0x3, 0x2, 0x2, 0x2, 0x3b4, 0x3b7, 0x3, 0x2, 0x2, + 0x2, 0x3b5, 0x3b6, 0x7, 0xa8, 0x2, 0x2, 0x3b6, 0x3b8, 0x5, 0xb0, 0x59, + 0x2, 0x3b7, 0x3b5, 0x3, 0x2, 0x2, 0x2, 0x3b7, 0x3b8, 0x3, 0x2, 0x2, + 0x2, 0x3b8, 0x3ba, 0x3, 0x2, 0x2, 0x2, 0x3b9, 0x399, 0x3, 0x2, 0x2, + 0x2, 0x3b9, 0x3a9, 0x3, 0x2, 0x2, 0x2, 0x3ba, 0x45, 0x3, 0x2, 0x2, 0x2, + 0x3bb, 0x3bc, 0x9, 0x4, 0x2, 0x2, 0x3bc, 0x3bd, 0x5, 0xb0, 0x59, 0x2, + 0x3bd, 0x47, 0x3, 0x2, 0x2, 0x2, 0x3be, 0x3bf, 0x5, 0xba, 0x5e, 0x2, + 0x3bf, 0x3c0, 0x5, 0xb0, 0x59, 0x2, 0x3c0, 0x3c1, 0x7, 0xa9, 0x2, 0x2, + 0x3c1, 0x3c2, 0x5, 0xaa, 0x56, 0x2, 0x3c2, 0x3c3, 0x7, 0x47, 0x2, 0x2, + 0x3c3, 0x3c4, 0x7, 0xbd, 0x2, 0x2, 0x3c4, 0x49, 0x3, 0x2, 0x2, 0x2, + 0x3c5, 0x3c6, 0x5, 0xba, 0x5e, 0x2, 0x3c6, 0x3c7, 0x5, 0x66, 0x34, 0x2, + 0x3c7, 0x4b, 0x3, 0x2, 0x2, 0x2, 0x3c8, 0x3c9, 0x7, 0x1a, 0x2, 0x2, + 0x3c9, 0x3ca, 0x7, 0xd0, 0x2, 0x2, 0x3ca, 0x3cf, 0x5, 0x4e, 0x28, 0x2, + 0x3cb, 0x3cc, 0x7, 0xc5, 0x2, 0x2, 0x3cc, 0x3ce, 0x5, 0x4e, 0x28, 0x2, + 0x3cd, 0x3cb, 0x3, 0x2, 0x2, 0x2, 0x3ce, 0x3d1, 0x3, 0x2, 0x2, 0x2, + 0x3cf, 0x3cd, 0x3, 0x2, 0x2, 0x2, 0x3cf, 0x3d0, 0x3, 0x2, 0x2, 0x2, + 0x3d0, 0x3d2, 0x3, 0x2, 0x2, 0x2, 0x3d1, 0x3cf, 0x3, 0x2, 0x2, 0x2, + 0x3d2, 0x3d3, 0x7, 0xda, 0x2, 0x2, 0x3d3, 0x4d, 0x3, 0x2, 0x2, 0x2, + 0x3d4, 0x3da, 0x5, 0xd6, 0x6c, 0x2, 0x3d5, 0x3d7, 0x7, 0xd0, 0x2, 0x2, + 0x3d6, 0x3d8, 0x5, 0xac, 0x57, 0x2, 0x3d7, 0x3d6, 0x3, 0x2, 0x2, 0x2, + 0x3d7, 0x3d8, 0x3, 0x2, 0x2, 0x2, 0x3d8, 0x3d9, 0x3, 0x2, 0x2, 0x2, + 0x3d9, 0x3db, 0x7, 0xda, 0x2, 0x2, 0x3da, 0x3d5, 0x3, 0x2, 0x2, 0x2, + 0x3da, 0x3db, 0x3, 0x2, 0x2, 0x2, 0x3db, 0x4f, 0x3, 0x2, 0x2, 0x2, 0x3dc, + 0x3e4, 0x5, 0xb0, 0x59, 0x2, 0x3dd, 0x3e5, 0x7, 0x29, 0x2, 0x2, 0x3de, + 0x3df, 0x7, 0xa2, 0x2, 0x2, 0x3df, 0x3e0, 0x7, 0x30, 0x2, 0x2, 0x3e0, + 0x3e5, 0x7, 0xbf, 0x2, 0x2, 0x3e1, 0x3e2, 0x7, 0xa2, 0x2, 0x2, 0x3e2, + 0x3e3, 0x7, 0xb1, 0x2, 0x2, 0x3e3, 0x3e5, 0x7, 0xbf, 0x2, 0x2, 0x3e4, + 0x3dd, 0x3, 0x2, 0x2, 0x2, 0x3e4, 0x3de, 0x3, 0x2, 0x2, 0x2, 0x3e4, + 0x3e1, 0x3, 0x2, 0x2, 0x2, 0x3e4, 0x3e5, 0x3, 0x2, 0x2, 0x2, 0x3e5, + 0x51, 0x3, 0x2, 0x2, 0x2, 0x3e6, 0x3e8, 0x9, 0x5, 0x2, 0x2, 0x3e7, 0x3e9, + 0x7, 0x9a, 0x2, 0x2, 0x3e8, 0x3e7, 0x3, 0x2, 0x2, 0x2, 0x3e8, 0x3e9, + 0x3, 0x2, 0x2, 0x2, 0x3e9, 0x3ea, 0x3, 0x2, 0x2, 0x2, 0x3ea, 0x3eb, + 0x5, 0xbc, 0x5f, 0x2, 0x3eb, 0x53, 0x3, 0x2, 0x2, 0x2, 0x3ec, 0x3ed, + 0x9, 0x6, 0x2, 0x2, 0x3ed, 0x3f0, 0x7, 0x22, 0x2, 0x2, 0x3ee, 0x3ef, + 0x7, 0x4d, 0x2, 0x2, 0x3ef, 0x3f1, 0x7, 0x38, 0x2, 0x2, 0x3f0, 0x3ee, + 0x3, 0x2, 0x2, 0x2, 0x3f0, 0x3f1, 0x3, 0x2, 0x2, 0x2, 0x3f1, 0x3f2, + 0x3, 0x2, 0x2, 0x2, 0x3f2, 0x3f4, 0x5, 0xc6, 0x64, 0x2, 0x3f3, 0x3f5, + 0x5, 0x2c, 0x17, 0x2, 0x3f4, 0x3f3, 0x3, 0x2, 0x2, 0x2, 0x3f4, 0x3f5, + 0x3, 0x2, 0x2, 0x2, 0x3f5, 0x40c, 0x3, 0x2, 0x2, 0x2, 0x3f6, 0x3fd, + 0x9, 0x6, 0x2, 0x2, 0x3f7, 0x3fe, 0x7, 0x2f, 0x2, 0x2, 0x3f8, 0x3fa, + 0x7, 0x9c, 0x2, 0x2, 0x3f9, 0x3f8, 0x3, 0x2, 0x2, 0x2, 0x3f9, 0x3fa, + 0x3, 0x2, 0x2, 0x2, 0x3fa, 0x3fb, 0x3, 0x2, 0x2, 0x2, 0x3fb, 0x3fe, + 0x7, 0x9a, 0x2, 0x2, 0x3fc, 0x3fe, 0x7, 0xb0, 0x2, 0x2, 0x3fd, 0x3f7, + 0x3, 0x2, 0x2, 0x2, 0x3fd, 0x3f9, 0x3, 0x2, 0x2, 0x2, 0x3fd, 0x3fc, + 0x3, 0x2, 0x2, 0x2, 0x3fe, 0x401, 0x3, 0x2, 0x2, 0x2, 0x3ff, 0x400, + 0x7, 0x4d, 0x2, 0x2, 0x400, 0x402, 0x7, 0x38, 0x2, 0x2, 0x401, 0x3ff, + 0x3, 0x2, 0x2, 0x2, 0x401, 0x402, 0x3, 0x2, 0x2, 0x2, 0x402, 0x403, + 0x3, 0x2, 0x2, 0x2, 0x403, 0x405, 0x5, 0xc0, 0x61, 0x2, 0x404, 0x406, + 0x5, 0x2c, 0x17, 0x2, 0x405, 0x404, 0x3, 0x2, 0x2, 0x2, 0x405, 0x406, + 0x3, 0x2, 0x2, 0x2, 0x406, 0x409, 0x3, 0x2, 0x2, 0x2, 0x407, 0x408, + 0x7, 0x71, 0x2, 0x2, 0x408, 0x40a, 0x7, 0x28, 0x2, 0x2, 0x409, 0x407, + 0x3, 0x2, 0x2, 0x2, 0x409, 0x40a, 0x3, 0x2, 0x2, 0x2, 0x40a, 0x40c, + 0x3, 0x2, 0x2, 0x2, 0x40b, 0x3ec, 0x3, 0x2, 0x2, 0x2, 0x40b, 0x3f6, + 0x3, 0x2, 0x2, 0x2, 0x40c, 0x55, 0x3, 0x2, 0x2, 0x2, 0x40d, 0x40e, 0x7, + 0x38, 0x2, 0x2, 0x40e, 0x40f, 0x7, 0x22, 0x2, 0x2, 0x40f, 0x41b, 0x5, + 0xc6, 0x64, 0x2, 0x410, 0x417, 0x7, 0x38, 0x2, 0x2, 0x411, 0x418, 0x7, + 0x2f, 0x2, 0x2, 0x412, 0x414, 0x7, 0x9c, 0x2, 0x2, 0x413, 0x412, 0x3, + 0x2, 0x2, 0x2, 0x413, 0x414, 0x3, 0x2, 0x2, 0x2, 0x414, 0x415, 0x3, + 0x2, 0x2, 0x2, 0x415, 0x418, 0x7, 0x9a, 0x2, 0x2, 0x416, 0x418, 0x7, + 0xb0, 0x2, 0x2, 0x417, 0x411, 0x3, 0x2, 0x2, 0x2, 0x417, 0x413, 0x3, + 0x2, 0x2, 0x2, 0x417, 0x416, 0x3, 0x2, 0x2, 0x2, 0x417, 0x418, 0x3, + 0x2, 0x2, 0x2, 0x418, 0x419, 0x3, 0x2, 0x2, 0x2, 0x419, 0x41b, 0x5, + 0xc0, 0x61, 0x2, 0x41a, 0x40d, 0x3, 0x2, 0x2, 0x2, 0x41a, 0x410, 0x3, + 0x2, 0x2, 0x2, 0x41b, 0x57, 0x3, 0x2, 0x2, 0x2, 0x41c, 0x41d, 0x7, 0x39, + 0x2, 0x2, 0x41d, 0x41e, 0x7, 0xf, 0x2, 0x2, 0x41e, 0x423, 0x5, 0x4, + 0x3, 0x2, 0x41f, 0x420, 0x7, 0x39, 0x2, 0x2, 0x420, 0x421, 0x7, 0x98, + 0x2, 0x2, 0x421, 0x423, 0x5, 0x4, 0x3, 0x2, 0x422, 0x41c, 0x3, 0x2, + 0x2, 0x2, 0x422, 0x41f, 0x3, 0x2, 0x2, 0x2, 0x423, 0x59, 0x3, 0x2, 0x2, + 0x2, 0x424, 0x425, 0x7, 0x54, 0x2, 0x2, 0x425, 0x427, 0x7, 0x56, 0x2, + 0x2, 0x426, 0x428, 0x7, 0x9a, 0x2, 0x2, 0x427, 0x426, 0x3, 0x2, 0x2, + 0x2, 0x427, 0x428, 0x3, 0x2, 0x2, 0x2, 0x428, 0x42c, 0x3, 0x2, 0x2, + 0x2, 0x429, 0x42d, 0x5, 0xc0, 0x61, 0x2, 0x42a, 0x42b, 0x7, 0x45, 0x2, + 0x2, 0x42b, 0x42d, 0x5, 0xbe, 0x60, 0x2, 0x42c, 0x429, 0x3, 0x2, 0x2, + 0x2, 0x42c, 0x42a, 0x3, 0x2, 0x2, 0x2, 0x42d, 0x42f, 0x3, 0x2, 0x2, + 0x2, 0x42e, 0x430, 0x5, 0x5c, 0x2f, 0x2, 0x42f, 0x42e, 0x3, 0x2, 0x2, + 0x2, 0x42f, 0x430, 0x3, 0x2, 0x2, 0x2, 0x430, 0x431, 0x3, 0x2, 0x2, + 0x2, 0x431, 0x432, 0x5, 0x5e, 0x30, 0x2, 0x432, 0x5b, 0x3, 0x2, 0x2, + 0x2, 0x433, 0x434, 0x7, 0xd0, 0x2, 0x2, 0x434, 0x439, 0x5, 0xba, 0x5e, + 0x2, 0x435, 0x436, 0x7, 0xc5, 0x2, 0x2, 0x436, 0x438, 0x5, 0xba, 0x5e, + 0x2, 0x437, 0x435, 0x3, 0x2, 0x2, 0x2, 0x438, 0x43b, 0x3, 0x2, 0x2, + 0x2, 0x439, 0x437, 0x3, 0x2, 0x2, 0x2, 0x439, 0x43a, 0x3, 0x2, 0x2, + 0x2, 0x43a, 0x43c, 0x3, 0x2, 0x2, 0x2, 0x43b, 0x439, 0x3, 0x2, 0x2, + 0x2, 0x43c, 0x43d, 0x7, 0xda, 0x2, 0x2, 0x43d, 0x5d, 0x3, 0x2, 0x2, + 0x2, 0x43e, 0x43f, 0x7, 0x41, 0x2, 0x2, 0x43f, 0x448, 0x5, 0xd6, 0x6c, + 0x2, 0x440, 0x448, 0x7, 0xaf, 0x2, 0x2, 0x441, 0x443, 0x5, 0x68, 0x35, + 0x2, 0x442, 0x444, 0x7, 0xdb, 0x2, 0x2, 0x443, 0x442, 0x3, 0x2, 0x2, + 0x2, 0x443, 0x444, 0x3, 0x2, 0x2, 0x2, 0x444, 0x445, 0x3, 0x2, 0x2, + 0x2, 0x445, 0x446, 0x7, 0x2, 0x2, 0x3, 0x446, 0x448, 0x3, 0x2, 0x2, + 0x2, 0x447, 0x43e, 0x3, 0x2, 0x2, 0x2, 0x447, 0x440, 0x3, 0x2, 0x2, + 0x2, 0x447, 0x441, 0x3, 0x2, 0x2, 0x2, 0x448, 0x5f, 0x3, 0x2, 0x2, 0x2, + 0x449, 0x44a, 0x7, 0x5b, 0x2, 0x2, 0x44a, 0x44c, 0x7, 0x6f, 0x2, 0x2, + 0x44b, 0x44d, 0x5, 0x2c, 0x17, 0x2, 0x44c, 0x44b, 0x3, 0x2, 0x2, 0x2, + 0x44c, 0x44d, 0x3, 0x2, 0x2, 0x2, 0x44d, 0x44e, 0x3, 0x2, 0x2, 0x2, + 0x44e, 0x450, 0x5, 0x78, 0x3d, 0x2, 0x44f, 0x451, 0x9, 0x7, 0x2, 0x2, + 0x450, 0x44f, 0x3, 0x2, 0x2, 0x2, 0x450, 0x451, 0x3, 0x2, 0x2, 0x2, + 0x451, 0x61, 0x3, 0x2, 0x2, 0x2, 0x452, 0x453, 0x7, 0x77, 0x2, 0x2, + 0x453, 0x454, 0x7, 0x9a, 0x2, 0x2, 0x454, 0x456, 0x5, 0xc0, 0x61, 0x2, + 0x455, 0x457, 0x5, 0x2c, 0x17, 0x2, 0x456, 0x455, 0x3, 0x2, 0x2, 0x2, + 0x456, 0x457, 0x3, 0x2, 0x2, 0x2, 0x457, 0x459, 0x3, 0x2, 0x2, 0x2, + 0x458, 0x45a, 0x5, 0x10, 0x9, 0x2, 0x459, 0x458, 0x3, 0x2, 0x2, 0x2, + 0x459, 0x45a, 0x3, 0x2, 0x2, 0x2, 0x45a, 0x45c, 0x3, 0x2, 0x2, 0x2, + 0x45b, 0x45d, 0x7, 0x3d, 0x2, 0x2, 0x45c, 0x45b, 0x3, 0x2, 0x2, 0x2, + 0x45c, 0x45d, 0x3, 0x2, 0x2, 0x2, 0x45d, 0x45f, 0x3, 0x2, 0x2, 0x2, + 0x45e, 0x460, 0x7, 0x26, 0x2, 0x2, 0x45f, 0x45e, 0x3, 0x2, 0x2, 0x2, + 0x45f, 0x460, 0x3, 0x2, 0x2, 0x2, 0x460, 0x63, 0x3, 0x2, 0x2, 0x2, 0x461, + 0x462, 0x7, 0x85, 0x2, 0x2, 0x462, 0x463, 0x7, 0x9a, 0x2, 0x2, 0x463, + 0x464, 0x5, 0xc0, 0x61, 0x2, 0x464, 0x465, 0x7, 0xa2, 0x2, 0x2, 0x465, + 0x46d, 0x5, 0xc0, 0x61, 0x2, 0x466, 0x467, 0x7, 0xc5, 0x2, 0x2, 0x467, + 0x468, 0x5, 0xc0, 0x61, 0x2, 0x468, 0x469, 0x7, 0xa2, 0x2, 0x2, 0x469, + 0x46a, 0x5, 0xc0, 0x61, 0x2, 0x46a, 0x46c, 0x3, 0x2, 0x2, 0x2, 0x46b, + 0x466, 0x3, 0x2, 0x2, 0x2, 0x46c, 0x46f, 0x3, 0x2, 0x2, 0x2, 0x46d, + 0x46b, 0x3, 0x2, 0x2, 0x2, 0x46d, 0x46e, 0x3, 0x2, 0x2, 0x2, 0x46e, + 0x471, 0x3, 0x2, 0x2, 0x2, 0x46f, 0x46d, 0x3, 0x2, 0x2, 0x2, 0x470, + 0x472, 0x5, 0x2c, 0x17, 0x2, 0x471, 0x470, 0x3, 0x2, 0x2, 0x2, 0x471, + 0x472, 0x3, 0x2, 0x2, 0x2, 0x472, 0x65, 0x3, 0x2, 0x2, 0x2, 0x473, 0x475, + 0x7, 0xd0, 0x2, 0x2, 0x474, 0x476, 0x5, 0x6e, 0x38, 0x2, 0x475, 0x474, + 0x3, 0x2, 0x2, 0x2, 0x475, 0x476, 0x3, 0x2, 0x2, 0x2, 0x476, 0x477, + 0x3, 0x2, 0x2, 0x2, 0x477, 0x478, 0x7, 0x8d, 0x2, 0x2, 0x478, 0x47a, + 0x5, 0xac, 0x57, 0x2, 0x479, 0x47b, 0x5, 0x7a, 0x3e, 0x2, 0x47a, 0x479, + 0x3, 0x2, 0x2, 0x2, 0x47a, 0x47b, 0x3, 0x2, 0x2, 0x2, 0x47b, 0x47d, + 0x3, 0x2, 0x2, 0x2, 0x47c, 0x47e, 0x5, 0x80, 0x41, 0x2, 0x47d, 0x47c, + 0x3, 0x2, 0x2, 0x2, 0x47d, 0x47e, 0x3, 0x2, 0x2, 0x2, 0x47e, 0x47f, + 0x3, 0x2, 0x2, 0x2, 0x47f, 0x480, 0x7, 0xda, 0x2, 0x2, 0x480, 0x67, + 0x3, 0x2, 0x2, 0x2, 0x481, 0x487, 0x5, 0x6a, 0x36, 0x2, 0x482, 0x483, + 0x7, 0xaa, 0x2, 0x2, 0x483, 0x484, 0x7, 0x6, 0x2, 0x2, 0x484, 0x486, + 0x5, 0x6a, 0x36, 0x2, 0x485, 0x482, 0x3, 0x2, 0x2, 0x2, 0x486, 0x489, + 0x3, 0x2, 0x2, 0x2, 0x487, 0x485, 0x3, 0x2, 0x2, 0x2, 0x487, 0x488, + 0x3, 0x2, 0x2, 0x2, 0x488, 0x69, 0x3, 0x2, 0x2, 0x2, 0x489, 0x487, 0x3, + 0x2, 0x2, 0x2, 0x48a, 0x490, 0x5, 0x6c, 0x37, 0x2, 0x48b, 0x48c, 0x7, + 0xd0, 0x2, 0x2, 0x48c, 0x48d, 0x5, 0x68, 0x35, 0x2, 0x48d, 0x48e, 0x7, + 0xda, 0x2, 0x2, 0x48e, 0x490, 0x3, 0x2, 0x2, 0x2, 0x48f, 0x48a, 0x3, + 0x2, 0x2, 0x2, 0x48f, 0x48b, 0x3, 0x2, 0x2, 0x2, 0x490, 0x6b, 0x3, 0x2, + 0x2, 0x2, 0x491, 0x493, 0x5, 0x6e, 0x38, 0x2, 0x492, 0x491, 0x3, 0x2, + 0x2, 0x2, 0x492, 0x493, 0x3, 0x2, 0x2, 0x2, 0x493, 0x494, 0x3, 0x2, + 0x2, 0x2, 0x494, 0x496, 0x7, 0x8d, 0x2, 0x2, 0x495, 0x497, 0x7, 0x31, + 0x2, 0x2, 0x496, 0x495, 0x3, 0x2, 0x2, 0x2, 0x496, 0x497, 0x3, 0x2, + 0x2, 0x2, 0x497, 0x499, 0x3, 0x2, 0x2, 0x2, 0x498, 0x49a, 0x5, 0x70, + 0x39, 0x2, 0x499, 0x498, 0x3, 0x2, 0x2, 0x2, 0x499, 0x49a, 0x3, 0x2, + 0x2, 0x2, 0x49a, 0x49b, 0x3, 0x2, 0x2, 0x2, 0x49b, 0x49d, 0x5, 0xac, + 0x57, 0x2, 0x49c, 0x49e, 0x5, 0x72, 0x3a, 0x2, 0x49d, 0x49c, 0x3, 0x2, + 0x2, 0x2, 0x49d, 0x49e, 0x3, 0x2, 0x2, 0x2, 0x49e, 0x4a0, 0x3, 0x2, + 0x2, 0x2, 0x49f, 0x4a1, 0x5, 0x74, 0x3b, 0x2, 0x4a0, 0x49f, 0x3, 0x2, + 0x2, 0x2, 0x4a0, 0x4a1, 0x3, 0x2, 0x2, 0x2, 0x4a1, 0x4a3, 0x3, 0x2, + 0x2, 0x2, 0x4a2, 0x4a4, 0x5, 0x76, 0x3c, 0x2, 0x4a3, 0x4a2, 0x3, 0x2, + 0x2, 0x2, 0x4a3, 0x4a4, 0x3, 0x2, 0x2, 0x2, 0x4a4, 0x4a6, 0x3, 0x2, + 0x2, 0x2, 0x4a5, 0x4a7, 0x5, 0x78, 0x3d, 0x2, 0x4a6, 0x4a5, 0x3, 0x2, + 0x2, 0x2, 0x4a6, 0x4a7, 0x3, 0x2, 0x2, 0x2, 0x4a7, 0x4a9, 0x3, 0x2, + 0x2, 0x2, 0x4a8, 0x4aa, 0x5, 0x7a, 0x3e, 0x2, 0x4a9, 0x4a8, 0x3, 0x2, + 0x2, 0x2, 0x4a9, 0x4aa, 0x3, 0x2, 0x2, 0x2, 0x4aa, 0x4ad, 0x3, 0x2, + 0x2, 0x2, 0x4ab, 0x4ac, 0x7, 0xb6, 0x2, 0x2, 0x4ac, 0x4ae, 0x9, 0x8, + 0x2, 0x2, 0x4ad, 0x4ab, 0x3, 0x2, 0x2, 0x2, 0x4ad, 0x4ae, 0x3, 0x2, + 0x2, 0x2, 0x4ae, 0x4b1, 0x3, 0x2, 0x2, 0x2, 0x4af, 0x4b0, 0x7, 0xb6, + 0x2, 0x2, 0x4b0, 0x4b2, 0x7, 0xa4, 0x2, 0x2, 0x4b1, 0x4af, 0x3, 0x2, + 0x2, 0x2, 0x4b1, 0x4b2, 0x3, 0x2, 0x2, 0x2, 0x4b2, 0x4b4, 0x3, 0x2, + 0x2, 0x2, 0x4b3, 0x4b5, 0x5, 0x7c, 0x3f, 0x2, 0x4b4, 0x4b3, 0x3, 0x2, + 0x2, 0x2, 0x4b4, 0x4b5, 0x3, 0x2, 0x2, 0x2, 0x4b5, 0x4b7, 0x3, 0x2, + 0x2, 0x2, 0x4b6, 0x4b8, 0x5, 0x7e, 0x40, 0x2, 0x4b7, 0x4b6, 0x3, 0x2, + 0x2, 0x2, 0x4b7, 0x4b8, 0x3, 0x2, 0x2, 0x2, 0x4b8, 0x4ba, 0x3, 0x2, + 0x2, 0x2, 0x4b9, 0x4bb, 0x5, 0x82, 0x42, 0x2, 0x4ba, 0x4b9, 0x3, 0x2, + 0x2, 0x2, 0x4ba, 0x4bb, 0x3, 0x2, 0x2, 0x2, 0x4bb, 0x4bd, 0x3, 0x2, + 0x2, 0x2, 0x4bc, 0x4be, 0x5, 0x84, 0x43, 0x2, 0x4bd, 0x4bc, 0x3, 0x2, + 0x2, 0x2, 0x4bd, 0x4be, 0x3, 0x2, 0x2, 0x2, 0x4be, 0x4c0, 0x3, 0x2, + 0x2, 0x2, 0x4bf, 0x4c1, 0x5, 0x86, 0x44, 0x2, 0x4c0, 0x4bf, 0x3, 0x2, + 0x2, 0x2, 0x4c0, 0x4c1, 0x3, 0x2, 0x2, 0x2, 0x4c1, 0x6d, 0x3, 0x2, 0x2, + 0x2, 0x4c2, 0x4c3, 0x7, 0xb6, 0x2, 0x2, 0x4c3, 0x4c4, 0x5, 0xac, 0x57, + 0x2, 0x4c4, 0x6f, 0x3, 0x2, 0x2, 0x2, 0x4c5, 0x4c6, 0x7, 0xa3, 0x2, + 0x2, 0x4c6, 0x4c9, 0x7, 0xbd, 0x2, 0x2, 0x4c7, 0x4c8, 0x7, 0xb6, 0x2, + 0x2, 0x4c8, 0x4ca, 0x7, 0x9f, 0x2, 0x2, 0x4c9, 0x4c7, 0x3, 0x2, 0x2, + 0x2, 0x4c9, 0x4ca, 0x3, 0x2, 0x2, 0x2, 0x4ca, 0x71, 0x3, 0x2, 0x2, 0x2, + 0x4cb, 0x4cc, 0x7, 0x43, 0x2, 0x2, 0x4cc, 0x4cd, 0x5, 0x88, 0x45, 0x2, + 0x4cd, 0x73, 0x3, 0x2, 0x2, 0x2, 0x4ce, 0x4d0, 0x9, 0x9, 0x2, 0x2, 0x4cf, + 0x4ce, 0x3, 0x2, 0x2, 0x2, 0x4cf, 0x4d0, 0x3, 0x2, 0x2, 0x2, 0x4d0, + 0x4d1, 0x3, 0x2, 0x2, 0x2, 0x4d1, 0x4d2, 0x7, 0xb, 0x2, 0x2, 0x4d2, + 0x4d3, 0x7, 0x59, 0x2, 0x2, 0x4d3, 0x4d4, 0x5, 0xac, 0x57, 0x2, 0x4d4, + 0x75, 0x3, 0x2, 0x2, 0x2, 0x4d5, 0x4d6, 0x7, 0x7e, 0x2, 0x2, 0x4d6, + 0x4d7, 0x5, 0xb0, 0x59, 0x2, 0x4d7, 0x77, 0x3, 0x2, 0x2, 0x2, 0x4d8, + 0x4d9, 0x7, 0xb5, 0x2, 0x2, 0x4d9, 0x4da, 0x5, 0xb0, 0x59, 0x2, 0x4da, + 0x79, 0x3, 0x2, 0x2, 0x2, 0x4db, 0x4dc, 0x7, 0x48, 0x2, 0x2, 0x4dc, + 0x4e3, 0x7, 0x14, 0x2, 0x2, 0x4dd, 0x4de, 0x9, 0x8, 0x2, 0x2, 0x4de, + 0x4df, 0x7, 0xd0, 0x2, 0x2, 0x4df, 0x4e0, 0x5, 0xac, 0x57, 0x2, 0x4e0, + 0x4e1, 0x7, 0xda, 0x2, 0x2, 0x4e1, 0x4e4, 0x3, 0x2, 0x2, 0x2, 0x4e2, + 0x4e4, 0x5, 0xac, 0x57, 0x2, 0x4e3, 0x4dd, 0x3, 0x2, 0x2, 0x2, 0x4e3, + 0x4e2, 0x3, 0x2, 0x2, 0x2, 0x4e4, 0x7b, 0x3, 0x2, 0x2, 0x2, 0x4e5, 0x4e6, + 0x7, 0x49, 0x2, 0x2, 0x4e6, 0x4e7, 0x5, 0xb0, 0x59, 0x2, 0x4e7, 0x7d, + 0x3, 0x2, 0x2, 0x2, 0x4e8, 0x4e9, 0x7, 0x79, 0x2, 0x2, 0x4e9, 0x4ea, + 0x7, 0x14, 0x2, 0x2, 0x4ea, 0x4eb, 0x5, 0x94, 0x4b, 0x2, 0x4eb, 0x7f, + 0x3, 0x2, 0x2, 0x2, 0x4ec, 0x4ed, 0x7, 0x79, 0x2, 0x2, 0x4ed, 0x4ee, + 0x7, 0x14, 0x2, 0x2, 0x4ee, 0x4ef, 0x5, 0xac, 0x57, 0x2, 0x4ef, 0x81, + 0x3, 0x2, 0x2, 0x2, 0x4f0, 0x4f1, 0x7, 0x62, 0x2, 0x2, 0x4f1, 0x4f2, + 0x5, 0x92, 0x4a, 0x2, 0x4f2, 0x4f3, 0x7, 0x14, 0x2, 0x2, 0x4f3, 0x4f4, + 0x5, 0xac, 0x57, 0x2, 0x4f4, 0x83, 0x3, 0x2, 0x2, 0x2, 0x4f5, 0x4f6, + 0x7, 0x62, 0x2, 0x2, 0x4f6, 0x4f9, 0x5, 0x92, 0x4a, 0x2, 0x4f7, 0x4f8, + 0x7, 0xb6, 0x2, 0x2, 0x4f8, 0x4fa, 0x7, 0x9f, 0x2, 0x2, 0x4f9, 0x4f7, + 0x3, 0x2, 0x2, 0x2, 0x4f9, 0x4fa, 0x3, 0x2, 0x2, 0x2, 0x4fa, 0x85, 0x3, + 0x2, 0x2, 0x2, 0x4fb, 0x4fc, 0x7, 0x91, 0x2, 0x2, 0x4fc, 0x4fd, 0x5, + 0x9a, 0x4e, 0x2, 0x4fd, 0x87, 0x3, 0x2, 0x2, 0x2, 0x4fe, 0x4ff, 0x8, + 0x45, 0x1, 0x2, 0x4ff, 0x501, 0x5, 0xbc, 0x5f, 0x2, 0x500, 0x502, 0x7, + 0x3d, 0x2, 0x2, 0x501, 0x500, 0x3, 0x2, 0x2, 0x2, 0x501, 0x502, 0x3, + 0x2, 0x2, 0x2, 0x502, 0x504, 0x3, 0x2, 0x2, 0x2, 0x503, 0x505, 0x5, + 0x90, 0x49, 0x2, 0x504, 0x503, 0x3, 0x2, 0x2, 0x2, 0x504, 0x505, 0x3, + 0x2, 0x2, 0x2, 0x505, 0x50b, 0x3, 0x2, 0x2, 0x2, 0x506, 0x507, 0x7, + 0xd0, 0x2, 0x2, 0x507, 0x508, 0x5, 0x88, 0x45, 0x2, 0x508, 0x509, 0x7, + 0xda, 0x2, 0x2, 0x509, 0x50b, 0x3, 0x2, 0x2, 0x2, 0x50a, 0x4fe, 0x3, + 0x2, 0x2, 0x2, 0x50a, 0x506, 0x3, 0x2, 0x2, 0x2, 0x50b, 0x51d, 0x3, + 0x2, 0x2, 0x2, 0x50c, 0x50d, 0xc, 0x5, 0x2, 0x2, 0x50d, 0x50e, 0x5, + 0x8c, 0x47, 0x2, 0x50e, 0x50f, 0x5, 0x88, 0x45, 0x6, 0x50f, 0x51c, 0x3, + 0x2, 0x2, 0x2, 0x510, 0x512, 0xc, 0x6, 0x2, 0x2, 0x511, 0x513, 0x9, + 0xa, 0x2, 0x2, 0x512, 0x511, 0x3, 0x2, 0x2, 0x2, 0x512, 0x513, 0x3, + 0x2, 0x2, 0x2, 0x513, 0x515, 0x3, 0x2, 0x2, 0x2, 0x514, 0x516, 0x5, + 0x8a, 0x46, 0x2, 0x515, 0x514, 0x3, 0x2, 0x2, 0x2, 0x515, 0x516, 0x3, + 0x2, 0x2, 0x2, 0x516, 0x517, 0x3, 0x2, 0x2, 0x2, 0x517, 0x518, 0x7, + 0x59, 0x2, 0x2, 0x518, 0x519, 0x5, 0x88, 0x45, 0x2, 0x519, 0x51a, 0x5, + 0x8e, 0x48, 0x2, 0x51a, 0x51c, 0x3, 0x2, 0x2, 0x2, 0x51b, 0x50c, 0x3, + 0x2, 0x2, 0x2, 0x51b, 0x510, 0x3, 0x2, 0x2, 0x2, 0x51c, 0x51f, 0x3, + 0x2, 0x2, 0x2, 0x51d, 0x51b, 0x3, 0x2, 0x2, 0x2, 0x51d, 0x51e, 0x3, + 0x2, 0x2, 0x2, 0x51e, 0x89, 0x3, 0x2, 0x2, 0x2, 0x51f, 0x51d, 0x3, 0x2, + 0x2, 0x2, 0x520, 0x522, 0x9, 0xb, 0x2, 0x2, 0x521, 0x520, 0x3, 0x2, + 0x2, 0x2, 0x521, 0x522, 0x3, 0x2, 0x2, 0x2, 0x522, 0x523, 0x3, 0x2, + 0x2, 0x2, 0x523, 0x52a, 0x7, 0x53, 0x2, 0x2, 0x524, 0x526, 0x7, 0x53, + 0x2, 0x2, 0x525, 0x527, 0x9, 0xb, 0x2, 0x2, 0x526, 0x525, 0x3, 0x2, + 0x2, 0x2, 0x526, 0x527, 0x3, 0x2, 0x2, 0x2, 0x527, 0x52a, 0x3, 0x2, + 0x2, 0x2, 0x528, 0x52a, 0x9, 0xb, 0x2, 0x2, 0x529, 0x521, 0x3, 0x2, + 0x2, 0x2, 0x529, 0x524, 0x3, 0x2, 0x2, 0x2, 0x529, 0x528, 0x3, 0x2, + 0x2, 0x2, 0x52a, 0x54c, 0x3, 0x2, 0x2, 0x2, 0x52b, 0x52d, 0x9, 0xc, + 0x2, 0x2, 0x52c, 0x52b, 0x3, 0x2, 0x2, 0x2, 0x52c, 0x52d, 0x3, 0x2, + 0x2, 0x2, 0x52d, 0x52e, 0x3, 0x2, 0x2, 0x2, 0x52e, 0x530, 0x9, 0xd, + 0x2, 0x2, 0x52f, 0x531, 0x7, 0x7a, 0x2, 0x2, 0x530, 0x52f, 0x3, 0x2, + 0x2, 0x2, 0x530, 0x531, 0x3, 0x2, 0x2, 0x2, 0x531, 0x53a, 0x3, 0x2, + 0x2, 0x2, 0x532, 0x534, 0x9, 0xd, 0x2, 0x2, 0x533, 0x535, 0x7, 0x7a, + 0x2, 0x2, 0x534, 0x533, 0x3, 0x2, 0x2, 0x2, 0x534, 0x535, 0x3, 0x2, + 0x2, 0x2, 0x535, 0x537, 0x3, 0x2, 0x2, 0x2, 0x536, 0x538, 0x9, 0xc, + 0x2, 0x2, 0x537, 0x536, 0x3, 0x2, 0x2, 0x2, 0x537, 0x538, 0x3, 0x2, + 0x2, 0x2, 0x538, 0x53a, 0x3, 0x2, 0x2, 0x2, 0x539, 0x52c, 0x3, 0x2, + 0x2, 0x2, 0x539, 0x532, 0x3, 0x2, 0x2, 0x2, 0x53a, 0x54c, 0x3, 0x2, + 0x2, 0x2, 0x53b, 0x53d, 0x9, 0xe, 0x2, 0x2, 0x53c, 0x53b, 0x3, 0x2, + 0x2, 0x2, 0x53c, 0x53d, 0x3, 0x2, 0x2, 0x2, 0x53d, 0x53e, 0x3, 0x2, + 0x2, 0x2, 0x53e, 0x540, 0x7, 0x44, 0x2, 0x2, 0x53f, 0x541, 0x7, 0x7a, + 0x2, 0x2, 0x540, 0x53f, 0x3, 0x2, 0x2, 0x2, 0x540, 0x541, 0x3, 0x2, + 0x2, 0x2, 0x541, 0x54a, 0x3, 0x2, 0x2, 0x2, 0x542, 0x544, 0x7, 0x44, + 0x2, 0x2, 0x543, 0x545, 0x7, 0x7a, 0x2, 0x2, 0x544, 0x543, 0x3, 0x2, + 0x2, 0x2, 0x544, 0x545, 0x3, 0x2, 0x2, 0x2, 0x545, 0x547, 0x3, 0x2, + 0x2, 0x2, 0x546, 0x548, 0x9, 0xe, 0x2, 0x2, 0x547, 0x546, 0x3, 0x2, + 0x2, 0x2, 0x547, 0x548, 0x3, 0x2, 0x2, 0x2, 0x548, 0x54a, 0x3, 0x2, + 0x2, 0x2, 0x549, 0x53c, 0x3, 0x2, 0x2, 0x2, 0x549, 0x542, 0x3, 0x2, + 0x2, 0x2, 0x54a, 0x54c, 0x3, 0x2, 0x2, 0x2, 0x54b, 0x529, 0x3, 0x2, + 0x2, 0x2, 0x54b, 0x539, 0x3, 0x2, 0x2, 0x2, 0x54b, 0x549, 0x3, 0x2, + 0x2, 0x2, 0x54c, 0x8b, 0x3, 0x2, 0x2, 0x2, 0x54d, 0x54f, 0x9, 0xa, 0x2, + 0x2, 0x54e, 0x54d, 0x3, 0x2, 0x2, 0x2, 0x54e, 0x54f, 0x3, 0x2, 0x2, + 0x2, 0x54f, 0x550, 0x3, 0x2, 0x2, 0x2, 0x550, 0x551, 0x7, 0x20, 0x2, + 0x2, 0x551, 0x554, 0x7, 0x59, 0x2, 0x2, 0x552, 0x554, 0x7, 0xc5, 0x2, + 0x2, 0x553, 0x54e, 0x3, 0x2, 0x2, 0x2, 0x553, 0x552, 0x3, 0x2, 0x2, + 0x2, 0x554, 0x8d, 0x3, 0x2, 0x2, 0x2, 0x555, 0x556, 0x7, 0x76, 0x2, + 0x2, 0x556, 0x55f, 0x5, 0xac, 0x57, 0x2, 0x557, 0x558, 0x7, 0xad, 0x2, + 0x2, 0x558, 0x559, 0x7, 0xd0, 0x2, 0x2, 0x559, 0x55a, 0x5, 0xac, 0x57, + 0x2, 0x55a, 0x55b, 0x7, 0xda, 0x2, 0x2, 0x55b, 0x55f, 0x3, 0x2, 0x2, + 0x2, 0x55c, 0x55d, 0x7, 0xad, 0x2, 0x2, 0x55d, 0x55f, 0x5, 0xac, 0x57, + 0x2, 0x55e, 0x555, 0x3, 0x2, 0x2, 0x2, 0x55e, 0x557, 0x3, 0x2, 0x2, + 0x2, 0x55e, 0x55c, 0x3, 0x2, 0x2, 0x2, 0x55f, 0x8f, 0x3, 0x2, 0x2, 0x2, + 0x560, 0x561, 0x7, 0x8b, 0x2, 0x2, 0x561, 0x564, 0x5, 0x98, 0x4d, 0x2, + 0x562, 0x563, 0x7, 0x75, 0x2, 0x2, 0x563, 0x565, 0x5, 0x98, 0x4d, 0x2, + 0x564, 0x562, 0x3, 0x2, 0x2, 0x2, 0x564, 0x565, 0x3, 0x2, 0x2, 0x2, + 0x565, 0x91, 0x3, 0x2, 0x2, 0x2, 0x566, 0x569, 0x5, 0xb0, 0x59, 0x2, + 0x567, 0x568, 0x9, 0xf, 0x2, 0x2, 0x568, 0x56a, 0x5, 0xb0, 0x59, 0x2, + 0x569, 0x567, 0x3, 0x2, 0x2, 0x2, 0x569, 0x56a, 0x3, 0x2, 0x2, 0x2, + 0x56a, 0x93, 0x3, 0x2, 0x2, 0x2, 0x56b, 0x570, 0x5, 0x96, 0x4c, 0x2, + 0x56c, 0x56d, 0x7, 0xc5, 0x2, 0x2, 0x56d, 0x56f, 0x5, 0x96, 0x4c, 0x2, + 0x56e, 0x56c, 0x3, 0x2, 0x2, 0x2, 0x56f, 0x572, 0x3, 0x2, 0x2, 0x2, + 0x570, 0x56e, 0x3, 0x2, 0x2, 0x2, 0x570, 0x571, 0x3, 0x2, 0x2, 0x2, + 0x571, 0x95, 0x3, 0x2, 0x2, 0x2, 0x572, 0x570, 0x3, 0x2, 0x2, 0x2, 0x573, + 0x575, 0x5, 0xb0, 0x59, 0x2, 0x574, 0x576, 0x9, 0x10, 0x2, 0x2, 0x575, + 0x574, 0x3, 0x2, 0x2, 0x2, 0x575, 0x576, 0x3, 0x2, 0x2, 0x2, 0x576, + 0x579, 0x3, 0x2, 0x2, 0x2, 0x577, 0x578, 0x7, 0x74, 0x2, 0x2, 0x578, + 0x57a, 0x9, 0x11, 0x2, 0x2, 0x579, 0x577, 0x3, 0x2, 0x2, 0x2, 0x579, + 0x57a, 0x3, 0x2, 0x2, 0x2, 0x57a, 0x57d, 0x3, 0x2, 0x2, 0x2, 0x57b, + 0x57c, 0x7, 0x1b, 0x2, 0x2, 0x57c, 0x57e, 0x7, 0xbf, 0x2, 0x2, 0x57d, + 0x57b, 0x3, 0x2, 0x2, 0x2, 0x57d, 0x57e, 0x3, 0x2, 0x2, 0x2, 0x57e, + 0x97, 0x3, 0x2, 0x2, 0x2, 0x57f, 0x582, 0x5, 0xca, 0x66, 0x2, 0x580, + 0x581, 0x7, 0xdc, 0x2, 0x2, 0x581, 0x583, 0x5, 0xca, 0x66, 0x2, 0x582, + 0x580, 0x3, 0x2, 0x2, 0x2, 0x582, 0x583, 0x3, 0x2, 0x2, 0x2, 0x583, + 0x99, 0x3, 0x2, 0x2, 0x2, 0x584, 0x589, 0x5, 0x9c, 0x4f, 0x2, 0x585, + 0x586, 0x7, 0xc5, 0x2, 0x2, 0x586, 0x588, 0x5, 0x9c, 0x4f, 0x2, 0x587, + 0x585, 0x3, 0x2, 0x2, 0x2, 0x588, 0x58b, 0x3, 0x2, 0x2, 0x2, 0x589, + 0x587, 0x3, 0x2, 0x2, 0x2, 0x589, 0x58a, 0x3, 0x2, 0x2, 0x2, 0x58a, + 0x9b, 0x3, 0x2, 0x2, 0x2, 0x58b, 0x589, 0x3, 0x2, 0x2, 0x2, 0x58c, 0x58d, + 0x5, 0xd6, 0x6c, 0x2, 0x58d, 0x58e, 0x7, 0xca, 0x2, 0x2, 0x58e, 0x58f, + 0x5, 0xcc, 0x67, 0x2, 0x58f, 0x9d, 0x3, 0x2, 0x2, 0x2, 0x590, 0x591, + 0x7, 0x90, 0x2, 0x2, 0x591, 0x592, 0x5, 0x9a, 0x4e, 0x2, 0x592, 0x9f, + 0x3, 0x2, 0x2, 0x2, 0x593, 0x594, 0x7, 0x92, 0x2, 0x2, 0x594, 0x595, + 0x7, 0x1f, 0x2, 0x2, 0x595, 0x596, 0x7, 0x22, 0x2, 0x2, 0x596, 0x5be, + 0x5, 0xc6, 0x64, 0x2, 0x597, 0x598, 0x7, 0x92, 0x2, 0x2, 0x598, 0x599, + 0x7, 0x1f, 0x2, 0x2, 0x599, 0x59a, 0x7, 0x2f, 0x2, 0x2, 0x59a, 0x5be, + 0x5, 0xc0, 0x61, 0x2, 0x59b, 0x59c, 0x7, 0x92, 0x2, 0x2, 0x59c, 0x59e, + 0x7, 0x1f, 0x2, 0x2, 0x59d, 0x59f, 0x7, 0x9c, 0x2, 0x2, 0x59e, 0x59d, + 0x3, 0x2, 0x2, 0x2, 0x59e, 0x59f, 0x3, 0x2, 0x2, 0x2, 0x59f, 0x5a1, + 0x3, 0x2, 0x2, 0x2, 0x5a0, 0x5a2, 0x7, 0x9a, 0x2, 0x2, 0x5a1, 0x5a0, + 0x3, 0x2, 0x2, 0x2, 0x5a1, 0x5a2, 0x3, 0x2, 0x2, 0x2, 0x5a2, 0x5a3, + 0x3, 0x2, 0x2, 0x2, 0x5a3, 0x5be, 0x5, 0xc0, 0x61, 0x2, 0x5a4, 0x5a5, + 0x7, 0x92, 0x2, 0x2, 0x5a5, 0x5be, 0x7, 0x23, 0x2, 0x2, 0x5a6, 0x5a7, + 0x7, 0x92, 0x2, 0x2, 0x5a7, 0x5aa, 0x7, 0x2e, 0x2, 0x2, 0x5a8, 0x5a9, + 0x7, 0x43, 0x2, 0x2, 0x5a9, 0x5ab, 0x5, 0xc6, 0x64, 0x2, 0x5aa, 0x5a8, + 0x3, 0x2, 0x2, 0x2, 0x5aa, 0x5ab, 0x3, 0x2, 0x2, 0x2, 0x5ab, 0x5be, + 0x3, 0x2, 0x2, 0x2, 0x5ac, 0x5ae, 0x7, 0x92, 0x2, 0x2, 0x5ad, 0x5af, + 0x7, 0x9c, 0x2, 0x2, 0x5ae, 0x5ad, 0x3, 0x2, 0x2, 0x2, 0x5ae, 0x5af, + 0x3, 0x2, 0x2, 0x2, 0x5af, 0x5b0, 0x3, 0x2, 0x2, 0x2, 0x5b0, 0x5b3, + 0x7, 0x9b, 0x2, 0x2, 0x5b1, 0x5b2, 0x9, 0x12, 0x2, 0x2, 0x5b2, 0x5b4, + 0x5, 0xc6, 0x64, 0x2, 0x5b3, 0x5b1, 0x3, 0x2, 0x2, 0x2, 0x5b3, 0x5b4, + 0x3, 0x2, 0x2, 0x2, 0x5b4, 0x5b8, 0x3, 0x2, 0x2, 0x2, 0x5b5, 0x5b6, + 0x7, 0x61, 0x2, 0x2, 0x5b6, 0x5b9, 0x7, 0xbf, 0x2, 0x2, 0x5b7, 0x5b9, + 0x5, 0x78, 0x3d, 0x2, 0x5b8, 0x5b5, 0x3, 0x2, 0x2, 0x2, 0x5b8, 0x5b7, + 0x3, 0x2, 0x2, 0x2, 0x5b8, 0x5b9, 0x3, 0x2, 0x2, 0x2, 0x5b9, 0x5bb, + 0x3, 0x2, 0x2, 0x2, 0x5ba, 0x5bc, 0x5, 0x84, 0x43, 0x2, 0x5bb, 0x5ba, + 0x3, 0x2, 0x2, 0x2, 0x5bb, 0x5bc, 0x3, 0x2, 0x2, 0x2, 0x5bc, 0x5be, + 0x3, 0x2, 0x2, 0x2, 0x5bd, 0x593, 0x3, 0x2, 0x2, 0x2, 0x5bd, 0x597, + 0x3, 0x2, 0x2, 0x2, 0x5bd, 0x59b, 0x3, 0x2, 0x2, 0x2, 0x5bd, 0x5a4, + 0x3, 0x2, 0x2, 0x2, 0x5bd, 0x5a6, 0x3, 0x2, 0x2, 0x2, 0x5bd, 0x5ac, + 0x3, 0x2, 0x2, 0x2, 0x5be, 0xa1, 0x3, 0x2, 0x2, 0x2, 0x5bf, 0x5c0, 0x7, + 0x99, 0x2, 0x2, 0x5c0, 0x5c1, 0x7, 0x3f, 0x2, 0x2, 0x5c1, 0x5c2, 0x7, + 0x32, 0x2, 0x2, 0x5c2, 0x5e2, 0x5, 0xc0, 0x61, 0x2, 0x5c3, 0x5c4, 0x7, + 0x99, 0x2, 0x2, 0x5c4, 0x5c5, 0x7, 0x3f, 0x2, 0x2, 0x5c5, 0x5e2, 0x7, + 0x65, 0x2, 0x2, 0x5c6, 0x5c7, 0x7, 0x99, 0x2, 0x2, 0x5c7, 0x5c8, 0x7, + 0x83, 0x2, 0x2, 0x5c8, 0x5e2, 0x7, 0x2e, 0x2, 0x2, 0x5c9, 0x5ca, 0x7, + 0x99, 0x2, 0x2, 0x5ca, 0x5cb, 0x7, 0x83, 0x2, 0x2, 0x5cb, 0x5cc, 0x7, + 0x2f, 0x2, 0x2, 0x5cc, 0x5e2, 0x5, 0xc0, 0x61, 0x2, 0x5cd, 0x5ce, 0x7, + 0x99, 0x2, 0x2, 0x5ce, 0x5d6, 0x9, 0x13, 0x2, 0x2, 0x5cf, 0x5d0, 0x7, + 0x32, 0x2, 0x2, 0x5d0, 0x5d7, 0x7, 0x8f, 0x2, 0x2, 0x5d1, 0x5d7, 0x7, + 0x3c, 0x2, 0x2, 0x5d2, 0x5d4, 0x7, 0xa8, 0x2, 0x2, 0x5d3, 0x5d2, 0x3, + 0x2, 0x2, 0x2, 0x5d3, 0x5d4, 0x3, 0x2, 0x2, 0x2, 0x5d4, 0x5d5, 0x3, + 0x2, 0x2, 0x2, 0x5d5, 0x5d7, 0x7, 0x69, 0x2, 0x2, 0x5d6, 0x5cf, 0x3, + 0x2, 0x2, 0x2, 0x5d6, 0x5d1, 0x3, 0x2, 0x2, 0x2, 0x5d6, 0x5d3, 0x3, + 0x2, 0x2, 0x2, 0x5d7, 0x5d8, 0x3, 0x2, 0x2, 0x2, 0x5d8, 0x5e2, 0x5, + 0xc0, 0x61, 0x2, 0x5d9, 0x5da, 0x7, 0x99, 0x2, 0x2, 0x5da, 0x5db, 0x9, + 0x13, 0x2, 0x2, 0x5db, 0x5dc, 0x7, 0x88, 0x2, 0x2, 0x5dc, 0x5e2, 0x7, + 0x8f, 0x2, 0x2, 0x5dd, 0x5de, 0x7, 0x99, 0x2, 0x2, 0x5de, 0x5df, 0x7, + 0x97, 0x2, 0x2, 0x5df, 0x5e0, 0x7, 0x87, 0x2, 0x2, 0x5e0, 0x5e2, 0x5, + 0xc0, 0x61, 0x2, 0x5e1, 0x5bf, 0x3, 0x2, 0x2, 0x2, 0x5e1, 0x5c3, 0x3, + 0x2, 0x2, 0x2, 0x5e1, 0x5c6, 0x3, 0x2, 0x2, 0x2, 0x5e1, 0x5c9, 0x3, + 0x2, 0x2, 0x2, 0x5e1, 0x5cd, 0x3, 0x2, 0x2, 0x2, 0x5e1, 0x5d9, 0x3, + 0x2, 0x2, 0x2, 0x5e1, 0x5dd, 0x3, 0x2, 0x2, 0x2, 0x5e2, 0xa3, 0x3, 0x2, + 0x2, 0x2, 0x5e3, 0x5e5, 0x7, 0xa7, 0x2, 0x2, 0x5e4, 0x5e6, 0x7, 0x9c, + 0x2, 0x2, 0x5e5, 0x5e4, 0x3, 0x2, 0x2, 0x2, 0x5e5, 0x5e6, 0x3, 0x2, + 0x2, 0x2, 0x5e6, 0x5e8, 0x3, 0x2, 0x2, 0x2, 0x5e7, 0x5e9, 0x7, 0x9a, + 0x2, 0x2, 0x5e8, 0x5e7, 0x3, 0x2, 0x2, 0x2, 0x5e8, 0x5e9, 0x3, 0x2, + 0x2, 0x2, 0x5e9, 0x5ec, 0x3, 0x2, 0x2, 0x2, 0x5ea, 0x5eb, 0x7, 0x4d, + 0x2, 0x2, 0x5eb, 0x5ed, 0x7, 0x38, 0x2, 0x2, 0x5ec, 0x5ea, 0x3, 0x2, + 0x2, 0x2, 0x5ec, 0x5ed, 0x3, 0x2, 0x2, 0x2, 0x5ed, 0x5ee, 0x3, 0x2, + 0x2, 0x2, 0x5ee, 0x5f0, 0x5, 0xc0, 0x61, 0x2, 0x5ef, 0x5f1, 0x5, 0x2c, + 0x17, 0x2, 0x5f0, 0x5ef, 0x3, 0x2, 0x2, 0x2, 0x5f0, 0x5f1, 0x3, 0x2, + 0x2, 0x2, 0x5f1, 0xa5, 0x3, 0x2, 0x2, 0x2, 0x5f2, 0x5f3, 0x7, 0xac, + 0x2, 0x2, 0x5f3, 0x5f4, 0x5, 0xc6, 0x64, 0x2, 0x5f4, 0xa7, 0x3, 0x2, + 0x2, 0x2, 0x5f5, 0x5f6, 0x7, 0xb2, 0x2, 0x2, 0x5f6, 0x5f8, 0x5, 0xc0, + 0x61, 0x2, 0x5f7, 0x5f9, 0x7, 0x37, 0x2, 0x2, 0x5f8, 0x5f7, 0x3, 0x2, + 0x2, 0x2, 0x5f8, 0x5f9, 0x3, 0x2, 0x2, 0x2, 0x5f9, 0x5fc, 0x3, 0x2, + 0x2, 0x2, 0x5fa, 0x5fb, 0x7, 0x62, 0x2, 0x2, 0x5fb, 0x5fd, 0x7, 0xbd, + 0x2, 0x2, 0x5fc, 0x5fa, 0x3, 0x2, 0x2, 0x2, 0x5fc, 0x5fd, 0x3, 0x2, + 0x2, 0x2, 0x5fd, 0xa9, 0x3, 0x2, 0x2, 0x2, 0x5fe, 0x62e, 0x5, 0xd6, + 0x6c, 0x2, 0x5ff, 0x600, 0x5, 0xd6, 0x6c, 0x2, 0x600, 0x601, 0x7, 0xd0, + 0x2, 0x2, 0x601, 0x602, 0x5, 0xd6, 0x6c, 0x2, 0x602, 0x609, 0x5, 0xaa, + 0x56, 0x2, 0x603, 0x604, 0x7, 0xc5, 0x2, 0x2, 0x604, 0x605, 0x5, 0xd6, + 0x6c, 0x2, 0x605, 0x606, 0x5, 0xaa, 0x56, 0x2, 0x606, 0x608, 0x3, 0x2, + 0x2, 0x2, 0x607, 0x603, 0x3, 0x2, 0x2, 0x2, 0x608, 0x60b, 0x3, 0x2, + 0x2, 0x2, 0x609, 0x607, 0x3, 0x2, 0x2, 0x2, 0x609, 0x60a, 0x3, 0x2, + 0x2, 0x2, 0x60a, 0x60c, 0x3, 0x2, 0x2, 0x2, 0x60b, 0x609, 0x3, 0x2, + 0x2, 0x2, 0x60c, 0x60d, 0x7, 0xda, 0x2, 0x2, 0x60d, 0x62e, 0x3, 0x2, + 0x2, 0x2, 0x60e, 0x60f, 0x5, 0xd6, 0x6c, 0x2, 0x60f, 0x610, 0x7, 0xd0, + 0x2, 0x2, 0x610, 0x615, 0x5, 0xda, 0x6e, 0x2, 0x611, 0x612, 0x7, 0xc5, + 0x2, 0x2, 0x612, 0x614, 0x5, 0xda, 0x6e, 0x2, 0x613, 0x611, 0x3, 0x2, + 0x2, 0x2, 0x614, 0x617, 0x3, 0x2, 0x2, 0x2, 0x615, 0x613, 0x3, 0x2, + 0x2, 0x2, 0x615, 0x616, 0x3, 0x2, 0x2, 0x2, 0x616, 0x618, 0x3, 0x2, + 0x2, 0x2, 0x617, 0x615, 0x3, 0x2, 0x2, 0x2, 0x618, 0x619, 0x7, 0xda, + 0x2, 0x2, 0x619, 0x62e, 0x3, 0x2, 0x2, 0x2, 0x61a, 0x61b, 0x5, 0xd6, + 0x6c, 0x2, 0x61b, 0x61c, 0x7, 0xd0, 0x2, 0x2, 0x61c, 0x621, 0x5, 0xaa, + 0x56, 0x2, 0x61d, 0x61e, 0x7, 0xc5, 0x2, 0x2, 0x61e, 0x620, 0x5, 0xaa, + 0x56, 0x2, 0x61f, 0x61d, 0x3, 0x2, 0x2, 0x2, 0x620, 0x623, 0x3, 0x2, + 0x2, 0x2, 0x621, 0x61f, 0x3, 0x2, 0x2, 0x2, 0x621, 0x622, 0x3, 0x2, + 0x2, 0x2, 0x622, 0x624, 0x3, 0x2, 0x2, 0x2, 0x623, 0x621, 0x3, 0x2, + 0x2, 0x2, 0x624, 0x625, 0x7, 0xda, 0x2, 0x2, 0x625, 0x62e, 0x3, 0x2, + 0x2, 0x2, 0x626, 0x627, 0x5, 0xd6, 0x6c, 0x2, 0x627, 0x629, 0x7, 0xd0, + 0x2, 0x2, 0x628, 0x62a, 0x5, 0xac, 0x57, 0x2, 0x629, 0x628, 0x3, 0x2, + 0x2, 0x2, 0x629, 0x62a, 0x3, 0x2, 0x2, 0x2, 0x62a, 0x62b, 0x3, 0x2, + 0x2, 0x2, 0x62b, 0x62c, 0x7, 0xda, 0x2, 0x2, 0x62c, 0x62e, 0x3, 0x2, + 0x2, 0x2, 0x62d, 0x5fe, 0x3, 0x2, 0x2, 0x2, 0x62d, 0x5ff, 0x3, 0x2, + 0x2, 0x2, 0x62d, 0x60e, 0x3, 0x2, 0x2, 0x2, 0x62d, 0x61a, 0x3, 0x2, + 0x2, 0x2, 0x62d, 0x626, 0x3, 0x2, 0x2, 0x2, 0x62e, 0xab, 0x3, 0x2, 0x2, + 0x2, 0x62f, 0x634, 0x5, 0xae, 0x58, 0x2, 0x630, 0x631, 0x7, 0xc5, 0x2, + 0x2, 0x631, 0x633, 0x5, 0xae, 0x58, 0x2, 0x632, 0x630, 0x3, 0x2, 0x2, + 0x2, 0x633, 0x636, 0x3, 0x2, 0x2, 0x2, 0x634, 0x632, 0x3, 0x2, 0x2, + 0x2, 0x634, 0x635, 0x3, 0x2, 0x2, 0x2, 0x635, 0xad, 0x3, 0x2, 0x2, 0x2, + 0x636, 0x634, 0x3, 0x2, 0x2, 0x2, 0x637, 0x638, 0x5, 0xc0, 0x61, 0x2, + 0x638, 0x639, 0x7, 0xc8, 0x2, 0x2, 0x639, 0x63b, 0x3, 0x2, 0x2, 0x2, + 0x63a, 0x637, 0x3, 0x2, 0x2, 0x2, 0x63a, 0x63b, 0x3, 0x2, 0x2, 0x2, + 0x63b, 0x63c, 0x3, 0x2, 0x2, 0x2, 0x63c, 0x643, 0x7, 0xc1, 0x2, 0x2, + 0x63d, 0x63e, 0x7, 0xd0, 0x2, 0x2, 0x63e, 0x63f, 0x5, 0x68, 0x35, 0x2, + 0x63f, 0x640, 0x7, 0xda, 0x2, 0x2, 0x640, 0x643, 0x3, 0x2, 0x2, 0x2, + 0x641, 0x643, 0x5, 0xb0, 0x59, 0x2, 0x642, 0x63a, 0x3, 0x2, 0x2, 0x2, + 0x642, 0x63d, 0x3, 0x2, 0x2, 0x2, 0x642, 0x641, 0x3, 0x2, 0x2, 0x2, + 0x643, 0xaf, 0x3, 0x2, 0x2, 0x2, 0x644, 0x645, 0x8, 0x59, 0x1, 0x2, + 0x645, 0x647, 0x7, 0x15, 0x2, 0x2, 0x646, 0x648, 0x5, 0xb0, 0x59, 0x2, + 0x647, 0x646, 0x3, 0x2, 0x2, 0x2, 0x647, 0x648, 0x3, 0x2, 0x2, 0x2, + 0x648, 0x64e, 0x3, 0x2, 0x2, 0x2, 0x649, 0x64a, 0x7, 0xb4, 0x2, 0x2, + 0x64a, 0x64b, 0x5, 0xb0, 0x59, 0x2, 0x64b, 0x64c, 0x7, 0x9e, 0x2, 0x2, + 0x64c, 0x64d, 0x5, 0xb0, 0x59, 0x2, 0x64d, 0x64f, 0x3, 0x2, 0x2, 0x2, + 0x64e, 0x649, 0x3, 0x2, 0x2, 0x2, 0x64f, 0x650, 0x3, 0x2, 0x2, 0x2, + 0x650, 0x64e, 0x3, 0x2, 0x2, 0x2, 0x650, 0x651, 0x3, 0x2, 0x2, 0x2, + 0x651, 0x654, 0x3, 0x2, 0x2, 0x2, 0x652, 0x653, 0x7, 0x34, 0x2, 0x2, + 0x653, 0x655, 0x5, 0xb0, 0x59, 0x2, 0x654, 0x652, 0x3, 0x2, 0x2, 0x2, + 0x654, 0x655, 0x3, 0x2, 0x2, 0x2, 0x655, 0x656, 0x3, 0x2, 0x2, 0x2, + 0x656, 0x657, 0x7, 0x35, 0x2, 0x2, 0x657, 0x6b0, 0x3, 0x2, 0x2, 0x2, + 0x658, 0x659, 0x7, 0x16, 0x2, 0x2, 0x659, 0x65a, 0x7, 0xd0, 0x2, 0x2, + 0x65a, 0x65b, 0x5, 0xb0, 0x59, 0x2, 0x65b, 0x65c, 0x7, 0xc, 0x2, 0x2, + 0x65c, 0x65d, 0x5, 0xaa, 0x56, 0x2, 0x65d, 0x65e, 0x7, 0xda, 0x2, 0x2, + 0x65e, 0x6b0, 0x3, 0x2, 0x2, 0x2, 0x65f, 0x660, 0x7, 0x24, 0x2, 0x2, + 0x660, 0x6b0, 0x7, 0xbf, 0x2, 0x2, 0x661, 0x662, 0x7, 0x3b, 0x2, 0x2, + 0x662, 0x663, 0x7, 0xd0, 0x2, 0x2, 0x663, 0x664, 0x5, 0xce, 0x68, 0x2, + 0x664, 0x665, 0x7, 0x43, 0x2, 0x2, 0x665, 0x666, 0x5, 0xb0, 0x59, 0x2, + 0x666, 0x667, 0x7, 0xda, 0x2, 0x2, 0x667, 0x6b0, 0x3, 0x2, 0x2, 0x2, + 0x668, 0x669, 0x7, 0x55, 0x2, 0x2, 0x669, 0x66a, 0x5, 0xb0, 0x59, 0x2, + 0x66a, 0x66b, 0x5, 0xce, 0x68, 0x2, 0x66b, 0x6b0, 0x3, 0x2, 0x2, 0x2, + 0x66c, 0x66d, 0x7, 0x96, 0x2, 0x2, 0x66d, 0x66e, 0x7, 0xd0, 0x2, 0x2, + 0x66e, 0x66f, 0x5, 0xb0, 0x59, 0x2, 0x66f, 0x670, 0x7, 0x43, 0x2, 0x2, + 0x670, 0x673, 0x5, 0xb0, 0x59, 0x2, 0x671, 0x672, 0x7, 0x40, 0x2, 0x2, + 0x672, 0x674, 0x5, 0xb0, 0x59, 0x2, 0x673, 0x671, 0x3, 0x2, 0x2, 0x2, + 0x673, 0x674, 0x3, 0x2, 0x2, 0x2, 0x674, 0x675, 0x3, 0x2, 0x2, 0x2, + 0x675, 0x676, 0x7, 0xda, 0x2, 0x2, 0x676, 0x6b0, 0x3, 0x2, 0x2, 0x2, + 0x677, 0x678, 0x7, 0xa1, 0x2, 0x2, 0x678, 0x6b0, 0x7, 0xbf, 0x2, 0x2, + 0x679, 0x67a, 0x7, 0xa6, 0x2, 0x2, 0x67a, 0x67b, 0x7, 0xd0, 0x2, 0x2, + 0x67b, 0x67c, 0x9, 0x14, 0x2, 0x2, 0x67c, 0x67d, 0x7, 0xbf, 0x2, 0x2, + 0x67d, 0x67e, 0x7, 0x43, 0x2, 0x2, 0x67e, 0x67f, 0x5, 0xb0, 0x59, 0x2, + 0x67f, 0x680, 0x7, 0xda, 0x2, 0x2, 0x680, 0x6b0, 0x3, 0x2, 0x2, 0x2, + 0x681, 0x687, 0x5, 0xd6, 0x6c, 0x2, 0x682, 0x684, 0x7, 0xd0, 0x2, 0x2, + 0x683, 0x685, 0x5, 0xac, 0x57, 0x2, 0x684, 0x683, 0x3, 0x2, 0x2, 0x2, + 0x684, 0x685, 0x3, 0x2, 0x2, 0x2, 0x685, 0x686, 0x3, 0x2, 0x2, 0x2, + 0x686, 0x688, 0x7, 0xda, 0x2, 0x2, 0x687, 0x682, 0x3, 0x2, 0x2, 0x2, + 0x687, 0x688, 0x3, 0x2, 0x2, 0x2, 0x688, 0x689, 0x3, 0x2, 0x2, 0x2, + 0x689, 0x68b, 0x7, 0xd0, 0x2, 0x2, 0x68a, 0x68c, 0x7, 0x31, 0x2, 0x2, + 0x68b, 0x68a, 0x3, 0x2, 0x2, 0x2, 0x68b, 0x68c, 0x3, 0x2, 0x2, 0x2, + 0x68c, 0x68e, 0x3, 0x2, 0x2, 0x2, 0x68d, 0x68f, 0x5, 0xb2, 0x5a, 0x2, + 0x68e, 0x68d, 0x3, 0x2, 0x2, 0x2, 0x68e, 0x68f, 0x3, 0x2, 0x2, 0x2, + 0x68f, 0x690, 0x3, 0x2, 0x2, 0x2, 0x690, 0x691, 0x7, 0xda, 0x2, 0x2, + 0x691, 0x6b0, 0x3, 0x2, 0x2, 0x2, 0x692, 0x6b0, 0x5, 0xcc, 0x67, 0x2, + 0x693, 0x694, 0x7, 0xc7, 0x2, 0x2, 0x694, 0x6b0, 0x5, 0xb0, 0x59, 0x13, + 0x695, 0x696, 0x7, 0x72, 0x2, 0x2, 0x696, 0x6b0, 0x5, 0xb0, 0x59, 0xe, + 0x697, 0x698, 0x5, 0xc0, 0x61, 0x2, 0x698, 0x699, 0x7, 0xc8, 0x2, 0x2, + 0x699, 0x69b, 0x3, 0x2, 0x2, 0x2, 0x69a, 0x697, 0x3, 0x2, 0x2, 0x2, + 0x69a, 0x69b, 0x3, 0x2, 0x2, 0x2, 0x69b, 0x69c, 0x3, 0x2, 0x2, 0x2, + 0x69c, 0x6b0, 0x7, 0xc1, 0x2, 0x2, 0x69d, 0x69e, 0x7, 0xd0, 0x2, 0x2, + 0x69e, 0x69f, 0x5, 0x68, 0x35, 0x2, 0x69f, 0x6a0, 0x7, 0xda, 0x2, 0x2, + 0x6a0, 0x6b0, 0x3, 0x2, 0x2, 0x2, 0x6a1, 0x6a2, 0x7, 0xd0, 0x2, 0x2, + 0x6a2, 0x6a3, 0x5, 0xb0, 0x59, 0x2, 0x6a3, 0x6a4, 0x7, 0xda, 0x2, 0x2, + 0x6a4, 0x6b0, 0x3, 0x2, 0x2, 0x2, 0x6a5, 0x6a6, 0x7, 0xd0, 0x2, 0x2, + 0x6a6, 0x6a7, 0x5, 0xac, 0x57, 0x2, 0x6a7, 0x6a8, 0x7, 0xda, 0x2, 0x2, + 0x6a8, 0x6b0, 0x3, 0x2, 0x2, 0x2, 0x6a9, 0x6ab, 0x7, 0xce, 0x2, 0x2, + 0x6aa, 0x6ac, 0x5, 0xac, 0x57, 0x2, 0x6ab, 0x6aa, 0x3, 0x2, 0x2, 0x2, + 0x6ab, 0x6ac, 0x3, 0x2, 0x2, 0x2, 0x6ac, 0x6ad, 0x3, 0x2, 0x2, 0x2, + 0x6ad, 0x6b0, 0x7, 0xd9, 0x2, 0x2, 0x6ae, 0x6b0, 0x5, 0xb8, 0x5d, 0x2, + 0x6af, 0x644, 0x3, 0x2, 0x2, 0x2, 0x6af, 0x658, 0x3, 0x2, 0x2, 0x2, + 0x6af, 0x65f, 0x3, 0x2, 0x2, 0x2, 0x6af, 0x661, 0x3, 0x2, 0x2, 0x2, + 0x6af, 0x668, 0x3, 0x2, 0x2, 0x2, 0x6af, 0x66c, 0x3, 0x2, 0x2, 0x2, + 0x6af, 0x677, 0x3, 0x2, 0x2, 0x2, 0x6af, 0x679, 0x3, 0x2, 0x2, 0x2, + 0x6af, 0x681, 0x3, 0x2, 0x2, 0x2, 0x6af, 0x692, 0x3, 0x2, 0x2, 0x2, + 0x6af, 0x693, 0x3, 0x2, 0x2, 0x2, 0x6af, 0x695, 0x3, 0x2, 0x2, 0x2, + 0x6af, 0x69a, 0x3, 0x2, 0x2, 0x2, 0x6af, 0x69d, 0x3, 0x2, 0x2, 0x2, + 0x6af, 0x6a1, 0x3, 0x2, 0x2, 0x2, 0x6af, 0x6a5, 0x3, 0x2, 0x2, 0x2, + 0x6af, 0x6a9, 0x3, 0x2, 0x2, 0x2, 0x6af, 0x6ae, 0x3, 0x2, 0x2, 0x2, + 0x6b0, 0x6f8, 0x3, 0x2, 0x2, 0x2, 0x6b1, 0x6b2, 0xc, 0x12, 0x2, 0x2, + 0x6b2, 0x6b3, 0x9, 0x15, 0x2, 0x2, 0x6b3, 0x6f7, 0x5, 0xb0, 0x59, 0x13, + 0x6b4, 0x6b5, 0xc, 0x11, 0x2, 0x2, 0x6b5, 0x6b6, 0x9, 0x16, 0x2, 0x2, + 0x6b6, 0x6f7, 0x5, 0xb0, 0x59, 0x12, 0x6b7, 0x6ca, 0xc, 0x10, 0x2, 0x2, + 0x6b8, 0x6cb, 0x7, 0xc9, 0x2, 0x2, 0x6b9, 0x6cb, 0x7, 0xca, 0x2, 0x2, + 0x6ba, 0x6cb, 0x7, 0xd2, 0x2, 0x2, 0x6bb, 0x6cb, 0x7, 0xcf, 0x2, 0x2, + 0x6bc, 0x6cb, 0x7, 0xcb, 0x2, 0x2, 0x6bd, 0x6cb, 0x7, 0xd1, 0x2, 0x2, + 0x6be, 0x6cb, 0x7, 0xcc, 0x2, 0x2, 0x6bf, 0x6c1, 0x7, 0x46, 0x2, 0x2, + 0x6c0, 0x6bf, 0x3, 0x2, 0x2, 0x2, 0x6c0, 0x6c1, 0x3, 0x2, 0x2, 0x2, + 0x6c1, 0x6c3, 0x3, 0x2, 0x2, 0x2, 0x6c2, 0x6c4, 0x7, 0x72, 0x2, 0x2, + 0x6c3, 0x6c2, 0x3, 0x2, 0x2, 0x2, 0x6c3, 0x6c4, 0x3, 0x2, 0x2, 0x2, + 0x6c4, 0x6c5, 0x3, 0x2, 0x2, 0x2, 0x6c5, 0x6cb, 0x7, 0x4f, 0x2, 0x2, + 0x6c6, 0x6c8, 0x7, 0x72, 0x2, 0x2, 0x6c7, 0x6c6, 0x3, 0x2, 0x2, 0x2, + 0x6c7, 0x6c8, 0x3, 0x2, 0x2, 0x2, 0x6c8, 0x6c9, 0x3, 0x2, 0x2, 0x2, + 0x6c9, 0x6cb, 0x9, 0x17, 0x2, 0x2, 0x6ca, 0x6b8, 0x3, 0x2, 0x2, 0x2, + 0x6ca, 0x6b9, 0x3, 0x2, 0x2, 0x2, 0x6ca, 0x6ba, 0x3, 0x2, 0x2, 0x2, + 0x6ca, 0x6bb, 0x3, 0x2, 0x2, 0x2, 0x6ca, 0x6bc, 0x3, 0x2, 0x2, 0x2, + 0x6ca, 0x6bd, 0x3, 0x2, 0x2, 0x2, 0x6ca, 0x6be, 0x3, 0x2, 0x2, 0x2, + 0x6ca, 0x6c0, 0x3, 0x2, 0x2, 0x2, 0x6ca, 0x6c7, 0x3, 0x2, 0x2, 0x2, + 0x6cb, 0x6cc, 0x3, 0x2, 0x2, 0x2, 0x6cc, 0x6f7, 0x5, 0xb0, 0x59, 0x11, + 0x6cd, 0x6ce, 0xc, 0xd, 0x2, 0x2, 0x6ce, 0x6cf, 0x7, 0x8, 0x2, 0x2, + 0x6cf, 0x6f7, 0x5, 0xb0, 0x59, 0xe, 0x6d0, 0x6d1, 0xc, 0xc, 0x2, 0x2, + 0x6d1, 0x6d2, 0x7, 0x78, 0x2, 0x2, 0x6d2, 0x6f7, 0x5, 0xb0, 0x59, 0xd, + 0x6d3, 0x6d5, 0xc, 0xb, 0x2, 0x2, 0x6d4, 0x6d6, 0x7, 0x72, 0x2, 0x2, + 0x6d5, 0x6d4, 0x3, 0x2, 0x2, 0x2, 0x6d5, 0x6d6, 0x3, 0x2, 0x2, 0x2, + 0x6d6, 0x6d7, 0x3, 0x2, 0x2, 0x2, 0x6d7, 0x6d8, 0x7, 0x12, 0x2, 0x2, + 0x6d8, 0x6d9, 0x5, 0xb0, 0x59, 0x2, 0x6d9, 0x6da, 0x7, 0x8, 0x2, 0x2, + 0x6da, 0x6db, 0x5, 0xb0, 0x59, 0xc, 0x6db, 0x6f7, 0x3, 0x2, 0x2, 0x2, + 0x6dc, 0x6dd, 0xc, 0xa, 0x2, 0x2, 0x6dd, 0x6de, 0x7, 0xd5, 0x2, 0x2, + 0x6de, 0x6df, 0x5, 0xb0, 0x59, 0x2, 0x6df, 0x6e0, 0x7, 0xc4, 0x2, 0x2, + 0x6e0, 0x6e1, 0x5, 0xb0, 0x59, 0xa, 0x6e1, 0x6f7, 0x3, 0x2, 0x2, 0x2, + 0x6e2, 0x6e3, 0xc, 0x15, 0x2, 0x2, 0x6e3, 0x6e4, 0x7, 0xce, 0x2, 0x2, + 0x6e4, 0x6e5, 0x5, 0xb0, 0x59, 0x2, 0x6e5, 0x6e6, 0x7, 0xd9, 0x2, 0x2, + 0x6e6, 0x6f7, 0x3, 0x2, 0x2, 0x2, 0x6e7, 0x6e8, 0xc, 0x14, 0x2, 0x2, + 0x6e8, 0x6e9, 0x7, 0xc8, 0x2, 0x2, 0x6e9, 0x6f7, 0x7, 0xbd, 0x2, 0x2, + 0x6ea, 0x6eb, 0xc, 0xf, 0x2, 0x2, 0x6eb, 0x6ed, 0x7, 0x57, 0x2, 0x2, + 0x6ec, 0x6ee, 0x7, 0x72, 0x2, 0x2, 0x6ed, 0x6ec, 0x3, 0x2, 0x2, 0x2, + 0x6ed, 0x6ee, 0x3, 0x2, 0x2, 0x2, 0x6ee, 0x6ef, 0x3, 0x2, 0x2, 0x2, + 0x6ef, 0x6f7, 0x7, 0x73, 0x2, 0x2, 0x6f0, 0x6f4, 0xc, 0x9, 0x2, 0x2, + 0x6f1, 0x6f5, 0x5, 0xd4, 0x6b, 0x2, 0x6f2, 0x6f3, 0x7, 0xc, 0x2, 0x2, + 0x6f3, 0x6f5, 0x5, 0xd6, 0x6c, 0x2, 0x6f4, 0x6f1, 0x3, 0x2, 0x2, 0x2, + 0x6f4, 0x6f2, 0x3, 0x2, 0x2, 0x2, 0x6f5, 0x6f7, 0x3, 0x2, 0x2, 0x2, + 0x6f6, 0x6b1, 0x3, 0x2, 0x2, 0x2, 0x6f6, 0x6b4, 0x3, 0x2, 0x2, 0x2, + 0x6f6, 0x6b7, 0x3, 0x2, 0x2, 0x2, 0x6f6, 0x6cd, 0x3, 0x2, 0x2, 0x2, + 0x6f6, 0x6d0, 0x3, 0x2, 0x2, 0x2, 0x6f6, 0x6d3, 0x3, 0x2, 0x2, 0x2, + 0x6f6, 0x6dc, 0x3, 0x2, 0x2, 0x2, 0x6f6, 0x6e2, 0x3, 0x2, 0x2, 0x2, + 0x6f6, 0x6e7, 0x3, 0x2, 0x2, 0x2, 0x6f6, 0x6ea, 0x3, 0x2, 0x2, 0x2, + 0x6f6, 0x6f0, 0x3, 0x2, 0x2, 0x2, 0x6f7, 0x6fa, 0x3, 0x2, 0x2, 0x2, + 0x6f8, 0x6f6, 0x3, 0x2, 0x2, 0x2, 0x6f8, 0x6f9, 0x3, 0x2, 0x2, 0x2, + 0x6f9, 0xb1, 0x3, 0x2, 0x2, 0x2, 0x6fa, 0x6f8, 0x3, 0x2, 0x2, 0x2, 0x6fb, + 0x700, 0x5, 0xb4, 0x5b, 0x2, 0x6fc, 0x6fd, 0x7, 0xc5, 0x2, 0x2, 0x6fd, + 0x6ff, 0x5, 0xb4, 0x5b, 0x2, 0x6fe, 0x6fc, 0x3, 0x2, 0x2, 0x2, 0x6ff, + 0x702, 0x3, 0x2, 0x2, 0x2, 0x700, 0x6fe, 0x3, 0x2, 0x2, 0x2, 0x700, + 0x701, 0x3, 0x2, 0x2, 0x2, 0x701, 0xb3, 0x3, 0x2, 0x2, 0x2, 0x702, 0x700, + 0x3, 0x2, 0x2, 0x2, 0x703, 0x706, 0x5, 0xb6, 0x5c, 0x2, 0x704, 0x706, + 0x5, 0xb0, 0x59, 0x2, 0x705, 0x703, 0x3, 0x2, 0x2, 0x2, 0x705, 0x704, + 0x3, 0x2, 0x2, 0x2, 0x706, 0xb5, 0x3, 0x2, 0x2, 0x2, 0x707, 0x708, 0x7, + 0xd0, 0x2, 0x2, 0x708, 0x70d, 0x5, 0xd6, 0x6c, 0x2, 0x709, 0x70a, 0x7, + 0xc5, 0x2, 0x2, 0x70a, 0x70c, 0x5, 0xd6, 0x6c, 0x2, 0x70b, 0x709, 0x3, + 0x2, 0x2, 0x2, 0x70c, 0x70f, 0x3, 0x2, 0x2, 0x2, 0x70d, 0x70b, 0x3, + 0x2, 0x2, 0x2, 0x70d, 0x70e, 0x3, 0x2, 0x2, 0x2, 0x70e, 0x710, 0x3, + 0x2, 0x2, 0x2, 0x70f, 0x70d, 0x3, 0x2, 0x2, 0x2, 0x710, 0x711, 0x7, + 0xda, 0x2, 0x2, 0x711, 0x71b, 0x3, 0x2, 0x2, 0x2, 0x712, 0x717, 0x5, + 0xd6, 0x6c, 0x2, 0x713, 0x714, 0x7, 0xc5, 0x2, 0x2, 0x714, 0x716, 0x5, + 0xd6, 0x6c, 0x2, 0x715, 0x713, 0x3, 0x2, 0x2, 0x2, 0x716, 0x719, 0x3, + 0x2, 0x2, 0x2, 0x717, 0x715, 0x3, 0x2, 0x2, 0x2, 0x717, 0x718, 0x3, + 0x2, 0x2, 0x2, 0x718, 0x71b, 0x3, 0x2, 0x2, 0x2, 0x719, 0x717, 0x3, + 0x2, 0x2, 0x2, 0x71a, 0x707, 0x3, 0x2, 0x2, 0x2, 0x71a, 0x712, 0x3, + 0x2, 0x2, 0x2, 0x71b, 0x71c, 0x3, 0x2, 0x2, 0x2, 0x71c, 0x71d, 0x7, + 0xc0, 0x2, 0x2, 0x71d, 0x71e, 0x5, 0xb0, 0x59, 0x2, 0x71e, 0xb7, 0x3, + 0x2, 0x2, 0x2, 0x71f, 0x720, 0x5, 0xc0, 0x61, 0x2, 0x720, 0x721, 0x7, + 0xc8, 0x2, 0x2, 0x721, 0x723, 0x3, 0x2, 0x2, 0x2, 0x722, 0x71f, 0x3, + 0x2, 0x2, 0x2, 0x722, 0x723, 0x3, 0x2, 0x2, 0x2, 0x723, 0x724, 0x3, + 0x2, 0x2, 0x2, 0x724, 0x725, 0x5, 0xba, 0x5e, 0x2, 0x725, 0xb9, 0x3, + 0x2, 0x2, 0x2, 0x726, 0x729, 0x5, 0xd6, 0x6c, 0x2, 0x727, 0x728, 0x7, + 0xc8, 0x2, 0x2, 0x728, 0x72a, 0x5, 0xd6, 0x6c, 0x2, 0x729, 0x727, 0x3, + 0x2, 0x2, 0x2, 0x729, 0x72a, 0x3, 0x2, 0x2, 0x2, 0x72a, 0xbb, 0x3, 0x2, + 0x2, 0x2, 0x72b, 0x72c, 0x8, 0x5f, 0x1, 0x2, 0x72c, 0x733, 0x5, 0xc0, + 0x61, 0x2, 0x72d, 0x733, 0x5, 0xbe, 0x60, 0x2, 0x72e, 0x72f, 0x7, 0xd0, + 0x2, 0x2, 0x72f, 0x730, 0x5, 0x68, 0x35, 0x2, 0x730, 0x731, 0x7, 0xda, + 0x2, 0x2, 0x731, 0x733, 0x3, 0x2, 0x2, 0x2, 0x732, 0x72b, 0x3, 0x2, + 0x2, 0x2, 0x732, 0x72d, 0x3, 0x2, 0x2, 0x2, 0x732, 0x72e, 0x3, 0x2, + 0x2, 0x2, 0x733, 0x73c, 0x3, 0x2, 0x2, 0x2, 0x734, 0x738, 0xc, 0x3, + 0x2, 0x2, 0x735, 0x739, 0x5, 0xd4, 0x6b, 0x2, 0x736, 0x737, 0x7, 0xc, + 0x2, 0x2, 0x737, 0x739, 0x5, 0xd6, 0x6c, 0x2, 0x738, 0x735, 0x3, 0x2, + 0x2, 0x2, 0x738, 0x736, 0x3, 0x2, 0x2, 0x2, 0x739, 0x73b, 0x3, 0x2, + 0x2, 0x2, 0x73a, 0x734, 0x3, 0x2, 0x2, 0x2, 0x73b, 0x73e, 0x3, 0x2, + 0x2, 0x2, 0x73c, 0x73a, 0x3, 0x2, 0x2, 0x2, 0x73c, 0x73d, 0x3, 0x2, + 0x2, 0x2, 0x73d, 0xbd, 0x3, 0x2, 0x2, 0x2, 0x73e, 0x73c, 0x3, 0x2, 0x2, + 0x2, 0x73f, 0x740, 0x5, 0xd6, 0x6c, 0x2, 0x740, 0x742, 0x7, 0xd0, 0x2, + 0x2, 0x741, 0x743, 0x5, 0xc2, 0x62, 0x2, 0x742, 0x741, 0x3, 0x2, 0x2, + 0x2, 0x742, 0x743, 0x3, 0x2, 0x2, 0x2, 0x743, 0x744, 0x3, 0x2, 0x2, + 0x2, 0x744, 0x745, 0x7, 0xda, 0x2, 0x2, 0x745, 0xbf, 0x3, 0x2, 0x2, + 0x2, 0x746, 0x747, 0x5, 0xc6, 0x64, 0x2, 0x747, 0x748, 0x7, 0xc8, 0x2, + 0x2, 0x748, 0x74a, 0x3, 0x2, 0x2, 0x2, 0x749, 0x746, 0x3, 0x2, 0x2, + 0x2, 0x749, 0x74a, 0x3, 0x2, 0x2, 0x2, 0x74a, 0x74b, 0x3, 0x2, 0x2, + 0x2, 0x74b, 0x74c, 0x5, 0xd6, 0x6c, 0x2, 0x74c, 0xc1, 0x3, 0x2, 0x2, + 0x2, 0x74d, 0x752, 0x5, 0xc4, 0x63, 0x2, 0x74e, 0x74f, 0x7, 0xc5, 0x2, + 0x2, 0x74f, 0x751, 0x5, 0xc4, 0x63, 0x2, 0x750, 0x74e, 0x3, 0x2, 0x2, + 0x2, 0x751, 0x754, 0x3, 0x2, 0x2, 0x2, 0x752, 0x750, 0x3, 0x2, 0x2, + 0x2, 0x752, 0x753, 0x3, 0x2, 0x2, 0x2, 0x753, 0xc3, 0x3, 0x2, 0x2, 0x2, + 0x754, 0x752, 0x3, 0x2, 0x2, 0x2, 0x755, 0x759, 0x5, 0xba, 0x5e, 0x2, + 0x756, 0x759, 0x5, 0xbe, 0x60, 0x2, 0x757, 0x759, 0x5, 0xcc, 0x67, 0x2, + 0x758, 0x755, 0x3, 0x2, 0x2, 0x2, 0x758, 0x756, 0x3, 0x2, 0x2, 0x2, + 0x758, 0x757, 0x3, 0x2, 0x2, 0x2, 0x759, 0xc5, 0x3, 0x2, 0x2, 0x2, 0x75a, + 0x75b, 0x5, 0xd6, 0x6c, 0x2, 0x75b, 0xc7, 0x3, 0x2, 0x2, 0x2, 0x75c, + 0x765, 0x7, 0xbb, 0x2, 0x2, 0x75d, 0x75e, 0x7, 0xc8, 0x2, 0x2, 0x75e, + 0x765, 0x9, 0x18, 0x2, 0x2, 0x75f, 0x760, 0x7, 0xbd, 0x2, 0x2, 0x760, + 0x762, 0x7, 0xc8, 0x2, 0x2, 0x761, 0x763, 0x9, 0x18, 0x2, 0x2, 0x762, + 0x761, 0x3, 0x2, 0x2, 0x2, 0x762, 0x763, 0x3, 0x2, 0x2, 0x2, 0x763, + 0x765, 0x3, 0x2, 0x2, 0x2, 0x764, 0x75c, 0x3, 0x2, 0x2, 0x2, 0x764, + 0x75d, 0x3, 0x2, 0x2, 0x2, 0x764, 0x75f, 0x3, 0x2, 0x2, 0x2, 0x765, + 0xc9, 0x3, 0x2, 0x2, 0x2, 0x766, 0x768, 0x9, 0x19, 0x2, 0x2, 0x767, + 0x766, 0x3, 0x2, 0x2, 0x2, 0x767, 0x768, 0x3, 0x2, 0x2, 0x2, 0x768, + 0x76f, 0x3, 0x2, 0x2, 0x2, 0x769, 0x770, 0x5, 0xc8, 0x65, 0x2, 0x76a, + 0x770, 0x7, 0xbc, 0x2, 0x2, 0x76b, 0x770, 0x7, 0xbd, 0x2, 0x2, 0x76c, + 0x770, 0x7, 0xbe, 0x2, 0x2, 0x76d, 0x770, 0x7, 0x51, 0x2, 0x2, 0x76e, + 0x770, 0x7, 0x70, 0x2, 0x2, 0x76f, 0x769, 0x3, 0x2, 0x2, 0x2, 0x76f, + 0x76a, 0x3, 0x2, 0x2, 0x2, 0x76f, 0x76b, 0x3, 0x2, 0x2, 0x2, 0x76f, + 0x76c, 0x3, 0x2, 0x2, 0x2, 0x76f, 0x76d, 0x3, 0x2, 0x2, 0x2, 0x76f, + 0x76e, 0x3, 0x2, 0x2, 0x2, 0x770, 0xcb, 0x3, 0x2, 0x2, 0x2, 0x771, 0x775, + 0x5, 0xca, 0x66, 0x2, 0x772, 0x775, 0x7, 0xbf, 0x2, 0x2, 0x773, 0x775, + 0x7, 0x73, 0x2, 0x2, 0x774, 0x771, 0x3, 0x2, 0x2, 0x2, 0x774, 0x772, + 0x3, 0x2, 0x2, 0x2, 0x774, 0x773, 0x3, 0x2, 0x2, 0x2, 0x775, 0xcd, 0x3, + 0x2, 0x2, 0x2, 0x776, 0x777, 0x9, 0x1a, 0x2, 0x2, 0x777, 0xcf, 0x3, + 0x2, 0x2, 0x2, 0x778, 0x779, 0x9, 0x1b, 0x2, 0x2, 0x779, 0xd1, 0x3, + 0x2, 0x2, 0x2, 0x77a, 0x77b, 0x9, 0x1c, 0x2, 0x2, 0x77b, 0xd3, 0x3, + 0x2, 0x2, 0x2, 0x77c, 0x77f, 0x7, 0xba, 0x2, 0x2, 0x77d, 0x77f, 0x5, + 0xd2, 0x6a, 0x2, 0x77e, 0x77c, 0x3, 0x2, 0x2, 0x2, 0x77e, 0x77d, 0x3, + 0x2, 0x2, 0x2, 0x77f, 0xd5, 0x3, 0x2, 0x2, 0x2, 0x780, 0x784, 0x7, 0xba, + 0x2, 0x2, 0x781, 0x784, 0x5, 0xce, 0x68, 0x2, 0x782, 0x784, 0x5, 0xd0, + 0x69, 0x2, 0x783, 0x780, 0x3, 0x2, 0x2, 0x2, 0x783, 0x781, 0x3, 0x2, + 0x2, 0x2, 0x783, 0x782, 0x3, 0x2, 0x2, 0x2, 0x784, 0xd7, 0x3, 0x2, 0x2, + 0x2, 0x785, 0x788, 0x5, 0xd6, 0x6c, 0x2, 0x786, 0x788, 0x7, 0x73, 0x2, + 0x2, 0x787, 0x785, 0x3, 0x2, 0x2, 0x2, 0x787, 0x786, 0x3, 0x2, 0x2, + 0x2, 0x788, 0xd9, 0x3, 0x2, 0x2, 0x2, 0x789, 0x78a, 0x7, 0xbf, 0x2, + 0x2, 0x78a, 0x78b, 0x7, 0xca, 0x2, 0x2, 0x78b, 0x78c, 0x5, 0xca, 0x66, + 0x2, 0x78c, 0xdb, 0x3, 0x2, 0x2, 0x2, 0x107, 0xe0, 0xe4, 0xe7, 0xea, 0xfe, 0x104, 0x10b, 0x113, 0x118, 0x11f, 0x124, 0x12b, 0x130, 0x136, 0x13c, 0x141, 0x147, 0x14c, 0x152, 0x157, 0x15d, 0x16b, 0x172, 0x179, 0x180, 0x186, 0x18b, 0x191, 0x196, 0x19c, 0x1a5, 0x1af, 0x1b9, 0x1cd, - 0x1d5, 0x1e4, 0x1eb, 0x1f9, 0x1ff, 0x205, 0x20c, 0x210, 0x213, 0x21a, - 0x21e, 0x221, 0x22c, 0x230, 0x233, 0x238, 0x23a, 0x23d, 0x240, 0x24a, - 0x24e, 0x251, 0x254, 0x259, 0x25b, 0x261, 0x267, 0x26b, 0x26e, 0x271, - 0x274, 0x277, 0x27c, 0x282, 0x286, 0x289, 0x28c, 0x290, 0x298, 0x2b2, - 0x2b4, 0x2b8, 0x2ce, 0x2d0, 0x2db, 0x2de, 0x2e7, 0x2f8, 0x303, 0x315, - 0x322, 0x333, 0x33c, 0x357, 0x359, 0x36e, 0x373, 0x378, 0x37b, 0x387, - 0x38c, 0x390, 0x393, 0x397, 0x39b, 0x3a0, 0x3a3, 0x3a7, 0x3a9, 0x3bf, - 0x3c7, 0x3ca, 0x3d4, 0x3d8, 0x3e0, 0x3e4, 0x3e9, 0x3ed, 0x3f1, 0x3f5, - 0x3f9, 0x3fb, 0x403, 0x407, 0x40a, 0x412, 0x417, 0x41c, 0x41f, 0x429, - 0x433, 0x437, 0x43c, 0x440, 0x446, 0x449, 0x44c, 0x44f, 0x45d, 0x461, - 0x465, 0x46a, 0x46d, 0x477, 0x47f, 0x482, 0x486, 0x489, 0x48d, 0x490, - 0x493, 0x496, 0x499, 0x49d, 0x4a1, 0x4a4, 0x4a7, 0x4aa, 0x4ad, 0x4b0, - 0x4b9, 0x4bf, 0x4d3, 0x4e9, 0x4f1, 0x4f4, 0x4fa, 0x502, 0x505, 0x50b, - 0x50d, 0x511, 0x516, 0x519, 0x51c, 0x520, 0x524, 0x527, 0x529, 0x52c, - 0x530, 0x534, 0x537, 0x539, 0x53b, 0x53e, 0x543, 0x54e, 0x554, 0x559, - 0x560, 0x565, 0x569, 0x56d, 0x572, 0x579, 0x58e, 0x591, 0x59a, 0x59e, - 0x5a3, 0x5a8, 0x5ab, 0x5ad, 0x5c3, 0x5c6, 0x5d1, 0x5d5, 0x5d8, 0x5dc, - 0x5e0, 0x5e8, 0x5ec, 0x5f9, 0x605, 0x611, 0x619, 0x61d, 0x624, 0x62a, - 0x632, 0x637, 0x640, 0x644, 0x663, 0x674, 0x677, 0x67b, 0x67e, 0x68a, - 0x69b, 0x69f, 0x6b0, 0x6b3, 0x6b7, 0x6ba, 0x6c5, 0x6dd, 0x6e4, 0x6e6, - 0x6e8, 0x6f0, 0x6f5, 0x6fd, 0x707, 0x70a, 0x712, 0x719, 0x722, 0x728, - 0x72c, 0x732, 0x739, 0x742, 0x748, 0x752, 0x754, 0x757, 0x75f, 0x764, - 0x76e, 0x773, 0x777, + 0x1d5, 0x1e4, 0x1eb, 0x1f9, 0x1ff, 0x205, 0x20c, 0x210, 0x213, 0x219, + 0x21c, 0x222, 0x226, 0x229, 0x234, 0x238, 0x23b, 0x240, 0x242, 0x245, + 0x248, 0x252, 0x256, 0x259, 0x25c, 0x261, 0x263, 0x26b, 0x26e, 0x271, + 0x277, 0x27b, 0x27e, 0x281, 0x284, 0x287, 0x28c, 0x292, 0x296, 0x299, + 0x29c, 0x2a0, 0x2a8, 0x2c2, 0x2c4, 0x2c8, 0x2de, 0x2e0, 0x2eb, 0x2ee, + 0x2f7, 0x308, 0x313, 0x325, 0x332, 0x343, 0x34c, 0x367, 0x369, 0x37e, + 0x383, 0x388, 0x38b, 0x397, 0x39c, 0x3a0, 0x3a3, 0x3a7, 0x3ab, 0x3b0, + 0x3b3, 0x3b7, 0x3b9, 0x3cf, 0x3d7, 0x3da, 0x3e4, 0x3e8, 0x3f0, 0x3f4, + 0x3f9, 0x3fd, 0x401, 0x405, 0x409, 0x40b, 0x413, 0x417, 0x41a, 0x422, + 0x427, 0x42c, 0x42f, 0x439, 0x443, 0x447, 0x44c, 0x450, 0x456, 0x459, + 0x45c, 0x45f, 0x46d, 0x471, 0x475, 0x47a, 0x47d, 0x487, 0x48f, 0x492, + 0x496, 0x499, 0x49d, 0x4a0, 0x4a3, 0x4a6, 0x4a9, 0x4ad, 0x4b1, 0x4b4, + 0x4b7, 0x4ba, 0x4bd, 0x4c0, 0x4c9, 0x4cf, 0x4e3, 0x4f9, 0x501, 0x504, + 0x50a, 0x512, 0x515, 0x51b, 0x51d, 0x521, 0x526, 0x529, 0x52c, 0x530, + 0x534, 0x537, 0x539, 0x53c, 0x540, 0x544, 0x547, 0x549, 0x54b, 0x54e, + 0x553, 0x55e, 0x564, 0x569, 0x570, 0x575, 0x579, 0x57d, 0x582, 0x589, + 0x59e, 0x5a1, 0x5aa, 0x5ae, 0x5b3, 0x5b8, 0x5bb, 0x5bd, 0x5d3, 0x5d6, + 0x5e1, 0x5e5, 0x5e8, 0x5ec, 0x5f0, 0x5f8, 0x5fc, 0x609, 0x615, 0x621, + 0x629, 0x62d, 0x634, 0x63a, 0x642, 0x647, 0x650, 0x654, 0x673, 0x684, + 0x687, 0x68b, 0x68e, 0x69a, 0x6ab, 0x6af, 0x6c0, 0x6c3, 0x6c7, 0x6ca, + 0x6d5, 0x6ed, 0x6f4, 0x6f6, 0x6f8, 0x700, 0x705, 0x70d, 0x717, 0x71a, + 0x722, 0x729, 0x732, 0x738, 0x73c, 0x742, 0x749, 0x752, 0x758, 0x762, + 0x764, 0x767, 0x76f, 0x774, 0x77e, 0x783, 0x787, }; atn::ATNDeserializer deserializer; diff --git a/src/Parsers/New/ClickHouseParser.g4 b/src/Parsers/New/ClickHouseParser.g4 index 28e5b1217ab..eb1908ed073 100644 --- a/src/Parsers/New/ClickHouseParser.g4 +++ b/src/Parsers/New/ClickHouseParser.g4 @@ -91,10 +91,10 @@ checkStmt: CHECK TABLE tableIdentifier partitionClause?; createStmt : (ATTACH | CREATE) DATABASE (IF NOT EXISTS)? databaseIdentifier clusterClause? engineExpr? # CreateDatabaseStmt - | (ATTACH | CREATE) DICTIONARY (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? dictionarySchemaClause dictionaryEngineClause # CreateDictionaryStmt + | (ATTACH | CREATE (OR REPLACE)? | REPLACE) DICTIONARY (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? dictionarySchemaClause dictionaryEngineClause # CreateDictionaryStmt | (ATTACH | CREATE) LIVE VIEW (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? (WITH TIMEOUT DECIMAL_LITERAL?)? destinationClause? tableSchemaClause? subqueryClause # CreateLiveViewStmt | (ATTACH | CREATE) MATERIALIZED VIEW (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? tableSchemaClause? (destinationClause | engineClause POPULATE?) subqueryClause # CreateMaterializedViewStmt - | (ATTACH | CREATE) TEMPORARY? TABLE (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? tableSchemaClause? engineClause? subqueryClause? # CreateTableStmt + | (ATTACH | CREATE (OR REPLACE)? | REPLACE) TEMPORARY? TABLE (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? tableSchemaClause? engineClause? subqueryClause? # CreateTableStmt | (ATTACH | CREATE) (OR REPLACE)? VIEW (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? tableSchemaClause? subqueryClause # CreateViewStmt ; diff --git a/src/Parsers/New/ClickHouseParser.h b/src/Parsers/New/ClickHouseParser.h index c860932ba1c..db9083b608b 100644 --- a/src/Parsers/New/ClickHouseParser.h +++ b/src/Parsers/New/ClickHouseParser.h @@ -793,11 +793,13 @@ public: DictionaryEngineClauseContext *dictionaryEngineClause(); antlr4::tree::TerminalNode *ATTACH(); antlr4::tree::TerminalNode *CREATE(); + antlr4::tree::TerminalNode *REPLACE(); antlr4::tree::TerminalNode *IF(); antlr4::tree::TerminalNode *NOT(); antlr4::tree::TerminalNode *EXISTS(); UuidClauseContext *uuidClause(); ClusterClauseContext *clusterClause(); + antlr4::tree::TerminalNode *OR(); virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -870,6 +872,7 @@ public: TableIdentifierContext *tableIdentifier(); antlr4::tree::TerminalNode *ATTACH(); antlr4::tree::TerminalNode *CREATE(); + antlr4::tree::TerminalNode *REPLACE(); antlr4::tree::TerminalNode *TEMPORARY(); antlr4::tree::TerminalNode *IF(); antlr4::tree::TerminalNode *NOT(); @@ -879,6 +882,7 @@ public: TableSchemaClauseContext *tableSchemaClause(); EngineClauseContext *engineClause(); SubqueryClauseContext *subqueryClause(); + antlr4::tree::TerminalNode *OR(); virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; diff --git a/tests/queries/0_stateless/01157_replace_table.sql b/tests/queries/0_stateless/01157_replace_table.sql index 337290bb4d2..a29b381a522 100644 --- a/tests/queries/0_stateless/01157_replace_table.sql +++ b/tests/queries/0_stateless/01157_replace_table.sql @@ -8,14 +8,17 @@ create table t (n UInt64, s String default 's' || toString(n)) engine=Memory; create table dist (n int) engine=Distributed(test_shard_localhost, currentDatabase(), t); create table buf (n int) engine=Buffer(currentDatabase(), dist, 1, 10, 100, 10, 100, 1000, 1000); +system stop distributed sends dist; insert into buf values (1); replace table buf (n int) engine=Distributed(test_shard_localhost, currentDatabase(), dist); replace table dist (n int) engine=Buffer(currentDatabase(), t, 1, 10, 100, 10, 100, 1000, 1000); +system stop distributed sends buf; insert into buf values (2); replace table buf (n int) engine=Buffer(currentDatabase(), dist, 1, 10, 100, 10, 100, 1000, 1000); replace table dist (n int) engine=Distributed(test_shard_localhost, currentDatabase(), t); +system stop distributed sends dist; insert into buf values (3); replace table buf (n int) engine=Null; replace table dist (n int) engine=Null; diff --git a/tests/queries/0_stateless/01185_create_or_replace_table.reference b/tests/queries/0_stateless/01185_create_or_replace_table.reference index 84df5f0f5b5..be187d9dcd4 100644 --- a/tests/queries/0_stateless/01185_create_or_replace_table.reference +++ b/tests/queries/0_stateless/01185_create_or_replace_table.reference @@ -1,8 +1,8 @@ t1 -CREATE TABLE test_01185.t1\n(\n `n` UInt64,\n `s` String\n)\nENGINE = MergeTree\nORDER BY n\nSETTINGS index_granularity = 8192 +CREATE TABLE default.t1\n(\n `n` UInt64,\n `s` String\n)\nENGINE = MergeTree\nORDER BY n\nSETTINGS index_granularity = 8192 t1 -CREATE TABLE test_01185.t1\n(\n `n` UInt64,\n `s` Nullable(String)\n)\nENGINE = MergeTree\nORDER BY n\nSETTINGS index_granularity = 8192 +CREATE TABLE default.t1\n(\n `n` UInt64,\n `s` Nullable(String)\n)\nENGINE = MergeTree\nORDER BY n\nSETTINGS index_granularity = 8192 2 \N t1 -CREATE TABLE test_01185.t1\n(\n `n` UInt64\n)\nENGINE = MergeTree\nORDER BY n\nSETTINGS index_granularity = 8192 +CREATE TABLE default.t1\n(\n `n` UInt64\n)\nENGINE = MergeTree\nORDER BY n\nSETTINGS index_granularity = 8192 3 diff --git a/tests/queries/0_stateless/01185_create_or_replace_table.sql b/tests/queries/0_stateless/01185_create_or_replace_table.sql index fe408cc7ac6..45900329b2c 100644 --- a/tests/queries/0_stateless/01185_create_or_replace_table.sql +++ b/tests/queries/0_stateless/01185_create_or_replace_table.sql @@ -1,23 +1,22 @@ -drop database if exists test_01185; -create database test_01185 engine=Atomic; +drop table if exists t1; -replace table test_01185.t1 (n UInt64, s String) engine=MergeTree order by n; -- { serverError 60 } -show tables from test_01185; -create or replace table test_01185.t1 (n UInt64, s String) engine=MergeTree order by n; -show tables from test_01185; -show create table test_01185.t1; +replace table t1 (n UInt64, s String) engine=MergeTree order by n; -- { serverError 60 } +show tables; +create or replace table t1 (n UInt64, s String) engine=MergeTree order by n; +show tables; +show create table t1; -insert into test_01185.t1 values (1, 'test'); -create or replace table test_01185.t1 (n UInt64, s Nullable(String)) engine=MergeTree order by n; -insert into test_01185.t1 values (2, null); -show tables from test_01185; -show create table test_01185.t1; -select * from test_01185.t1; +insert into t1 values (1, 'test'); +create or replace table t1 (n UInt64, s Nullable(String)) engine=MergeTree order by n; +insert into t1 values (2, null); +show tables; +show create table t1; +select * from t1; -replace table test_01185.t1 (n UInt64) engine=MergeTree order by n; -insert into test_01185.t1 values (3); -show tables from test_01185; -show create table test_01185.t1; -select * from test_01185.t1; +replace table t1 (n UInt64) engine=MergeTree order by n; +insert into t1 values (3); +show tables; +show create table t1; +select * from t1; -drop database test_01185; +drop table t1; diff --git a/tests/queries/0_stateless/01915_create_or_replace_dictionary.sql b/tests/queries/0_stateless/01915_create_or_replace_dictionary.sql index c9df6114ec9..1520dd41973 100644 --- a/tests/queries/0_stateless/01915_create_or_replace_dictionary.sql +++ b/tests/queries/0_stateless/01915_create_or_replace_dictionary.sql @@ -1,51 +1,51 @@ -DROP DATABASE IF EXISTS 01915_db; -CREATE DATABASE 01915_db ENGINE=Atomic; +DROP DATABASE IF EXISTS test_01915_db; +CREATE DATABASE test_01915_db ENGINE=Atomic; -DROP TABLE IF EXISTS 01915_db.test_source_table_1; -CREATE TABLE 01915_db.test_source_table_1 +DROP TABLE IF EXISTS test_01915_db.test_source_table_1; +CREATE TABLE test_01915_db.test_source_table_1 ( id UInt64, value String ) ENGINE=TinyLog; -INSERT INTO 01915_db.test_source_table_1 VALUES (0, 'Value0'); +INSERT INTO test_01915_db.test_source_table_1 VALUES (0, 'Value0'); -DROP DICTIONARY IF EXISTS 01915_db.test_dictionary; -CREATE OR REPLACE DICTIONARY 01915_db.test_dictionary +DROP DICTIONARY IF EXISTS test_01915_db.test_dictionary; +CREATE OR REPLACE DICTIONARY test_01915_db.test_dictionary ( id UInt64, value String ) PRIMARY KEY id LAYOUT(DIRECT()) -SOURCE(CLICKHOUSE(DB '01915_db' TABLE 'test_source_table_1')); +SOURCE(CLICKHOUSE(DB 'test_01915_db' TABLE 'test_source_table_1')); -SELECT * FROM 01915_db.test_dictionary; +SELECT * FROM test_01915_db.test_dictionary; -DROP TABLE IF EXISTS 01915_db.test_source_table_2; -CREATE TABLE 01915_db.test_source_table_2 +DROP TABLE IF EXISTS test_01915_db.test_source_table_2; +CREATE TABLE test_01915_db.test_source_table_2 ( id UInt64, value_1 String ) ENGINE=TinyLog; -INSERT INTO 01915_db.test_source_table_2 VALUES (0, 'Value1'); +INSERT INTO test_01915_db.test_source_table_2 VALUES (0, 'Value1'); -CREATE OR REPLACE DICTIONARY 01915_db.test_dictionary +CREATE OR REPLACE DICTIONARY test_01915_db.test_dictionary ( id UInt64, value_1 String ) PRIMARY KEY id LAYOUT(HASHED()) -SOURCE(CLICKHOUSE(DB '01915_db' TABLE 'test_source_table_2')) +SOURCE(CLICKHOUSE(DB 'test_01915_db' TABLE 'test_source_table_2')) LIFETIME(0); -SELECT * FROM 01915_db.test_dictionary; +SELECT * FROM test_01915_db.test_dictionary; -DROP DICTIONARY 01915_db.test_dictionary; +DROP DICTIONARY test_01915_db.test_dictionary; -DROP TABLE 01915_db.test_source_table_1; -DROP TABLE 01915_db.test_source_table_2; +DROP TABLE test_01915_db.test_source_table_1; +DROP TABLE test_01915_db.test_source_table_2; -DROP DATABASE 01915_db; +DROP DATABASE test_01915_db; diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index c9d517b0285..a56b42a4f75 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -110,6 +110,7 @@ "00738_lock_for_inner_table", "01153_attach_mv_uuid", "01157_replace_table", + "01185_create_or_replace_table", /// Sometimes cannot lock file most likely due to concurrent or adjacent tests, but we don't care how it works in Ordinary database. "rocksdb", "01914_exchange_dictionaries" /// Requires Atomic database @@ -519,7 +520,6 @@ "01924_argmax_bitmap_state", "01913_replace_dictionary", "01914_exchange_dictionaries", - "01915_create_or_replace_dictionary", "01913_names_of_tuple_literal", "01925_merge_prewhere_table" ], From c0bbe67cc9de0ad4477236d90e690130578d0a00 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 6 Jul 2021 13:26:03 +0300 Subject: [PATCH 016/198] fix --- src/Interpreters/DDLTask.cpp | 2 +- src/Interpreters/DDLTask.h | 6 +++++- src/Interpreters/InterpreterCreateQuery.cpp | 8 +++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index bf8380f5af6..1dc56d2e28a 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -362,7 +362,7 @@ ContextMutablePtr DatabaseReplicatedTask::makeQueryContext(ContextPtr from_conte query_context->getClientInfo().query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; query_context->setCurrentDatabase(database->getDatabaseName()); - auto txn = std::make_shared(zookeeper, database->zookeeper_path, is_initial_query); + auto txn = std::make_shared(zookeeper, database->zookeeper_path, is_initial_query, entry_path); query_context->initZooKeeperMetadataTransaction(txn); if (is_initial_query) diff --git a/src/Interpreters/DDLTask.h b/src/Interpreters/DDLTask.h index 00bb740f9c5..ac278e048b6 100644 --- a/src/Interpreters/DDLTask.h +++ b/src/Interpreters/DDLTask.h @@ -169,13 +169,15 @@ class ZooKeeperMetadataTransaction ZooKeeperPtr current_zookeeper; String zookeeper_path; bool is_initial_query; + String task_path; Coordination::Requests ops; public: - ZooKeeperMetadataTransaction(const ZooKeeperPtr & current_zookeeper_, const String & zookeeper_path_, bool is_initial_query_) + ZooKeeperMetadataTransaction(const ZooKeeperPtr & current_zookeeper_, const String & zookeeper_path_, bool is_initial_query_, const String & task_path_) : current_zookeeper(current_zookeeper_) , zookeeper_path(zookeeper_path_) , is_initial_query(is_initial_query_) + , task_path(task_path_) { } @@ -185,6 +187,8 @@ public: String getDatabaseZooKeeperPath() const { return zookeeper_path; } + String getTaskZooKeeperPath() const { return task_path; } + ZooKeeperPtr getZooKeeper() const { return current_zookeeper; } void addOp(Coordination::RequestPtr && op) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index a4db21ec22f..7183306c9bb 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1098,7 +1098,8 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, /// Execute drop as separate query, because [CREATE OR] REPLACE query can be considered as /// successfully executed after RENAME/EXCHANGE query. drop_context->resetZooKeeperMetadataTransaction(); - auto drop_txn = std::make_shared(txn->getZooKeeper(), txn->getDatabaseZooKeeperPath(), txn->isInitialQuery()); + auto drop_txn = std::make_shared(txn->getZooKeeper(), txn->getDatabaseZooKeeperPath(), + txn->isInitialQuery(), txn->getTaskZooKeeperPath()); drop_context->initZooKeeperMetadataTransaction(drop_txn); } return drop_context; @@ -1117,6 +1118,11 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, UInt64 name_hash = sipHash64(create.database + create.table); UInt16 random_suffix = thread_local_rng(); + if (auto txn = current_context->getZooKeeperMetadataTransaction()) + { + /// Avoid different table name on database replicas + random_suffix = sipHash64(txn->getTaskZooKeeperPath()); + } create.table = fmt::format("_tmp_replace_{}_{}", getHexUIntLowercase(name_hash), getHexUIntLowercase(random_suffix)); From cb042afa9da982c59fd5192e8b3beb0f722dc60c Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 13 Jul 2021 17:24:45 +0300 Subject: [PATCH 017/198] pushdown limit while reading in order of primary key --- src/Interpreters/InterpreterSelectQuery.cpp | 6 ++++-- .../Merges/AggregatingSortedTransform.h | 2 +- .../Merges/CollapsingSortedTransform.h | 2 +- .../FinishAggregatingInOrderTransform.h | 2 +- .../Merges/GraphiteRollupSortedTransform.h | 2 +- src/Processors/Merges/IMergingTransform.cpp | 6 +++--- src/Processors/Merges/IMergingTransform.h | 8 ++++---- .../Merges/MergingSortedTransform.cpp | 4 ++-- .../Merges/MergingSortedTransform.h | 2 +- .../Merges/ReplacingSortedTransform.h | 2 +- .../Merges/SummingSortedTransform.h | 2 +- .../Merges/VersionedCollapsingTransform.h | 2 +- .../QueryPlan/FinishSortingStep.cpp | 4 ++-- .../QueryPlan/ReadFromMergeTree.cpp | 19 ++++++++++--------- src/Processors/QueryPlan/ReadFromMergeTree.h | 4 ++-- .../MergeTreeInOrderSelectProcessor.cpp | 8 ++++---- .../MergeTreeInOrderSelectProcessor.h | 2 +- .../MergeTreeReverseSelectProcessor.cpp | 4 ++-- .../MergeTreeReverseSelectProcessor.h | 2 +- .../MergeTree/MergeTreeSelectProcessor.cpp | 4 ++-- .../MergeTree/MergeTreeSelectProcessor.h | 4 ++-- src/Storages/ReadInOrderOptimizer.cpp | 5 +++-- src/Storages/ReadInOrderOptimizer.h | 2 +- src/Storages/SelectQueryInfo.h | 5 +++-- 24 files changed, 54 insertions(+), 49 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 22314b0aab6..a337c4ab74d 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1926,11 +1926,13 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc } } + /// If we don't have filtration, we can pushdown limit to reading stage for optimizations. + UInt64 limit = (query.where() && query.prewhere()) ? getLimitForSorting(query, context) : 0; if (query_info.projection) query_info.projection->input_order_info - = query_info.projection->order_optimizer->getInputOrder(query_info.projection->desc->metadata, context); + = query_info.projection->order_optimizer->getInputOrder(query_info.projection->desc->metadata, context, limit); else - query_info.input_order_info = query_info.order_optimizer->getInputOrder(metadata_snapshot, context); + query_info.input_order_info = query_info.order_optimizer->getInputOrder(metadata_snapshot, context, limit); } StreamLocalLimits limits; diff --git a/src/Processors/Merges/AggregatingSortedTransform.h b/src/Processors/Merges/AggregatingSortedTransform.h index b4a7d4c8106..e8bf90c2b31 100644 --- a/src/Processors/Merges/AggregatingSortedTransform.h +++ b/src/Processors/Merges/AggregatingSortedTransform.h @@ -16,7 +16,7 @@ public: const Block & header, size_t num_inputs, SortDescription description_, size_t max_block_size) : IMergingTransform( - num_inputs, header, header, true, false, + num_inputs, header, header, /*have_all_inputs_=*/ true, /*has_limit_below_one_block_=*/ false, header, num_inputs, std::move(description_), diff --git a/src/Processors/Merges/CollapsingSortedTransform.h b/src/Processors/Merges/CollapsingSortedTransform.h index 059e90c1de8..87c466f31e8 100644 --- a/src/Processors/Merges/CollapsingSortedTransform.h +++ b/src/Processors/Merges/CollapsingSortedTransform.h @@ -20,7 +20,7 @@ public: WriteBuffer * out_row_sources_buf_ = nullptr, bool use_average_block_sizes = false) : IMergingTransform( - num_inputs, header, header, true, false, + num_inputs, header, header, /*have_all_inputs_=*/ true, /*has_limit_below_one_block_=*/ false, header, num_inputs, std::move(description_), diff --git a/src/Processors/Merges/FinishAggregatingInOrderTransform.h b/src/Processors/Merges/FinishAggregatingInOrderTransform.h index f3bcc9dd878..6d5e334311f 100644 --- a/src/Processors/Merges/FinishAggregatingInOrderTransform.h +++ b/src/Processors/Merges/FinishAggregatingInOrderTransform.h @@ -19,7 +19,7 @@ public: SortDescription description, size_t max_block_size) : IMergingTransform( - num_inputs, header, header, true, false, + num_inputs, header, header, /*have_all_inputs_=*/ true, /*has_limit_below_one_block_=*/ false, header, num_inputs, params, diff --git a/src/Processors/Merges/GraphiteRollupSortedTransform.h b/src/Processors/Merges/GraphiteRollupSortedTransform.h index 9432482fe16..46272f00eed 100644 --- a/src/Processors/Merges/GraphiteRollupSortedTransform.h +++ b/src/Processors/Merges/GraphiteRollupSortedTransform.h @@ -15,7 +15,7 @@ public: SortDescription description_, size_t max_block_size, Graphite::Params params_, time_t time_of_merge_) : IMergingTransform( - num_inputs, header, header, true, false, + num_inputs, header, header, /*have_all_inputs_=*/ true, /*has_limit_below_one_block_=*/ false, header, num_inputs, std::move(description_), diff --git a/src/Processors/Merges/IMergingTransform.cpp b/src/Processors/Merges/IMergingTransform.cpp index 8be21e3e291..cba78390c97 100644 --- a/src/Processors/Merges/IMergingTransform.cpp +++ b/src/Processors/Merges/IMergingTransform.cpp @@ -15,10 +15,10 @@ IMergingTransformBase::IMergingTransformBase( const Block & input_header, const Block & output_header, bool have_all_inputs_, - bool expected_one_block_) + bool has_limit_below_one_block_) : IProcessor(InputPorts(num_inputs, input_header), {output_header}) , have_all_inputs(have_all_inputs_) - , expected_one_block(expected_one_block_) + , has_limit_below_one_block(has_limit_below_one_block_) { } @@ -79,7 +79,7 @@ IProcessor::Status IMergingTransformBase::prepareInitializeInputs() /// setNotNeeded after reading first chunk, because in optimismtic case /// (e.g. with optimized 'ORDER BY primary_key LIMIT n' and small 'n') /// we won't have to read any chunks anymore; - auto chunk = input.pull(expected_one_block); + auto chunk = input.pull(has_limit_below_one_block); if (!chunk.hasRows()) { if (!input.isFinished()) diff --git a/src/Processors/Merges/IMergingTransform.h b/src/Processors/Merges/IMergingTransform.h index 695aa4eb896..8b0a44ae025 100644 --- a/src/Processors/Merges/IMergingTransform.h +++ b/src/Processors/Merges/IMergingTransform.h @@ -17,7 +17,7 @@ public: const Block & input_header, const Block & output_header, bool have_all_inputs_, - bool expected_one_block_); + bool has_limit_below_one_block_); OutputPort & getOutputPort() { return outputs.front(); } @@ -67,7 +67,7 @@ private: std::vector input_states; std::atomic have_all_inputs; bool is_initialized = false; - bool expected_one_block = false; + bool has_limit_below_one_block = false; IProcessor::Status prepareInitializeInputs(); }; @@ -83,9 +83,9 @@ public: const Block & input_header, const Block & output_header, bool have_all_inputs_, - bool expected_one_block_, + bool has_limit_below_one_block_, Args && ... args) - : IMergingTransformBase(num_inputs, input_header, output_header, have_all_inputs_, expected_one_block_) + : IMergingTransformBase(num_inputs, input_header, output_header, have_all_inputs_, has_limit_below_one_block_) , algorithm(std::forward(args) ...) { } diff --git a/src/Processors/Merges/MergingSortedTransform.cpp b/src/Processors/Merges/MergingSortedTransform.cpp index f185ec6ad8e..92fafa4242c 100644 --- a/src/Processors/Merges/MergingSortedTransform.cpp +++ b/src/Processors/Merges/MergingSortedTransform.cpp @@ -13,13 +13,13 @@ MergingSortedTransform::MergingSortedTransform( SortDescription description_, size_t max_block_size, UInt64 limit_, - bool expected_one_block_, + bool has_limit_below_one_block_, WriteBuffer * out_row_sources_buf_, bool quiet_, bool use_average_block_sizes, bool have_all_inputs_) : IMergingTransform( - num_inputs, header, header, have_all_inputs_, expected_one_block_, + num_inputs, header, header, have_all_inputs_, has_limit_below_one_block_, header, num_inputs, std::move(description_), diff --git a/src/Processors/Merges/MergingSortedTransform.h b/src/Processors/Merges/MergingSortedTransform.h index 0e616eb2c78..1fa9b1275bd 100644 --- a/src/Processors/Merges/MergingSortedTransform.h +++ b/src/Processors/Merges/MergingSortedTransform.h @@ -17,7 +17,7 @@ public: SortDescription description, size_t max_block_size, UInt64 limit_ = 0, - bool expected_one_block_ = false, + bool has_limit_below_one_block_ = false, WriteBuffer * out_row_sources_buf_ = nullptr, bool quiet_ = false, bool use_average_block_sizes = false, diff --git a/src/Processors/Merges/ReplacingSortedTransform.h b/src/Processors/Merges/ReplacingSortedTransform.h index c1df68f3a76..e760cdf0d2b 100644 --- a/src/Processors/Merges/ReplacingSortedTransform.h +++ b/src/Processors/Merges/ReplacingSortedTransform.h @@ -18,7 +18,7 @@ public: WriteBuffer * out_row_sources_buf_ = nullptr, bool use_average_block_sizes = false) : IMergingTransform( - num_inputs, header, header, true, false, + num_inputs, header, header, /*have_all_inputs_=*/ true, /*has_limit_below_one_block_=*/ false, header, num_inputs, std::move(description_), diff --git a/src/Processors/Merges/SummingSortedTransform.h b/src/Processors/Merges/SummingSortedTransform.h index 730d6604950..0287caed5aa 100644 --- a/src/Processors/Merges/SummingSortedTransform.h +++ b/src/Processors/Merges/SummingSortedTransform.h @@ -19,7 +19,7 @@ public: const Names & partition_key_columns, size_t max_block_size) : IMergingTransform( - num_inputs, header, header, true, false, + num_inputs, header, header, /*have_all_inputs_=*/ true, /*has_limit_below_one_block_=*/ false, header, num_inputs, std::move(description_), diff --git a/src/Processors/Merges/VersionedCollapsingTransform.h b/src/Processors/Merges/VersionedCollapsingTransform.h index 6251f842898..f260e20f1da 100644 --- a/src/Processors/Merges/VersionedCollapsingTransform.h +++ b/src/Processors/Merges/VersionedCollapsingTransform.h @@ -19,7 +19,7 @@ public: WriteBuffer * out_row_sources_buf_ = nullptr, bool use_average_block_sizes = false) : IMergingTransform( - num_inputs, header, header, true, false, + num_inputs, header, header, /*have_all_inputs_=*/ true, /*has_limit_below_one_block_=*/ false, header, num_inputs, std::move(description_), diff --git a/src/Processors/QueryPlan/FinishSortingStep.cpp b/src/Processors/QueryPlan/FinishSortingStep.cpp index 6f21865592b..d3eac99c8ac 100644 --- a/src/Processors/QueryPlan/FinishSortingStep.cpp +++ b/src/Processors/QueryPlan/FinishSortingStep.cpp @@ -58,14 +58,14 @@ void FinishSortingStep::transformPipeline(QueryPipeline & pipeline, const BuildQ if (pipeline.getNumStreams() > 1) { UInt64 limit_for_merging = (need_finish_sorting ? 0 : limit); - bool expected_one_block = limit_for_merging && limit_for_merging < max_block_size; + bool has_limit_below_one_block = limit_for_merging && limit_for_merging < max_block_size; auto transform = std::make_shared( pipeline.getHeader(), pipeline.getNumStreams(), prefix_description, max_block_size, limit_for_merging, - expected_one_block); + has_limit_below_one_block); pipeline.addTransform(std::move(transform)); } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 8930adf662a..3f048b6c355 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -177,30 +177,31 @@ ProcessorPtr ReadFromMergeTree::createSource( const RangesInDataPart & part, const Names & required_columns, bool use_uncompressed_cache, - bool one_range_per_task) + bool has_limit_below_one_block) { return std::make_shared( data, metadata_snapshot, part.data_part, max_block_size, preferred_block_size_bytes, preferred_max_column_in_block_size_bytes, required_columns, part.ranges, use_uncompressed_cache, - prewhere_info, actions_settings, true, reader_settings, virt_column_names, part.part_index_in_query, one_range_per_task); + prewhere_info, actions_settings, true, reader_settings, virt_column_names, part.part_index_in_query, has_limit_below_one_block); } Pipe ReadFromMergeTree::readInOrder( RangesInDataParts parts_with_range, Names required_columns, ReadType read_type, - bool use_uncompressed_cache) + bool use_uncompressed_cache, + UInt64 limit) { Pipes pipes; /// For reading in order it makes sense to read only /// one range per task to reduce number of read rows. - bool one_range_per_task = read_type != ReadType::Default; + bool has_limit_below_one_block = read_type != ReadType::Default && limit && limit < max_block_size; for (const auto & part : parts_with_range) { auto source = read_type == ReadType::InReverseOrder - ? createSource(part, required_columns, use_uncompressed_cache, one_range_per_task) - : createSource(part, required_columns, use_uncompressed_cache, one_range_per_task); + ? createSource(part, required_columns, use_uncompressed_cache, has_limit_below_one_block) + : createSource(part, required_columns, use_uncompressed_cache, has_limit_below_one_block); pipes.emplace_back(std::move(source)); } @@ -226,7 +227,7 @@ Pipe ReadFromMergeTree::read( return readFromPool(parts_with_range, required_columns, max_streams, min_marks_for_concurrent_read, use_uncompressed_cache); - auto pipe = readInOrder(parts_with_range, required_columns, read_type, use_uncompressed_cache); + auto pipe = readInOrder(parts_with_range, required_columns, read_type, use_uncompressed_cache, 0); /// Use ConcatProcessor to concat sources together. /// It is needed to read in parts order (and so in PK order) if single thread is used. @@ -460,8 +461,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( ? ReadFromMergeTree::ReadType::InOrder : ReadFromMergeTree::ReadType::InReverseOrder; - pipes.emplace_back(read(std::move(new_parts), column_names, read_type, - requested_num_streams, info.min_marks_for_concurrent_read, info.use_uncompressed_cache)); + pipes.emplace_back(readInOrder(std::move(new_parts), column_names, read_type, + info.use_uncompressed_cache, input_order_info->limit)); } if (need_preliminary_merge) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 12d5959bf29..ecba08bea29 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -109,10 +109,10 @@ private: Pipe read(RangesInDataParts parts_with_range, Names required_columns, ReadType read_type, size_t max_streams, size_t min_marks_for_concurrent_read, bool use_uncompressed_cache); Pipe readFromPool(RangesInDataParts parts_with_ranges, Names required_columns, size_t max_streams, size_t min_marks_for_concurrent_read, bool use_uncompressed_cache); - Pipe readInOrder(RangesInDataParts parts_with_range, Names required_columns, ReadType read_type, bool use_uncompressed_cache); + Pipe readInOrder(RangesInDataParts parts_with_range, Names required_columns, ReadType read_type, bool use_uncompressed_cache, UInt64 limit); template - ProcessorPtr createSource(const RangesInDataPart & part, const Names & required_columns, bool use_uncompressed_cache, bool one_range_per_task); + ProcessorPtr createSource(const RangesInDataPart & part, const Names & required_columns, bool use_uncompressed_cache, bool has_limit_below_one_block); Pipe spreadMarkRangesAmongStreams( RangesInDataParts && parts_with_ranges, diff --git a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp index 0ec60568836..7a6bc747411 100644 --- a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp @@ -23,13 +23,13 @@ MergeTreeInOrderSelectProcessor::MergeTreeInOrderSelectProcessor( bool check_columns_, const MergeTreeReaderSettings & reader_settings_, const Names & virt_column_names_, - bool one_range_per_task_, + bool has_limit_below_one_block_, bool quiet) : MergeTreeSelectProcessor{ storage_, metadata_snapshot_, owned_data_part_, max_block_size_rows_, preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, required_columns_, std::move(mark_ranges_), use_uncompressed_cache_, prewhere_info_, - std::move(actions_settings), check_columns_, reader_settings_, virt_column_names_, one_range_per_task_} + std::move(actions_settings), check_columns_, reader_settings_, virt_column_names_, has_limit_below_one_block_} { if (!quiet) LOG_DEBUG(log, "Reading {} ranges in order from part {}, approx. {} rows starting from {}", @@ -40,7 +40,6 @@ MergeTreeInOrderSelectProcessor::MergeTreeInOrderSelectProcessor( bool MergeTreeInOrderSelectProcessor::getNewTask() try { - /// Produce no more than one task if (all_mark_ranges.empty()) { finish(); @@ -52,7 +51,8 @@ try : std::make_unique(data_part, ordered_names, metadata_snapshot->getSampleBlock()); MarkRanges mark_ranges_for_task; - if (one_range_per_task) + /// If we need to read few rows, set one range per task to reduce number of read data. + if (has_limit_below_one_block) { mark_ranges_for_task = { std::move(all_mark_ranges.front()) }; all_mark_ranges.pop_front(); diff --git a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h index d1bc1bfe4c4..f0471c27c0f 100644 --- a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h @@ -26,7 +26,7 @@ public: bool check_columns, const MergeTreeReaderSettings & reader_settings, const Names & virt_column_names = {}, - bool one_range_per_task_ = false, + bool has_limit_below_one_block_ = false, bool quiet = false); String getName() const override { return "MergeTreeInOrder"; } diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index d93650e8a2c..47e7b69ae39 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -23,13 +23,13 @@ MergeTreeReverseSelectProcessor::MergeTreeReverseSelectProcessor( bool check_columns_, const MergeTreeReaderSettings & reader_settings_, const Names & virt_column_names_, - bool one_range_per_task_, + bool has_limit_below_one_block_, bool quiet) : MergeTreeSelectProcessor{ storage_, metadata_snapshot_, owned_data_part_, max_block_size_rows_, preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, required_columns_, std::move(mark_ranges_), use_uncompressed_cache_, prewhere_info_, - std::move(actions_settings), check_columns_, reader_settings_, virt_column_names_, one_range_per_task_} + std::move(actions_settings), check_columns_, reader_settings_, virt_column_names_, has_limit_below_one_block_} { if (!quiet) LOG_DEBUG(log, "Reading {} ranges in reverse order from part {}, approx. {} rows starting from {}", diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h index 33fe51c1bb6..22588727211 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h @@ -27,7 +27,7 @@ public: bool check_columns, const MergeTreeReaderSettings & reader_settings, const Names & virt_column_names = {}, - bool one_range_per_task_ = false, + bool has_limit_below_one_block_ = false, bool quiet = false); String getName() const override { return "MergeTreeReverse"; } diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 3351e64c5b0..a8544110374 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -22,7 +22,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( bool check_columns_, const MergeTreeReaderSettings & reader_settings_, const Names & virt_column_names_, - bool one_range_per_task_) + bool has_limit_below_one_block_) : MergeTreeBaseSelectProcessor{ metadata_snapshot_->getSampleBlockForColumns(required_columns_, storage_.getVirtuals(), storage_.getStorageID()), storage_, metadata_snapshot_, prewhere_info_, std::move(actions_settings), max_block_size_rows_, @@ -31,7 +31,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( required_columns{std::move(required_columns_)}, data_part{owned_data_part_}, all_mark_ranges(std::move(mark_ranges_)), - one_range_per_task(one_range_per_task_), + has_limit_below_one_block(has_limit_below_one_block_), check_columns(check_columns_), total_rows(data_part->index_granularity.getRowsCountInRanges(all_mark_ranges)) { diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index c172a8fdb14..a9a464041ff 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -31,7 +31,7 @@ public: bool check_columns, const MergeTreeReaderSettings & reader_settings, const Names & virt_column_names = {}, - bool one_range_per_task_ = false); + bool has_limit_below_one_block_ = false); ~MergeTreeSelectProcessor() override; @@ -61,7 +61,7 @@ protected: size_t part_index_in_query = 0; /// If true, every task will be created only with one range. /// It reduces amount of read data for queries with small LIMIT. - bool one_range_per_task = false; + bool has_limit_below_one_block = false; bool check_columns; size_t total_rows; diff --git a/src/Storages/ReadInOrderOptimizer.cpp b/src/Storages/ReadInOrderOptimizer.cpp index 87273330b34..912d284bfc0 100644 --- a/src/Storages/ReadInOrderOptimizer.cpp +++ b/src/Storages/ReadInOrderOptimizer.cpp @@ -37,7 +37,7 @@ ReadInOrderOptimizer::ReadInOrderOptimizer( array_join_result_to_source = syntax_result->array_join_result_to_source; } -InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder(const StorageMetadataPtr & metadata_snapshot, ContextPtr context) const +InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder(const StorageMetadataPtr & metadata_snapshot, ContextPtr context, UInt64 limit) const { Names sorting_key_columns = metadata_snapshot->getSortingKeyColumns(); if (!metadata_snapshot->hasSortingKey()) @@ -155,7 +155,8 @@ InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder(const StorageMetadataPtr & if (order_key_prefix_descr.empty()) return {}; - return std::make_shared(std::move(order_key_prefix_descr), read_direction); + + return std::make_shared(std::move(order_key_prefix_descr), read_direction, limit); } } diff --git a/src/Storages/ReadInOrderOptimizer.h b/src/Storages/ReadInOrderOptimizer.h index 0abf2923a98..2686d081855 100644 --- a/src/Storages/ReadInOrderOptimizer.h +++ b/src/Storages/ReadInOrderOptimizer.h @@ -22,7 +22,7 @@ public: const SortDescription & required_sort_description, const TreeRewriterResultPtr & syntax_result); - InputOrderInfoPtr getInputOrder(const StorageMetadataPtr & metadata_snapshot, ContextPtr context) const; + InputOrderInfoPtr getInputOrder(const StorageMetadataPtr & metadata_snapshot, ContextPtr context, UInt64 limit = 0) const; private: /// Actions for every element of order expression to analyze functions for monotonicity diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index cf2c4d72f59..3b3c0fa1258 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -83,9 +83,10 @@ struct InputOrderInfo { SortDescription order_key_prefix_descr; int direction; + UInt64 limit; - InputOrderInfo(const SortDescription & order_key_prefix_descr_, int direction_) - : order_key_prefix_descr(order_key_prefix_descr_), direction(direction_) {} + InputOrderInfo(const SortDescription & order_key_prefix_descr_, int direction_, UInt64 limit_) + : order_key_prefix_descr(order_key_prefix_descr_), direction(direction_), limit(limit_) {} bool operator ==(const InputOrderInfo & other) const { From 03c785931adbc60e647463673ee837183d0c9517 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 16 Jul 2021 17:27:38 +0300 Subject: [PATCH 018/198] fix pushdown of limit to reading stage --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 4 ++-- src/Storages/MergeTree/MergeTreeSelectProcessor.cpp | 2 ++ src/Storages/MergeTree/MergeTreeSelectProcessor.h | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 14b977f6864..0750dd6e27c 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1925,7 +1925,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc } /// If we don't have filtration, we can pushdown limit to reading stage for optimizations. - UInt64 limit = (query.where() && query.prewhere()) ? getLimitForSorting(query, context) : 0; + UInt64 limit = (query.where() || query.prewhere()) ? 0 : getLimitForSorting(query, context); if (query_info.projection) query_info.projection->input_order_info = query_info.projection->order_optimizer->getInputOrder(query_info.projection->desc->metadata, context, limit); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 612b3d5da3e..6abf37eb008 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -181,8 +181,8 @@ ProcessorPtr ReadFromMergeTree::createSource( { return std::make_shared( data, metadata_snapshot, part.data_part, max_block_size, preferred_block_size_bytes, - preferred_max_column_in_block_size_bytes, required_columns, part.ranges, use_uncompressed_cache, - prewhere_info, actions_settings, true, reader_settings, virt_column_names, has_limit_below_one_block); + preferred_max_column_in_block_size_bytes, required_columns, part.ranges, use_uncompressed_cache, prewhere_info, + actions_settings, true, reader_settings, virt_column_names, part.part_index_in_query, has_limit_below_one_block); } Pipe ReadFromMergeTree::readInOrder( diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 638e04820f3..440b47b12cf 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -22,6 +22,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( bool check_columns, const MergeTreeReaderSettings & reader_settings_, const Names & virt_column_names_, + size_t part_index_in_query_, bool has_limit_below_one_block_) : MergeTreeBaseSelectProcessor{ metadata_snapshot_->getSampleBlockForColumns(required_columns_, storage_.getVirtuals(), storage_.getStorageID()), @@ -31,6 +32,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( required_columns{std::move(required_columns_)}, data_part{owned_data_part_}, all_mark_ranges(std::move(mark_ranges_)), + part_index_in_query(part_index_in_query_), has_limit_below_one_block(has_limit_below_one_block_), total_rows(data_part->index_granularity.getRowsCountInRanges(all_mark_ranges)) { diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 83f1a0dab94..621a3bd740c 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -31,6 +31,7 @@ public: bool check_columns, const MergeTreeReaderSettings & reader_settings, const Names & virt_column_names = {}, + size_t part_index_in_query_ = 0, bool has_limit_below_one_block_ = false); ~MergeTreeSelectProcessor() override; From 048e089a589df66a1713a9ffcea78cb4f3748f42 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 16 Jul 2021 17:47:03 +0300 Subject: [PATCH 019/198] remove useless code --- src/Storages/MergeTree/MergeTreeSelectProcessor.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 621a3bd740c..0d83823de42 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -36,15 +36,10 @@ public: ~MergeTreeSelectProcessor() override; - String getName() const override = 0; - /// Closes readers and unlock part locks void finish(); protected: - - bool getNewTask() override = 0; - /// Used by Task Names required_columns; /// Names from header. Used in order to order columns in read blocks. From f61ea15d56e0624accec14f7f1e966ad3e1e3ce0 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 16 Jul 2021 20:40:00 +0300 Subject: [PATCH 020/198] fix pushdown of limit to reading stage --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 0750dd6e27c..95a8da4216c 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1925,7 +1925,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc } /// If we don't have filtration, we can pushdown limit to reading stage for optimizations. - UInt64 limit = (query.where() || query.prewhere()) ? 0 : getLimitForSorting(query, context); + UInt64 limit = (query.where() || query.prewhere() || query.groupBy()) ? 0 : getLimitForSorting(query, context); if (query_info.projection) query_info.projection->input_order_info = query_info.projection->order_optimizer->getInputOrder(query_info.projection->desc->metadata, context, limit); From d6fe824340cb4ac7956f72899a0ef498be6c352a Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Wed, 21 Jul 2021 09:20:17 +0200 Subject: [PATCH 021/198] Add Map type support in mapPopulateSeries --- src/Functions/array/mapPopulateSeries.cpp | 351 ++++++++++++------ ...01925_map_populate_series_on_map.reference | 37 ++ .../01925_map_populate_series_on_map.sql | 35 ++ 3 files changed, 319 insertions(+), 104 deletions(-) create mode 100644 tests/queries/0_stateless/01925_map_populate_series_on_map.reference create mode 100644 tests/queries/0_stateless/01925_map_populate_series_on_map.sql diff --git a/src/Functions/array/mapPopulateSeries.cpp b/src/Functions/array/mapPopulateSeries.cpp index eb2f6192346..f8bcbf8d451 100644 --- a/src/Functions/array/mapPopulateSeries.cpp +++ b/src/Functions/array/mapPopulateSeries.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -7,6 +8,7 @@ #include #include #include "Core/ColumnWithTypeAndName.h" +#include "DataTypes/DataTypeMap.h" #include "DataTypes/IDataType.h" namespace DB @@ -32,31 +34,17 @@ private: bool isVariadic() const override { return true; } bool useDefaultImplementationForConstants() const override { return true; } - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + void checkTypes(const DataTypePtr & key_type, const DataTypePtr max_key_type) const { - if (arguments.size() < 2) - throw Exception{getName() + " accepts at least two arrays for key and value", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - if (arguments.size() > 3) - throw Exception{"too many arguments in " + getName() + " call", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - const DataTypeArray * key_array_type = checkAndGetDataType(arguments[0].get()); - const DataTypeArray * val_array_type = checkAndGetDataType(arguments[1].get()); - - if (!key_array_type || !val_array_type) - throw Exception{getName() + " accepts two arrays for key and value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - DataTypePtr keys_type = key_array_type->getNestedType(); - WhichDataType which_key(keys_type); - if (!(which_key.isNativeInt() || which_key.isNativeUInt())) + WhichDataType which_key(key_type); + if (!(which_key.isInt() || which_key.isUInt())) { throw Exception( - "Keys for " + getName() + " should be of native integer type (signed or unsigned)", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + "Keys for " + getName() + " should be of integer type (signed or unsigned)", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - if (arguments.size() == 3) + if (max_key_type) { - DataTypePtr max_key_type = arguments[2]; WhichDataType which_max_key(max_key_type); if (which_max_key.isNullable()) @@ -64,53 +52,186 @@ private: "Max key argument in arguments of function " + getName() + " can not be Nullable", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (keys_type->getTypeId() != max_key_type->getTypeId()) + if (key_type->getTypeId() != max_key_type->getTypeId()) throw Exception("Max key type in " + getName() + " should be same as keys type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } + } + + DataTypePtr getReturnTypeForTuple(const DataTypes & arguments) const + { + if (arguments.size() < 2) + throw Exception(getName() + " accepts at least two arrays for key and value", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (arguments.size() > 3) + throw Exception("too many arguments in " + getName() + " call", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const DataTypeArray * key_array_type = checkAndGetDataType(arguments[0].get()); + const DataTypeArray * val_array_type = checkAndGetDataType(arguments[1].get()); + + if (!key_array_type || !val_array_type) + throw Exception(getName() + " accepts two arrays for key and value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + const auto & key_type = key_array_type->getNestedType(); + + if (arguments.size() == 3) + this->checkTypes(key_type, arguments[2]); + else + this->checkTypes(key_type, nullptr); return std::make_shared(DataTypes{arguments[0], arguments[1]}); } - template - ColumnPtr execute2(ColumnPtr key_column, ColumnPtr val_column, ColumnPtr max_key_column, const DataTypeTuple & res_type) const + DataTypePtr getReturnTypeForMap(const DataTypes & arguments) const { - MutableColumnPtr res_tuple = res_type.createColumn(); + const auto * map = assert_cast(arguments[0].get()); + if (arguments.size() == 1) + this->checkTypes(map->getKeyType(), nullptr); + else if (arguments.size() == 2) + this->checkTypes(map->getKeyType(), arguments[1]); + else + throw Exception("too many arguments in " + getName() + " call", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - auto * to_tuple = assert_cast(res_tuple.get()); - auto & to_keys_arr = assert_cast(to_tuple->getColumn(0)); - auto & to_keys_data = to_keys_arr.getData(); - auto & to_keys_offsets = to_keys_arr.getOffsets(); + return std::make_shared(map->getKeyType(), map->getValueType()); + } - auto & to_vals_arr = assert_cast(to_tuple->getColumn(1)); - auto & to_values_data = to_vals_arr.getData(); + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.empty()) + throw Exception(getName() + " accepts at least one map", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - bool max_key_is_const = false, key_is_const = false, val_is_const = false; + if (arguments[0]->getTypeId() == TypeIndex::Array) + return getReturnTypeForTuple(arguments); + else if (arguments[0]->getTypeId() == TypeIndex::Map) + return getReturnTypeForMap(arguments); + else + throw Exception(getName() + " only accepts maps", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } - const auto * keys_array = checkAndGetColumn(key_column.get()); - if (!keys_array) + // Struct holds input and output columns references, + // Both arrays and maps have similar columns to work with but extracted differently + template + struct ColumnsInOut + { + // inputs + const PaddedPODArray & in_keys_data; + const PaddedPODArray & in_vals_data; + const IColumn::Offsets & in_key_offsets; + const IColumn::Offsets & in_val_offsets; + size_t row_count; + bool key_is_const; + bool val_is_const; + + // outputs + PaddedPODArray & out_keys_data; + PaddedPODArray & out_vals_data; + + IColumn::Offsets & out_keys_offsets; + // with map argument this field will not be used + IColumn::Offsets * out_vals_offsets; + }; + + template + ColumnsInOut getInOutDataFromArrays(MutableColumnPtr & res_column, ColumnPtr * arg_columns) const + { + auto * out_tuple = assert_cast(res_column.get()); + auto & out_keys_array = assert_cast(out_tuple->getColumn(0)); + auto & out_vals_array = assert_cast(out_tuple->getColumn(1)); + + const auto * key_column = arg_columns[0].get(); + const auto * in_keys_array = checkAndGetColumn(key_column); + + bool key_is_const = false, val_is_const = false; + + if (!in_keys_array) { - const ColumnConst * const_array = checkAndGetColumnConst(key_column.get()); + const ColumnConst * const_array = checkAndGetColumnConst(key_column); if (!const_array) throw Exception("Expected array column, found " + key_column->getName(), ErrorCodes::ILLEGAL_COLUMN); - keys_array = checkAndGetColumn(const_array->getDataColumnPtr().get()); + in_keys_array = checkAndGetColumn(const_array->getDataColumnPtr().get()); key_is_const = true; } - const auto * values_array = checkAndGetColumn(val_column.get()); - if (!values_array) + const auto * val_column = arg_columns[1].get(); + const auto * in_values_array = checkAndGetColumn(val_column); + if (!in_values_array) { - const ColumnConst * const_array = checkAndGetColumnConst(val_column.get()); + const ColumnConst * const_array = checkAndGetColumnConst(val_column); if (!const_array) throw Exception("Expected array column, found " + val_column->getName(), ErrorCodes::ILLEGAL_COLUMN); - values_array = checkAndGetColumn(const_array->getDataColumnPtr().get()); + in_values_array = checkAndGetColumn(const_array->getDataColumnPtr().get()); val_is_const = true; } - if (!keys_array || !values_array) + if (!in_keys_array || !in_values_array) /* something went wrong */ - throw Exception{"Illegal columns in arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN}; + throw Exception("Illegal columns in arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); + + const auto & in_keys_data = assert_cast &>(in_keys_array->getData()).getData(); + const auto & in_values_data = assert_cast &>(in_values_array->getData()).getData(); + const auto & in_keys_offsets = in_keys_array->getOffsets(); + const auto & in_vals_offsets = in_values_array->getOffsets(); + + auto & out_keys_data = assert_cast &>(out_keys_array.getData()).getData(); + auto & out_vals_data = assert_cast &>(out_vals_array.getData()).getData(); + auto & out_keys_offsets = out_keys_array.getOffsets(); + + size_t row_count = key_is_const ? in_values_array->size() : in_keys_array->size(); + IColumn::Offsets * out_vals_offsets = &out_vals_array.getOffsets(); + + return { + in_keys_data, + in_values_data, + in_keys_offsets, + in_vals_offsets, + row_count, + key_is_const, + val_is_const, + out_keys_data, + out_vals_data, + out_keys_offsets, + out_vals_offsets}; + } + + template + ColumnsInOut getInOutDataFromMap(MutableColumnPtr & res_column, ColumnPtr * arg_columns) const + { + const auto * in_map = assert_cast(arg_columns[0].get()); + const auto & in_nested_array = in_map->getNestedColumn(); + const auto & in_nested_tuple = in_map->getNestedData(); + const auto & in_keys_data = assert_cast &>(in_nested_tuple.getColumn(0)).getData(); + const auto & in_vals_data = assert_cast &>(in_nested_tuple.getColumn(1)).getData(); + const auto & in_keys_offsets = in_nested_array.getOffsets(); + + auto * out_map = assert_cast(res_column.get()); + auto & out_nested_array = out_map->getNestedColumn(); + auto & out_nested_tuple = out_map->getNestedData(); + auto & out_keys_data = assert_cast &>(out_nested_tuple.getColumn(0)).getData(); + auto & out_vals_data = assert_cast &>(out_nested_tuple.getColumn(1)).getData(); + auto & out_keys_offsets = out_nested_array.getOffsets(); + + return { + in_keys_data, + in_vals_data, + in_keys_offsets, + in_keys_offsets, + in_nested_array.size(), + false, + false, + out_keys_data, + out_vals_data, + out_keys_offsets, + nullptr}; + } + + template + ColumnPtr execute2(ColumnPtr * arg_columns, ColumnPtr max_key_column, const DataTypePtr & res_type) const + { + MutableColumnPtr res_column = res_type->createColumn(); + bool max_key_is_const = false; + auto inout = res_column->getDataType() == TypeIndex::Tuple ? getInOutDataFromArrays(res_column, arg_columns) + : getInOutDataFromMap(res_column, arg_columns); KeyType max_key_const{0}; @@ -121,49 +242,40 @@ private: max_key_is_const = true; } - auto & keys_data = assert_cast &>(keys_array->getData()).getData(); - auto & values_data = assert_cast &>(values_array->getData()).getData(); - - // Original offsets - const IColumn::Offsets & key_offsets = keys_array->getOffsets(); - const IColumn::Offsets & val_offsets = values_array->getOffsets(); - IColumn::Offset offset{0}; - size_t row_count = key_is_const ? values_array->size() : keys_array->size(); - std::map res_map; //Iterate through two arrays and fill result values. - for (size_t row = 0; row < row_count; ++row) + for (size_t row = 0; row < inout.row_count; ++row) { - size_t key_offset = 0, val_offset = 0, array_size = key_offsets[0], val_array_size = val_offsets[0]; + size_t key_offset = 0, val_offset = 0, items_count = inout.in_key_offsets[0], val_array_size = inout.in_val_offsets[0]; res_map.clear(); - if (!key_is_const) + if (!inout.key_is_const) { - key_offset = row > 0 ? key_offsets[row - 1] : 0; - array_size = key_offsets[row] - key_offset; + key_offset = row > 0 ? inout.in_key_offsets[row - 1] : 0; + items_count = inout.in_key_offsets[row] - key_offset; } - if (!val_is_const) + if (!inout.val_is_const) { - val_offset = row > 0 ? val_offsets[row - 1] : 0; - val_array_size = val_offsets[row] - val_offset; + val_offset = row > 0 ? inout.in_val_offsets[row - 1] : 0; + val_array_size = inout.in_val_offsets[row] - val_offset; } - if (array_size != val_array_size) + if (items_count != val_array_size) throw Exception("Key and value array should have same amount of elements", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - if (array_size == 0) + if (items_count == 0) { - to_keys_offsets.push_back(offset); + inout.out_keys_offsets.push_back(offset); continue; } - for (size_t i = 0; i < array_size; ++i) + for (size_t i = 0; i < items_count; ++i) { - res_map.insert({keys_data[key_offset + i], values_data[val_offset + i]}); + res_map.insert({inout.in_keys_data[key_offset + i], inout.in_vals_data[val_offset + i]}); } auto min_key = res_map.begin()->first; @@ -184,7 +296,7 @@ private: /* no need to add anything, max key is less that first key */ if (max_key < min_key) { - to_keys_offsets.push_back(offset); + inout.out_keys_offsets.push_back(offset); continue; } } @@ -197,16 +309,16 @@ private: KeyType key; for (key = min_key;; ++key) { - to_keys_data.insert(key); + inout.out_keys_data.push_back(key); auto it = res_map.find(key); if (it != res_map.end()) { - to_values_data.insert(it->second); + inout.out_vals_data.push_back(it->second); } else { - to_values_data.insertDefault(); + inout.out_vals_data.push_back(0); } ++offset; @@ -214,80 +326,112 @@ private: break; } - to_keys_offsets.push_back(offset); + inout.out_keys_offsets.push_back(offset); } - to_vals_arr.getOffsets().insert(to_keys_offsets.begin(), to_keys_offsets.end()); - return res_tuple; + if (inout.out_vals_offsets) + inout.out_vals_offsets->insert(inout.out_keys_offsets.begin(), inout.out_keys_offsets.end()); + + return res_column; } template - ColumnPtr execute1(ColumnPtr key_column, ColumnPtr val_column, ColumnPtr max_key_column, const DataTypeTuple & res_type) const + ColumnPtr execute1(ColumnPtr * arg_columns, ColumnPtr max_key_column, const DataTypePtr & res_type, const DataTypePtr & val_type) const { - const auto & val_type = (assert_cast(res_type.getElements()[1].get()))->getNestedType(); switch (val_type->getTypeId()) { case TypeIndex::Int8: - return execute2(key_column, val_column, max_key_column, res_type); + return execute2(arg_columns, max_key_column, res_type); case TypeIndex::Int16: - return execute2(key_column, val_column, max_key_column, res_type); + return execute2(arg_columns, max_key_column, res_type); case TypeIndex::Int32: - return execute2(key_column, val_column, max_key_column, res_type); + return execute2(arg_columns, max_key_column, res_type); case TypeIndex::Int64: - return execute2(key_column, val_column, max_key_column, res_type); + return execute2(arg_columns, max_key_column, res_type); + case TypeIndex::Int128: + return execute2(arg_columns, max_key_column, res_type); + case TypeIndex::Int256: + return execute2(arg_columns, max_key_column, res_type); case TypeIndex::UInt8: - return execute2(key_column, val_column, max_key_column, res_type); + return execute2(arg_columns, max_key_column, res_type); case TypeIndex::UInt16: - return execute2(key_column, val_column, max_key_column, res_type); + return execute2(arg_columns, max_key_column, res_type); case TypeIndex::UInt32: - return execute2(key_column, val_column, max_key_column, res_type); + return execute2(arg_columns, max_key_column, res_type); case TypeIndex::UInt64: - return execute2(key_column, val_column, max_key_column, res_type); + return execute2(arg_columns, max_key_column, res_type); + case TypeIndex::UInt128: + return execute2(arg_columns, max_key_column, res_type); + case TypeIndex::UInt256: + return execute2(arg_columns, max_key_column, res_type); default: - throw Exception{"Illegal columns in arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN}; + throw Exception("Illegal columns in arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); } } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override { - auto col1 = arguments[0]; - auto col2 = arguments[1]; - - const auto * k = assert_cast(col1.type.get()); - const auto * v = assert_cast(col2.type.get()); - - /* determine output type */ - const DataTypeTuple & res_type = DataTypeTuple( - DataTypes{std::make_shared(k->getNestedType()), std::make_shared(v->getNestedType())}); - + DataTypePtr res_type, key_type, val_type; ColumnPtr max_key_column = nullptr; + ColumnPtr arg_columns[] = {arguments[0].column, nullptr}; - if (arguments.size() == 3) + if (arguments[0].type->getTypeId() == TypeIndex::Array) { - /* max key provided */ - max_key_column = arguments[2].column; + key_type = assert_cast(arguments[0].type.get())->getNestedType(); + val_type = assert_cast(arguments[1].type.get())->getNestedType(); + res_type = getReturnTypeImpl(DataTypes{arguments[0].type, arguments[1].type}); + + arg_columns[1] = arguments[1].column; + if (arguments.size() == 3) + { + /* max key provided */ + max_key_column = arguments[2].column; + } + } + else + { + assert(arguments[0].type->getTypeId() == TypeIndex::Map); + + const auto * map_type = assert_cast(arguments[0].type.get()); + res_type = getReturnTypeImpl(DataTypes{arguments[0].type}); + key_type = map_type->getKeyType(); + val_type = map_type->getValueType(); + + if (arguments.size() == 2) + { + /* max key provided */ + max_key_column = arguments[1].column; + } } - switch (k->getNestedType()->getTypeId()) + switch (key_type->getTypeId()) { case TypeIndex::Int8: - return execute1(col1.column, col2.column, max_key_column, res_type); + return execute1(arg_columns, max_key_column, res_type, val_type); case TypeIndex::Int16: - return execute1(col1.column, col2.column, max_key_column, res_type); + return execute1(arg_columns, max_key_column, res_type, val_type); case TypeIndex::Int32: - return execute1(col1.column, col2.column, max_key_column, res_type); + return execute1(arg_columns, max_key_column, res_type, val_type); case TypeIndex::Int64: - return execute1(col1.column, col2.column, max_key_column, res_type); + return execute1(arg_columns, max_key_column, res_type, val_type); + case TypeIndex::Int128: + return execute1(arg_columns, max_key_column, res_type, val_type); + case TypeIndex::Int256: + return execute1(arg_columns, max_key_column, res_type, val_type); case TypeIndex::UInt8: - return execute1(col1.column, col2.column, max_key_column, res_type); + return execute1(arg_columns, max_key_column, res_type, val_type); case TypeIndex::UInt16: - return execute1(col1.column, col2.column, max_key_column, res_type); + return execute1(arg_columns, max_key_column, res_type, val_type); case TypeIndex::UInt32: - return execute1(col1.column, col2.column, max_key_column, res_type); + return execute1(arg_columns, max_key_column, res_type, val_type); case TypeIndex::UInt64: - return execute1(col1.column, col2.column, max_key_column, res_type); + return execute1(arg_columns, max_key_column, res_type, val_type); + case TypeIndex::UInt128: + return execute1(arg_columns, max_key_column, res_type, val_type); + case TypeIndex::UInt256: + return execute1(arg_columns, max_key_column, res_type, val_type); default: - throw Exception{"Illegal columns in arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN}; + throw Exception("Illegal columns in arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); } } }; @@ -296,5 +440,4 @@ void registerFunctionMapPopulateSeries(FunctionFactory & factory) { factory.registerFunction(); } - } diff --git a/tests/queries/0_stateless/01925_map_populate_series_on_map.reference b/tests/queries/0_stateless/01925_map_populate_series_on_map.reference new file mode 100644 index 00000000000..74ed35f0638 --- /dev/null +++ b/tests/queries/0_stateless/01925_map_populate_series_on_map.reference @@ -0,0 +1,37 @@ +{1:1} +{1:1,2:2} +{1:1,2:0,3:2} +{1:1,2:0,3:0,4:2} +{1:1,2:0,3:0,4:0,5:2} +{1:1,2:0,3:0} +{1:1,2:2,3:0} +{1:1,2:0,3:2} +{1:1,2:0,3:0} +{1:1,2:0,3:0} +{1:1,2:0,3:0,4:0,5:0,6:0,7:0,8:0,9:0,10:0} +{1:1,2:2,3:0,4:0,5:0,6:0,7:0,8:0,9:0,10:0} +{1:1,2:0,3:2,4:0,5:0,6:0,7:0,8:0,9:0,10:0} +{1:1,2:0,3:0,4:2,5:0,6:0,7:0,8:0,9:0,10:0} +{1:1,2:0,3:0,4:0,5:2,6:0,7:0,8:0,9:0,10:0} +{1:1,2:0} +{1:1,2:2,3:0} +{1:1,2:0,3:2,4:0} +{1:1,2:0,3:0,4:2,5:0} +{1:1,2:0,3:0,4:0,5:2,6:0} +{1:1,2:1} Map(UInt8,UInt8) +{1:1,2:1} Map(UInt16,UInt16) +{1:1,2:1} Map(UInt32,UInt32) +{1:1,2:1} Map(UInt64,UInt64) +{1:1,2:1} Map(UInt128,UInt128) +{1:1,2:1} Map(UInt256,UInt256) +{1:1,2:1} Map(Int16,Int16) +{1:1,2:1} Map(Int16,Int16) +{1:1,2:1} Map(Int32,Int32) +{1:1,2:1} Map(Int64,Int64) +{1:1,2:1} Map(Int128,Int128) +{1:1,2:1} Map(Int256,Int256) +{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int16,Int16) +{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int16,Int16) +{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int32,Int32) +{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int64,Int64) +{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0} Map(Int64,Int64) diff --git a/tests/queries/0_stateless/01925_map_populate_series_on_map.sql b/tests/queries/0_stateless/01925_map_populate_series_on_map.sql new file mode 100644 index 00000000000..f45543a8dba --- /dev/null +++ b/tests/queries/0_stateless/01925_map_populate_series_on_map.sql @@ -0,0 +1,35 @@ +drop table if exists map_test; +set allow_experimental_map_type = 1; +create table map_test engine=TinyLog() as (select (number + 1) as n, map(1, 1, number,2) as m from numbers(1, 5)); + +select mapPopulateSeries(m) from map_test; +select mapPopulateSeries(m, toUInt64(3)) from map_test; +select mapPopulateSeries(m, toUInt64(10)) from map_test; +select mapPopulateSeries(m, 1000) from map_test; -- { serverError 43 } +select mapPopulateSeries(m, n) from map_test; + +drop table map_test; + +select mapPopulateSeries(map(toUInt8(1), toUInt8(1), 2, 1)) as res, toTypeName(res); +select mapPopulateSeries(map(toUInt16(1), toUInt16(1), 2, 1)) as res, toTypeName(res); +select mapPopulateSeries(map(toUInt32(1), toUInt32(1), 2, 1)) as res, toTypeName(res); +select mapPopulateSeries(map(toUInt64(1), toUInt64(1), 2, 1)) as res, toTypeName(res); +select mapPopulateSeries(map(toUInt128(1), toUInt128(1), 2, 1)) as res, toTypeName(res); +select mapPopulateSeries(map(toUInt256(1), toUInt256(1), 2, 1)) as res, toTypeName(res); + +select mapPopulateSeries(map(toInt8(1), toInt8(1), 2, 1)) as res, toTypeName(res); +select mapPopulateSeries(map(toInt16(1), toInt16(1), 2, 1)) as res, toTypeName(res); +select mapPopulateSeries(map(toInt32(1), toInt32(1), 2, 1)) as res, toTypeName(res); +select mapPopulateSeries(map(toInt64(1), toInt64(1), 2, 1)) as res, toTypeName(res); +select mapPopulateSeries(map(toInt128(1), toInt128(1), 2, 1)) as res, toTypeName(res); +select mapPopulateSeries(map(toInt256(1), toInt256(1), 2, 1)) as res, toTypeName(res); + +select mapPopulateSeries(map(toInt8(-10), toInt8(1), 2, 1)) as res, toTypeName(res); +select mapPopulateSeries(map(toInt16(-10), toInt16(1), 2, 1)) as res, toTypeName(res); +select mapPopulateSeries(map(toInt32(-10), toInt32(1), 2, 1)) as res, toTypeName(res); +select mapPopulateSeries(map(toInt64(-10), toInt64(1), 2, 1)) as res, toTypeName(res); +select mapPopulateSeries(map(toInt64(-10), toInt64(1), 2, 1), toInt64(-5)) as res, toTypeName(res); + +select mapPopulateSeries(); -- { serverError 42 } +select mapPopulateSeries('asdf'); -- { serverError 43 } +select mapPopulateSeries(map('1', 1, '2', 1)) as res, toTypeName(res); -- { serverError 43 } From 0f480e6b06b2d9ddfeb8a900fb0f6fbf8782757c Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Wed, 21 Jul 2021 10:08:29 +0200 Subject: [PATCH 022/198] Fix docs for map functions --- .../functions/tuple-map-functions.md | 77 +++++++++++++++---- 1 file changed, 61 insertions(+), 16 deletions(-) diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index dcfa18e04bf..908d3e1bbd3 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -103,81 +103,126 @@ Result: Query with `Map` type: ``` sql +SELECT mapAdd(map(1,1), map(1,1)); +``` + +Result: + +``` text +┌─mapAdd(map(1, 1), map(1, 1))─┐ +│ {1:2} │ +└──────────────────────────────┘ ``` ## mapSubtract {#function-mapsubtract} Collect all the keys and subtract corresponding values. -**Syntax** +**Syntax** ``` sql mapSubtract(Tuple(Array, Array), Tuple(Array, Array) [, ...]) ``` -**Arguments** +**Arguments** -Arguments are [tuples](../../sql-reference/data-types/tuple.md#tuplet1-t2) of two [arrays](../../sql-reference/data-types/array.md#data-type-array), where items in the first array represent keys, and the second array contains values for the each key. All key arrays should have same type, and all value arrays should contain items which are promote to the one type ([Int64](../../sql-reference/data-types/int-uint.md#int-ranges), [UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges) or [Float64](../../sql-reference/data-types/float.md#float32-float64)). The common promoted type is used as a type for the result array. +Arguments are [maps](../../sql-reference/data-types/map.md) or [tuples](../../sql-reference/data-types/tuple.md#tuplet1-t2) of two [arrays](../../sql-reference/data-types/array.md#data-type-array), where items in the first array represent keys, and the second array contains values for the each key. All key arrays should have same type, and all value arrays should contain items which are promote to the one type ([Int64](../../sql-reference/data-types/int-uint.md#int-ranges), [UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges) or [Float64](../../sql-reference/data-types/float.md#float32-float64)). The common promoted type is used as a type for the result array. **Returned value** -- Returns one [tuple](../../sql-reference/data-types/tuple.md#tuplet1-t2), where the first array contains the sorted keys and the second array contains values. +- Depending on the arguments returns one [map](../../sql-reference/data-types/map.md) or [tuple](../../sql-reference/data-types/tuple.md#tuplet1-t2), where the first array contains the sorted keys and the second array contains values. **Example** -Query: +Query with a tuple map: -```sql +``` sql SELECT mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])) as res, toTypeName(res) as type; ``` Result: -```text +``` text ┌─res────────────┬─type──────────────────────────────┐ │ ([1,2],[-1,0]) │ Tuple(Array(UInt8), Array(Int64)) │ └────────────────┴───────────────────────────────────┘ ``` +Query with `Map` type: + +``` sql +SELECT mapSubtract(map(1,1), map(1,1)); +``` + +Result: + +``` text +┌─mapSubtract(map(1, 1), map(1, 1))─┐ +│ {1:0} │ +└───────────────────────────────────┘ +``` + ## mapPopulateSeries {#function-mappopulateseries} Fills missing keys in the maps (key and value array pair), where keys are integers. Also, it supports specifying the max key, which is used to extend the keys array. +Arguments are [maps](../../sql-reference/data-types/map.md) or two [arrays](../../sql-reference/data-types/array.md#data-type-array), where the first array represent keys, and the second array contains values for the each key. -**Syntax** +For array arguments the number of elements in `keys` and `values` must be the same for each row. + +**Syntax** ``` sql mapPopulateSeries(keys, values[, max]) +mapPopulateSeries(map[, max]) ``` -Generates a map, where keys are a series of numbers, from minimum to maximum keys (or `max` argument if it specified) taken from `keys` array with a step size of one, and corresponding values taken from `values` array. If the value is not specified for the key, then it uses the default value in the resulting map. For repeated keys, only the first value (in order of appearing) gets associated with the key. - -The number of elements in `keys` and `values` must be the same for each row. +Generates a map (a tuple with two arrays or a value of `Map` type, depending on the arguments), where keys are a series of numbers, from minimum to maximum keys (or `max` argument if it specified) taken from the map with a step size of one, and corresponding values. If the value is not specified for the key, then it uses the default value in the resulting map. For repeated keys, only the first value (in order of appearing) gets associated with the key. **Arguments** +Mapped arrays: + - `keys` — Array of keys. [Array](../../sql-reference/data-types/array.md#data-type-array)([Int](../../sql-reference/data-types/int-uint.md#uint-ranges)). - `values` — Array of values. [Array](../../sql-reference/data-types/array.md#data-type-array)([Int](../../sql-reference/data-types/int-uint.md#uint-ranges)). +or + +- `map` — Map with integer keys. [Map](../../sql-reference/data-types/map.md). + **Returned value** -- Returns a [tuple](../../sql-reference/data-types/tuple.md#tuplet1-t2) of two [arrays](../../sql-reference/data-types/array.md#data-type-array): keys in sorted order, and values the corresponding keys. +- Depending on the arguments returns a [map](../../sql-reference/data-types/map.md) or a [tuple](../../sql-reference/data-types/tuple.md#tuplet1-t2) of two [arrays](../../sql-reference/data-types/array.md#data-type-array): keys in sorted order, and values the corresponding keys. **Example** -Query: +Query with mapped arrays: -```sql +``` sql select mapPopulateSeries([1,2,4], [11,22,44], 5) as res, toTypeName(res) as type; ``` Result: -```text +``` text ┌─res──────────────────────────┬─type──────────────────────────────┐ │ ([1,2,3,4,5],[11,22,0,44,0]) │ Tuple(Array(UInt8), Array(UInt8)) │ └──────────────────────────────┴───────────────────────────────────┘ ``` +Query with `Map` type: + +``` sql +SELECT mapPopulateSeries(map(1, 10, 5, 20), 6); +``` + +Result: + +``` text +┌─mapPopulateSeries(map(1, 10, 5, 20), 6)─┐ +│ {1:10,2:0,3:0,4:0,5:20,6:0} │ +└─────────────────────────────────────────┘ +``` + ## mapContains {#mapcontains} Determines whether the `map` contains the `key` parameter. @@ -188,7 +233,7 @@ Determines whether the `map` contains the `key` parameter. mapContains(map, key) ``` -**Parameters** +**Parameters** - `map` — Map. [Map](../../sql-reference/data-types/map.md). - `key` — Key. Type matches the type of keys of `map` parameter. From e4027755ad001dc899d683832ca00560edc10118 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Mon, 26 Jul 2021 07:09:54 +0300 Subject: [PATCH 023/198] Update order-by.md Some minor corrections. --- .../statements/select/order-by.md | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index a8fec5cfa26..ac90977b162 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -274,7 +274,7 @@ This modifier also can be combined with [LIMIT … WITH TIES modifier](../../../ `WITH FILL` modifier can be set after `ORDER BY expr` with optional `FROM expr`, `TO expr` and `STEP expr` parameters. All missed values of `expr` column will be filled sequentially and other columns will be filled as defaults. -Use following syntax for filling multiple columns add `WITH FILL` modifier with optional parameters after each field name in `ORDER BY` section. +To fill multiple columns, add `WITH FILL` modifier with optional parameters after each field name in `ORDER BY` section. ``` sql ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr] @@ -286,16 +286,16 @@ When `TO const_expr` not defined sequence of filling use maximum `expr` field va When `STEP const_numeric_expr` defined then `const_numeric_expr` interprets `as is` for numeric types as `days` for Date type and as `seconds` for DateTime type. When `STEP const_numeric_expr` omitted then sequence of filling use `1.0` for numeric type, `1 day` for Date type and `1 second` for DateTime type. -For example, the following query +Example query without `WITH FILL`: ``` sql SELECT n, source FROM ( SELECT toFloat32(number % 10) AS n, 'original' AS source FROM numbers(10) WHERE number % 3 = 1 -) ORDER BY n +) ORDER BY n; ``` -returns +Result: ``` text ┌─n─┬─source───┐ @@ -305,16 +305,16 @@ returns └───┴──────────┘ ``` -but after apply `WITH FILL` modifier +Same query after applying `WITH FILL` modifier: ``` sql SELECT n, source FROM ( SELECT toFloat32(number % 10) AS n, 'original' AS source FROM numbers(10) WHERE number % 3 = 1 -) ORDER BY n WITH FILL FROM 0 TO 5.51 STEP 0.5 +) ORDER BY n WITH FILL FROM 0 TO 5.51 STEP 0.5; ``` -returns +Result: ``` text ┌───n─┬─source───┐ @@ -334,7 +334,7 @@ returns └─────┴──────────┘ ``` -For the case when we have multiple fields `ORDER BY field2 WITH FILL, field1 WITH FILL` order of filling will follow the order of fields in `ORDER BY` clause. +For the case with multiple fields `ORDER BY field2 WITH FILL, field1 WITH FILL` order of filling will follow the order of fields in `ORDER BY` clause. Example: @@ -350,7 +350,7 @@ ORDER BY d1 WITH FILL STEP 5; ``` -returns +Result: ``` text ┌───d1───────┬───d2───────┬─source───┐ @@ -366,7 +366,7 @@ returns Field `d1` does not fill and use default value cause we do not have repeated values for `d2` value, and sequence for `d1` can’t be properly calculated. -The following query with a changed field in `ORDER BY` +The following query with a changed field in `ORDER BY`: ``` sql SELECT @@ -380,7 +380,7 @@ ORDER BY d2 WITH FILL; ``` -returns +Result: ``` text ┌───d1───────┬───d2───────┬─source───┐ @@ -400,4 +400,4 @@ returns └────────────┴────────────┴──────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/order-by/) +[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/order-by/) \ No newline at end of file From c0352044cd8e6a8d53785bb895ae24c52a68672a Mon Sep 17 00:00:00 2001 From: Mikhail Date: Mon, 26 Jul 2021 07:17:29 +0300 Subject: [PATCH 024/198] Update order-by.md Requested edit. --- docs/en/sql-reference/statements/select/order-by.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index ac90977b162..3d71e2799ea 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -280,7 +280,7 @@ To fill multiple columns, add `WITH FILL` modifier with optional parameters afte ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr] ``` -`WITH FILL` can be applied only for fields with Numeric (all kind of float, decimal, int) or Date/DateTime types. +`WITH FILL` can be applied only for fields with Numeric (all kind of float, decimal, int) or Date/DateTime types. When applied for `String` fields, missed values will be filled with empty strings. When `FROM const_expr` not defined sequence of filling use minimal `expr` field value from `ORDER BY`. When `TO const_expr` not defined sequence of filling use maximum `expr` field value from `ORDER BY`. When `STEP const_numeric_expr` defined then `const_numeric_expr` interprets `as is` for numeric types as `days` for Date type and as `seconds` for DateTime type. From b428d282e0fd58a1821c3027b3515a788b2410db Mon Sep 17 00:00:00 2001 From: tavplubix Date: Mon, 26 Jul 2021 11:38:30 +0300 Subject: [PATCH 025/198] Update arcadia_skip_list.txt --- tests/queries/0_stateless/arcadia_skip_list.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 903c72f044a..eab220fa213 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -93,6 +93,8 @@ 01138_join_on_distributed_and_tmp 01153_attach_mv_uuid 01155_rename_move_materialized_view +01157_replace_table +01185_create_or_replace_table 01191_rename_dictionary 01200_mutations_memory_consumption 01211_optimize_skip_unused_shards_type_mismatch From d541cd5c00595db58233f4132ae6febb8688303b Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Wed, 28 Jul 2021 04:18:16 +0300 Subject: [PATCH 026/198] Update docs/en/sql-reference/statements/select/order-by.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/select/order-by.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index 3d71e2799ea..d73986a7aeb 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -280,7 +280,7 @@ To fill multiple columns, add `WITH FILL` modifier with optional parameters afte ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr] ``` -`WITH FILL` can be applied only for fields with Numeric (all kind of float, decimal, int) or Date/DateTime types. When applied for `String` fields, missed values will be filled with empty strings. +`WITH FILL` can be applied for fields with Numeric (all kind of float, decimal, int) or Date/DateTime types. When applied for `String` fields, missed values are filled with empty strings. When `FROM const_expr` not defined sequence of filling use minimal `expr` field value from `ORDER BY`. When `TO const_expr` not defined sequence of filling use maximum `expr` field value from `ORDER BY`. When `STEP const_numeric_expr` defined then `const_numeric_expr` interprets `as is` for numeric types as `days` for Date type and as `seconds` for DateTime type. @@ -400,4 +400,4 @@ Result: └────────────┴────────────┴──────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/order-by/) \ No newline at end of file +[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/order-by/) From 08a7140882c3967bb6f01cd6490102f0c3037c4b Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Wed, 28 Jul 2021 04:18:33 +0300 Subject: [PATCH 027/198] Update docs/en/sql-reference/statements/select/order-by.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/select/order-by.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index d73986a7aeb..09f061b6ca4 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -286,7 +286,7 @@ When `TO const_expr` not defined sequence of filling use maximum `expr` field va When `STEP const_numeric_expr` defined then `const_numeric_expr` interprets `as is` for numeric types as `days` for Date type and as `seconds` for DateTime type. When `STEP const_numeric_expr` omitted then sequence of filling use `1.0` for numeric type, `1 day` for Date type and `1 second` for DateTime type. -Example query without `WITH FILL`: +Example of query without `WITH FILL`: ``` sql SELECT n, source FROM ( From fce9351256adeb9374a46b8699003a1a45a34a76 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 23 Mar 2021 15:01:13 +0000 Subject: [PATCH 028/198] MongoDB SSL Connection --- src/Storages/StorageMongoDB.cpp | 19 +++++++++-- src/Storages/StorageMongoDB.h | 3 ++ src/Storages/StorageMongoDBSocketFactory.cpp | 36 ++++++++++++++++++++ src/Storages/StorageMongoDBSocketFactory.h | 19 +++++++++++ 4 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 src/Storages/StorageMongoDBSocketFactory.cpp create mode 100644 src/Storages/StorageMongoDBSocketFactory.h diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index e27d16ecc68..5e758225b44 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -1,4 +1,5 @@ #include "StorageMongoDB.h" +#include "StorageMongoDBSocketFactory.h" #include #include @@ -33,6 +34,7 @@ StorageMongoDB::StorageMongoDB( const std::string & collection_name_, const std::string & username_, const std::string & password_, + const std::string & options_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const String & comment) @@ -43,6 +45,8 @@ StorageMongoDB::StorageMongoDB( , collection_name(collection_name_) , username(username_) , password(password_) + , options(options_) + , uri("mongodb://" + username_ + ":" + password_ + "@" + host_ + ":" + std::to_string(port_) + "/" + database_name_ + "?" + options_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); @@ -56,7 +60,10 @@ void StorageMongoDB::connectIfNotConnected() { std::lock_guard lock{connection_mutex}; if (!connection) - connection = std::make_shared(host, port); + { + StorageMongoDBSocketFactory factory; + connection = std::make_shared(uri, factory); + } if (!authenticated) { @@ -102,9 +109,9 @@ void registerStorageMongoDB(StorageFactory & factory) { ASTs & engine_args = args.engine_args; - if (engine_args.size() != 5) + if (engine_args.size() < 5 || engine_args.size() > 6) throw Exception( - "Storage MongoDB requires 5 parameters: MongoDB('host:port', database, collection, 'user', 'password').", + "Storage MongoDB requires from 5 to 6 parameters: MongoDB('host:port', database, collection, 'user', 'password' [, 'options']).", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); for (auto & engine_arg : engine_args) @@ -118,6 +125,11 @@ void registerStorageMongoDB(StorageFactory & factory) const String & username = engine_args[3]->as().value.safeGet(); const String & password = engine_args[4]->as().value.safeGet(); + String options = ""; + + if (engine_args.size() >= 6) + options = engine_args[5]->as().value.safeGet(); + return StorageMongoDB::create( args.table_id, parsed_host_port.first, @@ -126,6 +138,7 @@ void registerStorageMongoDB(StorageFactory & factory) collection, username, password, + options, args.columns, args.constraints, args.comment); diff --git a/src/Storages/StorageMongoDB.h b/src/Storages/StorageMongoDB.h index 2553acdd40c..3014b88a9ca 100644 --- a/src/Storages/StorageMongoDB.h +++ b/src/Storages/StorageMongoDB.h @@ -26,6 +26,7 @@ public: const std::string & collection_name_, const std::string & username_, const std::string & password_, + const std::string & options_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const String & comment); @@ -50,6 +51,8 @@ private: const std::string collection_name; const std::string username; const std::string password; + const std::string options; + const std::string uri; std::shared_ptr connection; bool authenticated = false; diff --git a/src/Storages/StorageMongoDBSocketFactory.cpp b/src/Storages/StorageMongoDBSocketFactory.cpp new file mode 100644 index 00000000000..ed2548c7e3b --- /dev/null +++ b/src/Storages/StorageMongoDBSocketFactory.cpp @@ -0,0 +1,36 @@ +#include "StorageMongoDBSocketFactory.h" + +#include +#include +#include + + +#pragma clang diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wunused-parameter" + + +namespace DB +{ + +Poco::Net::StreamSocket StorageMongoDBSocketFactory::createSocket(const std::string & host, int port, Poco::Timespan connectTimeout, bool secure) +{ + return secure ? createSecureSocket(host, port) : createPlainSocket(host, port); +} + +Poco::Net::StreamSocket StorageMongoDBSocketFactory::createPlainSocket(const std::string & host, int port) +{ + Poco::Net::SocketAddress address(host, port); + Poco::Net::StreamSocket socket(address); + + return socket; +} + +Poco::Net::StreamSocket StorageMongoDBSocketFactory::createSecureSocket(const std::string & host, int port) +{ + Poco::Net::SocketAddress address(host, port); + Poco::Net::SecureStreamSocket socket(address, host); + + return socket; +} + +} diff --git a/src/Storages/StorageMongoDBSocketFactory.h b/src/Storages/StorageMongoDBSocketFactory.h new file mode 100644 index 00000000000..91b05698401 --- /dev/null +++ b/src/Storages/StorageMongoDBSocketFactory.h @@ -0,0 +1,19 @@ +#pragma once + +#include + + +namespace DB +{ + +class StorageMongoDBSocketFactory : public Poco::MongoDB::Connection::SocketFactory +{ +public: + virtual Poco::Net::StreamSocket createSocket(const std::string & host, int port, Poco::Timespan connectTimeout, bool secure) override; + +private: + Poco::Net::StreamSocket createPlainSocket(const std::string & host, int port); + Poco::Net::StreamSocket createSecureSocket(const std::string & host, int port); +}; + +} From 7e785d6b123ac12f0fcce84c737a7946ffa746b1 Mon Sep 17 00:00:00 2001 From: OmarBazaraa Date: Tue, 23 Mar 2021 15:18:12 +0000 Subject: [PATCH 029/198] Docs --- .../table-engines/integrations/mongodb.md | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/mongodb.md b/docs/en/engines/table-engines/integrations/mongodb.md index a378ab03f55..9454b783b22 100644 --- a/docs/en/engines/table-engines/integrations/mongodb.md +++ b/docs/en/engines/table-engines/integrations/mongodb.md @@ -15,7 +15,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name name1 [type1], name2 [type2], ... -) ENGINE = MongoDB(host:port, database, collection, user, password); +) ENGINE = MongoDB(host:port, database, collection, user, password [, options]); ``` **Engine Parameters** @@ -30,18 +30,30 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name - `password` — User password. +- `options` — MongoDB connection string options (optional parameter). + ## Usage Example {#usage-example} -Table in ClickHouse which allows to read data from MongoDB collection: +Create a table in ClickHouse which allows to read data from MongoDB collection: ``` text CREATE TABLE mongo_table ( - key UInt64, + key UInt64, data String ) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'testuser', 'clickhouse'); ``` +To read from an SSL secured MongoDB server: + +``` text +CREATE TABLE mongo_table_ssl +( + key UInt64, + data String +) ENGINE = MongoDB('mongo2:27017', 'test', 'simple_table', 'testuser', 'clickhouse', 'ssl=true'); +``` + Query: ``` sql @@ -54,4 +66,6 @@ SELECT COUNT() FROM mongo_table; └─────────┘ ``` + + [Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/mongodb/) From fea1a5d32157cc312b9d7dfd6ed093fed76e7b43 Mon Sep 17 00:00:00 2001 From: OmarBazaraa Date: Tue, 23 Mar 2021 15:27:06 +0000 Subject: [PATCH 030/198] Refactor --- docs/en/engines/table-engines/integrations/mongodb.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/mongodb.md b/docs/en/engines/table-engines/integrations/mongodb.md index 9454b783b22..9839893d4e8 100644 --- a/docs/en/engines/table-engines/integrations/mongodb.md +++ b/docs/en/engines/table-engines/integrations/mongodb.md @@ -66,6 +66,4 @@ SELECT COUNT() FROM mongo_table; └─────────┘ ``` - - [Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/mongodb/) From 8acc290bc5abff9aba959d0cdda5a50b85a3eaea Mon Sep 17 00:00:00 2001 From: OmarBazaraa Date: Tue, 27 Jul 2021 15:54:13 +0000 Subject: [PATCH 031/198] Add integration tests --- .../compose/docker_compose_mongo_secure.yml | 13 ++++++ tests/integration/helpers/cluster.py | 34 ++++++++++---- tests/integration/helpers/mongo_cert.pem | 44 +++++++++++++++++++ tests/integration/helpers/mongo_secure.conf | 5 +++ .../configs_secure/config.d/ssl_conf.xml | 8 ++++ .../integration/test_storage_mongodb/test.py | 41 ++++++++++++++--- 6 files changed, 131 insertions(+), 14 deletions(-) create mode 100644 docker/test/integration/runner/compose/docker_compose_mongo_secure.yml create mode 100644 tests/integration/helpers/mongo_cert.pem create mode 100644 tests/integration/helpers/mongo_secure.conf create mode 100644 tests/integration/test_storage_mongodb/configs_secure/config.d/ssl_conf.xml diff --git a/docker/test/integration/runner/compose/docker_compose_mongo_secure.yml b/docker/test/integration/runner/compose/docker_compose_mongo_secure.yml new file mode 100644 index 00000000000..5d283cfc343 --- /dev/null +++ b/docker/test/integration/runner/compose/docker_compose_mongo_secure.yml @@ -0,0 +1,13 @@ +version: '2.3' +services: + mongo1: + image: mongo:3.6 + restart: always + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: clickhouse + volumes: + - ${MONGO_CONFIG_PATH}:/mongo/ + ports: + - ${MONGO_EXTERNAL_PORT}:${MONGO_INTERNAL_PORT} + command: --config /mongo/mongo_secure.conf --profile=2 --verbose diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 720d5e2068b..7f89bda5111 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -110,6 +110,7 @@ def subprocess_check_call(args, detach=False, nothrow=False): #logging.info('run:' + ' '.join(args)) return run_and_check(args, detach=detach, nothrow=nothrow) + def get_odbc_bridge_path(): path = os.environ.get('CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH') if path is None: @@ -261,6 +262,7 @@ class ClickHouseCluster: self.with_hdfs = False self.with_kerberized_hdfs = False self.with_mongo = False + self.with_mongo_secure = False self.with_net_trics = False self.with_redis = False self.with_cassandra = False @@ -548,7 +550,6 @@ class ClickHouseCluster: return self.base_mysql_client_cmd - def setup_mysql_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_mysql = True env_variables['MYSQL_HOST'] = self.mysql_host @@ -680,6 +681,17 @@ class ClickHouseCluster: '--file', p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')] return self.base_rabbitmq_cmd + def setup_mongo_secure_cmd(self, instance, env_variables, docker_compose_yml_dir): + self.with_mongo = self.with_mongo_secure = True + env_variables['MONGO_HOST'] = self.mongo_host + env_variables['MONGO_EXTERNAL_PORT'] = str(self.mongo_port) + env_variables['MONGO_INTERNAL_PORT'] = "27017" + env_variables['MONGO_CONFIG_PATH'] = HELPERS_DIR + self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mongo_secure.yml')]) + self.base_mongo_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, + '--file', p.join(docker_compose_yml_dir, 'docker_compose_mongo_secure.yml')] + return self.base_mongo_cmd + def setup_mongo_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_mongo = True env_variables['MONGO_HOST'] = self.mongo_host @@ -723,7 +735,8 @@ class ClickHouseCluster: macros=None, with_zookeeper=False, with_zookeeper_secure=False, with_mysql_client=False, with_mysql=False, with_mysql8=False, with_mysql_cluster=False, with_kafka=False, with_kerberized_kafka=False, with_rabbitmq=False, clickhouse_path_dir=None, - with_odbc_drivers=False, with_postgres=False, with_postgres_cluster=False, with_hdfs=False, with_kerberized_hdfs=False, with_mongo=False, + with_odbc_drivers=False, with_postgres=False, with_postgres_cluster=False, with_hdfs=False, + with_kerberized_hdfs=False, with_mongo=False, with_mongo_secure=False, with_redis=False, with_minio=False, with_cassandra=False, with_jdbc_bridge=False, hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag=None, stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None, @@ -776,7 +789,7 @@ class ClickHouseCluster: with_kerberized_kafka=with_kerberized_kafka, with_rabbitmq=with_rabbitmq, with_kerberized_hdfs=with_kerberized_hdfs, - with_mongo=with_mongo, + with_mongo=with_mongo or with_mongo_secure, with_redis=with_redis, with_minio=with_minio, with_cassandra=with_cassandra, @@ -861,8 +874,11 @@ class ClickHouseCluster: if with_kerberized_hdfs and not self.with_kerberized_hdfs: cmds.append(self.setup_kerberized_hdfs_cmd(instance, env_variables, docker_compose_yml_dir)) - if with_mongo and not self.with_mongo: - cmds.append(self.setup_mongo_cmd(instance, env_variables, docker_compose_yml_dir)) + if (with_mongo or with_mongo_secure) and not (self.with_mongo or self.with_mongo_secure): + if with_mongo_secure: + cmds.append(self.setup_mongo_secure_cmd(instance, env_variables, docker_compose_yml_dir)) + else + cmds.append(self.setup_mongo_cmd(instance, env_variables, docker_compose_yml_dir)) if self.with_net_trics: for cmd in cmds: @@ -1234,7 +1250,6 @@ class ClickHouseCluster: logging.debug("Waiting for Kafka to start up") time.sleep(1) - def wait_hdfs_to_start(self, timeout=300, check_marker=False): start = time.time() while time.time() - start < timeout: @@ -1251,9 +1266,11 @@ class ClickHouseCluster: raise Exception("Can't wait HDFS to start") - def wait_mongo_to_start(self, timeout=180): + def wait_mongo_to_start(self, timeout=30, secure=False): connection_str = 'mongodb://{user}:{password}@{host}:{port}'.format( host='localhost', port=self.mongo_port, user='root', password='clickhouse') + if secure: + connection_str += '/?tls=true&tlsAllowInvalidCertificates=true' connection = pymongo.MongoClient(connection_str) start = time.time() while time.time() - start < timeout: @@ -1320,7 +1337,6 @@ class ClickHouseCluster: raise Exception("Can't wait Schema Registry to start") - def wait_cassandra_to_start(self, timeout=180): self.cassandra_ip = self.get_instance_ip(self.cassandra_host) cass_client = cassandra.cluster.Cluster([self.cassandra_ip], port=self.cassandra_port, load_balancing_policy=RoundRobinPolicy()) @@ -1505,7 +1521,7 @@ class ClickHouseCluster: if self.with_mongo and self.base_mongo_cmd: logging.debug('Setup Mongo') run_and_check(self.base_mongo_cmd + common_opts) - self.wait_mongo_to_start(30) + self.wait_mongo_to_start(30, secure=self.with_mongo_secure) if self.with_redis and self.base_redis_cmd: logging.debug('Setup Redis') diff --git a/tests/integration/helpers/mongo_cert.pem b/tests/integration/helpers/mongo_cert.pem new file mode 100644 index 00000000000..9e18b1d4469 --- /dev/null +++ b/tests/integration/helpers/mongo_cert.pem @@ -0,0 +1,44 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAtz2fpa8hyUff8u8jYlh20HbkOO8hQi64Ke2Prack2Br0lhOr +1MI6I8nVk5iDrt+7ix2Cnt+2aZKb6HJv0CG1V25yWg+jgsXeIT1KHTJf8rTmYxhb +t+ye+S1Z0h/Rt+xqSd9XXfzOLPGHYfyx6ZQ4AumO/HoEFD4IH/qiREjwtOfRXuhz +CohqtUTyYR7pJmZqBSuGac461WVRisnjfKRxeVa3itc84/RgktgYej2x4PQBFk13 +xAXKrWmHkwdgWklTuuK8Gtoqz65Y4/J9CSl+Bd08QDdRnaVvq1u1eNTZg1BVyeRv +jFYBMSathKASrng5nK66Fdilw6tO/9khaP0SDQIDAQABAoIBAAm/5qGrKtIJ1/mW +Dbzq1g+Lc+MvngZmc/gPIsjrjsNM09y0WT0txGgpEgsTX1ZLoy/otw16+7qsSU1Z +4WcilAJ95umx0VJg8suz9iCNkJtaUrPNFPw5Q9AgQJo0hTUTCCi8EGr4y4OKqlhl +WJYEA+LryGbYmyT0k/wXmtClTOFjKS09mK4deQ1DqbBxayR9MUZgRJzEODA8eGXs +Rc6fJUenMVNMzIVLgpossRtKImoZNcf5UtCKL3HECunndQeMu4zuqLMU+EzL1F/o +iHKF7v3CVmsK0OxNJfOfT0abN3XaJttFwTJyghQjgP8OX1IKjlj3vo9xwEDfVUEf +GVIER0UCgYEA2j+kjaT3Dw2lNlBEsA8uvVlLJHBc9JBtMGduGYsIEgsL/iStqXv4 +xoA9N3CwkN/rrQpDfi/16DMXAAYdjGulPwiMNFBY22TCzhtPC2mAnfaSForxwZCs +lwc3KkIloo3N5XvN78AuZf8ewiS+bOEj+HHHqqSb1+u/csuaXO9neesCgYEA1u/I +Mlt/pxJkH+c3yOskwCh/CNhq9szi8G9LXROIQ58BT2ydJSEPpt7AhUTtQGimQQTW +KLiffJSkjuVaFckR1GjCoAmFGYw9wUb+TmFNScz5pJ2dXse8aBysAMIQfEIcRAEa +gKnkLBH6nw3+/Hm3xwoBc35t8Pa2ek7LsWDfbecCgYBhilQW4gVw+t49uf4Y2ZBA +G+pTbMx+mRXTrkYssFB5D+raOLZMqxVyUdoKLxkahpkkCxRDD1hN4JeE8Ta/jVSb +KUzQDKDJ3OybhOT86rgK4SpFXO/TXL9l+FmVT17WmZ3N1Fkjr7aM60pp5lYc/zo+ +TUu5XjwwcjJsMcbZhj2u5QKBgQCDNuUn4PYAP9kCJPzIWs0XxmEvPDeorZIJqFgA +3XC9n2+EVlFlHlbYz3oGofqY7Io6fUJkn7k1q+T+G4QwcozA+KeAXe90lkoJGVcc +8IfnewwYc+RjvVoG0SIsYE0CHrX0yhus2oqiYON4gGnfJkuMZk5WfKOPjH4AEuSF +SBd+lwKBgQCHG/DA6u2mYmezPF9hebWFoyAVSr2PDXDhu8cNNHCpx9GewJXhuK/P +tW8mazHzUuJKBvmaUXDIXFh4K6FNhjH16p5jR1w3hsPE7NEZhjfVRaUYPmBqaOYR +jp8H+Sh5g4Rwbtfp6Qhu6UAKi/y6Vozs5GkJtSiNrjNDVrD+sGGrXA== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICqDCCAZACFBdaMnuT0pWhmrh05UT3HXJ+kI0yMA0GCSqGSIb3DQEBCwUAMA0x +CzAJBgNVBAMMAmNhMB4XDTIxMDQwNjE3MDQxNVoXDTIyMDQwNjE3MDQxNVowFDES +MBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEAtz2fpa8hyUff8u8jYlh20HbkOO8hQi64Ke2Prack2Br0lhOr1MI6I8nVk5iD +rt+7ix2Cnt+2aZKb6HJv0CG1V25yWg+jgsXeIT1KHTJf8rTmYxhbt+ye+S1Z0h/R +t+xqSd9XXfzOLPGHYfyx6ZQ4AumO/HoEFD4IH/qiREjwtOfRXuhzCohqtUTyYR7p +JmZqBSuGac461WVRisnjfKRxeVa3itc84/RgktgYej2x4PQBFk13xAXKrWmHkwdg +WklTuuK8Gtoqz65Y4/J9CSl+Bd08QDdRnaVvq1u1eNTZg1BVyeRvjFYBMSathKAS +rng5nK66Fdilw6tO/9khaP0SDQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAct2If +isMLHIqyL9GjY4b0xcxF4svFU/DUwNanStmoFMW1ifPf1cCqeMzyQOxBCDdMs0RT +hBbDYHW0BMXDqYIr3Ktbu38/3iVyr3pb56YOCKy8yHXpmKEaUBhCknSLcQyvNfeS +tM+DWsKFTZfyR5px+WwXbGKVMYwLaTON+/wcv1MeKMig3CxluaCpEJVYYwAiUc4K +sgvQNAunwGmPLPoXtUnpR2ZWiQA5R6yjS1oIe+8vpryFP6kjhWs0HR0jZEtLulV5 +WXUuxkqTXiBIvYpsmusoR44e9rptwLbV1wL/LUScRt9ttqFM3N5/Pof+2UwkSjGB +GAyPmw0Pkqtt+lva +-----END CERTIFICATE----- diff --git a/tests/integration/helpers/mongo_secure.conf b/tests/integration/helpers/mongo_secure.conf new file mode 100644 index 00000000000..1128b16b546 --- /dev/null +++ b/tests/integration/helpers/mongo_secure.conf @@ -0,0 +1,5 @@ +net: + ssl: + mode: requireSSL + PEMKeyFile: /mongo/mongo_cert.pem + allowConnectionsWithoutCertificates: true diff --git a/tests/integration/test_storage_mongodb/configs_secure/config.d/ssl_conf.xml b/tests/integration/test_storage_mongodb/configs_secure/config.d/ssl_conf.xml new file mode 100644 index 00000000000..e14aac81e17 --- /dev/null +++ b/tests/integration/test_storage_mongodb/configs_secure/config.d/ssl_conf.xml @@ -0,0 +1,8 @@ + + + + + none + + + diff --git a/tests/integration/test_storage_mongodb/test.py b/tests/integration/test_storage_mongodb/test.py index 75af909faec..55fa6b7dbc3 100644 --- a/tests/integration/test_storage_mongodb/test.py +++ b/tests/integration/test_storage_mongodb/test.py @@ -5,24 +5,29 @@ from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', with_mongo=True) - @pytest.fixture(scope="module") -def started_cluster(): +def started_cluster(request): try: + cluster = ClickHouseCluster(__file__) + node = cluster.add_instance('node', + main_configs=["configs_secure/config.d/ssl_conf.xml"], + with_mongo=True, + with_mongo_secure=request.param) cluster.start() yield cluster finally: cluster.shutdown() -def get_mongo_connection(started_cluster): +def get_mongo_connection(started_cluster, secure=False): connection_str = 'mongodb://root:clickhouse@localhost:{}'.format(started_cluster.mongo_port) + if secure: + connection_str += '/?tls=true&tlsAllowInvalidCertificates=true' return pymongo.MongoClient(connection_str) +@pytest.mark.parametrize('started_cluster', [False], indirect=['started_cluster']) def test_simple_select(started_cluster): mongo_connection = get_mongo_connection(started_cluster) db = mongo_connection['test'] @@ -33,6 +38,7 @@ def test_simple_select(started_cluster): data.append({'key': i, 'data': hex(i * i)}) simple_mongo_table.insert_many(data) + node = started_cluster.instance['node'] node.query( "CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse')") @@ -42,6 +48,7 @@ def test_simple_select(started_cluster): assert node.query("SELECT data from simple_mongo_table where key = 42") == hex(42 * 42) + '\n' +@pytest.mark.parametrize('started_cluster', [False], indirect=['started_cluster']) def test_complex_data_type(started_cluster): mongo_connection = get_mongo_connection(started_cluster) db = mongo_connection['test'] @@ -52,6 +59,7 @@ def test_complex_data_type(started_cluster): data.append({'key': i, 'data': hex(i * i), 'dict': {'a': i, 'b': str(i)}}) incomplete_mongo_table.insert_many(data) + node = started_cluster.instance['node'] node.query( "CREATE TABLE incomplete_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'complex_table', 'root', 'clickhouse')") @@ -61,6 +69,7 @@ def test_complex_data_type(started_cluster): assert node.query("SELECT data from incomplete_mongo_table where key = 42") == hex(42 * 42) + '\n' +@pytest.mark.parametrize('started_cluster', [False], indirect=['started_cluster']) def test_incorrect_data_type(started_cluster): mongo_connection = get_mongo_connection(started_cluster) db = mongo_connection['test'] @@ -71,6 +80,7 @@ def test_incorrect_data_type(started_cluster): data.append({'key': i, 'data': hex(i * i), 'aaaa': 'Hello'}) strange_mongo_table.insert_many(data) + node = started_cluster.instance['node'] node.query( "CREATE TABLE strange_mongo_table(key String, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'strange_table', 'root', 'clickhouse')") @@ -85,3 +95,24 @@ def test_incorrect_data_type(started_cluster): with pytest.raises(QueryRuntimeException): node.query("SELECT bbbb FROM strange_mongo_table2") + + +@pytest.mark.parametrize('started_cluster', [True], indirect=['started_cluster']) +def test_secure_connection(started_cluster): + mongo_connection = get_mongo_connection(secure=True) + db = mongo_connection['test'] + db.add_user('root', 'clickhouse') + simple_mongo_table = db['simple_table'] + data = [] + for i in range(0, 100): + data.append({'key': i, 'data': hex(i * i)}) + simple_mongo_table.insert_many(data) + + node = started_cluster.instance['node'] + node.query( + "CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', 'ssl=true')") + + assert node.query("SELECT COUNT() FROM simple_mongo_table") == '100\n' + assert node.query("SELECT sum(key) FROM simple_mongo_table") == str(sum(range(0, 100))) + '\n' + + assert node.query("SELECT data from simple_mongo_table where key = 42") == hex(42 * 42) + '\n' From 58ea75915d7434672abbff20f492d6fe3d8c2e18 Mon Sep 17 00:00:00 2001 From: OmarBazaraa Date: Mon, 24 May 2021 12:11:03 +0000 Subject: [PATCH 032/198] Fix --- src/Storages/StorageMongoDBSocketFactory.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Storages/StorageMongoDBSocketFactory.cpp b/src/Storages/StorageMongoDBSocketFactory.cpp index ed2548c7e3b..a6ff53a23d7 100644 --- a/src/Storages/StorageMongoDBSocketFactory.cpp +++ b/src/Storages/StorageMongoDBSocketFactory.cpp @@ -4,6 +4,9 @@ #include #include +#if USE_SSL +# include +#endif #pragma clang diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic ignored "-Wunused-parameter" @@ -14,7 +17,11 @@ namespace DB Poco::Net::StreamSocket StorageMongoDBSocketFactory::createSocket(const std::string & host, int port, Poco::Timespan connectTimeout, bool secure) { +#if USE_SSL return secure ? createSecureSocket(host, port) : createPlainSocket(host, port); +#else + return createPlainSocket(host, port); +#endif } Poco::Net::StreamSocket StorageMongoDBSocketFactory::createPlainSocket(const std::string & host, int port) @@ -25,6 +32,7 @@ Poco::Net::StreamSocket StorageMongoDBSocketFactory::createPlainSocket(const std return socket; } +#if USE_SSL Poco::Net::StreamSocket StorageMongoDBSocketFactory::createSecureSocket(const std::string & host, int port) { Poco::Net::SocketAddress address(host, port); @@ -32,5 +40,6 @@ Poco::Net::StreamSocket StorageMongoDBSocketFactory::createSecureSocket(const st return socket; } +#endif } From 0d029c1cbc8fe71d364c6e798c1966d0d3a18a53 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Fri, 28 May 2021 15:02:10 +0300 Subject: [PATCH 033/198] Update StorageMongoDBSocketFactory.cpp --- src/Storages/StorageMongoDBSocketFactory.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/StorageMongoDBSocketFactory.cpp b/src/Storages/StorageMongoDBSocketFactory.cpp index a6ff53a23d7..3352238a5c8 100644 --- a/src/Storages/StorageMongoDBSocketFactory.cpp +++ b/src/Storages/StorageMongoDBSocketFactory.cpp @@ -2,7 +2,6 @@ #include #include -#include #if USE_SSL # include From 14171435606d041e18d9c6f4b17a24d69e8d7504 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Fri, 28 May 2021 18:25:24 +0300 Subject: [PATCH 034/198] Update StorageMongoDBSocketFactory.cpp --- src/Storages/StorageMongoDBSocketFactory.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Storages/StorageMongoDBSocketFactory.cpp b/src/Storages/StorageMongoDBSocketFactory.cpp index 3352238a5c8..cb6ffd052c7 100644 --- a/src/Storages/StorageMongoDBSocketFactory.cpp +++ b/src/Storages/StorageMongoDBSocketFactory.cpp @@ -1,5 +1,9 @@ #include "StorageMongoDBSocketFactory.h" +#if !defined(ARCADIA_BUILD) +# include +#endif + #include #include From 6b9e0f5a6f0359a5eac66e099a915c6bd26e65cd Mon Sep 17 00:00:00 2001 From: OmarBazaraa Date: Tue, 27 Jul 2021 17:43:41 +0000 Subject: [PATCH 035/198] Fix --- tests/integration/helpers/cluster.py | 2 +- tests/integration/test_storage_mongodb/test.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 7f89bda5111..6fe01b5df03 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -877,7 +877,7 @@ class ClickHouseCluster: if (with_mongo or with_mongo_secure) and not (self.with_mongo or self.with_mongo_secure): if with_mongo_secure: cmds.append(self.setup_mongo_secure_cmd(instance, env_variables, docker_compose_yml_dir)) - else + else: cmds.append(self.setup_mongo_cmd(instance, env_variables, docker_compose_yml_dir)) if self.with_net_trics: diff --git a/tests/integration/test_storage_mongodb/test.py b/tests/integration/test_storage_mongodb/test.py index 55fa6b7dbc3..415f1c1cb33 100644 --- a/tests/integration/test_storage_mongodb/test.py +++ b/tests/integration/test_storage_mongodb/test.py @@ -38,7 +38,7 @@ def test_simple_select(started_cluster): data.append({'key': i, 'data': hex(i * i)}) simple_mongo_table.insert_many(data) - node = started_cluster.instance['node'] + node = started_cluster.instances['node'] node.query( "CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse')") @@ -59,7 +59,7 @@ def test_complex_data_type(started_cluster): data.append({'key': i, 'data': hex(i * i), 'dict': {'a': i, 'b': str(i)}}) incomplete_mongo_table.insert_many(data) - node = started_cluster.instance['node'] + node = started_cluster.instances['node'] node.query( "CREATE TABLE incomplete_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'complex_table', 'root', 'clickhouse')") @@ -80,7 +80,7 @@ def test_incorrect_data_type(started_cluster): data.append({'key': i, 'data': hex(i * i), 'aaaa': 'Hello'}) strange_mongo_table.insert_many(data) - node = started_cluster.instance['node'] + node = started_cluster.instances['node'] node.query( "CREATE TABLE strange_mongo_table(key String, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'strange_table', 'root', 'clickhouse')") @@ -99,7 +99,7 @@ def test_incorrect_data_type(started_cluster): @pytest.mark.parametrize('started_cluster', [True], indirect=['started_cluster']) def test_secure_connection(started_cluster): - mongo_connection = get_mongo_connection(secure=True) + mongo_connection = get_mongo_connection(started_cluster, secure=True) db = mongo_connection['test'] db.add_user('root', 'clickhouse') simple_mongo_table = db['simple_table'] @@ -108,7 +108,7 @@ def test_secure_connection(started_cluster): data.append({'key': i, 'data': hex(i * i)}) simple_mongo_table.insert_many(data) - node = started_cluster.instance['node'] + node = started_cluster.instances['node'] node.query( "CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', 'ssl=true')") From a154ccef7bfa8caff93bf20545ff30c5f508cb2b Mon Sep 17 00:00:00 2001 From: OmarBazaraa Date: Wed, 28 Jul 2021 08:54:36 +0000 Subject: [PATCH 036/198] Fix --- src/Storages/StorageMongoDBSocketFactory.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageMongoDBSocketFactory.cpp b/src/Storages/StorageMongoDBSocketFactory.cpp index cb6ffd052c7..ac323a50252 100644 --- a/src/Storages/StorageMongoDBSocketFactory.cpp +++ b/src/Storages/StorageMongoDBSocketFactory.cpp @@ -11,8 +11,11 @@ # include #endif -#pragma clang diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wunused-parameter" +#ifdef __clang__ +# pragma clang diagnostic ignored "-Wunused-parameter" +#else +# pragma GCC diagnostic ignored "-Wunused-parameter" +#endif namespace DB From d8fb1cb0e0e86a47abe2db29db7309835448a2f7 Mon Sep 17 00:00:00 2001 From: OmarBazaraa Date: Wed, 28 Jul 2021 14:08:57 +0000 Subject: [PATCH 037/198] Ref. --- src/Storages/StorageMongoDBSocketFactory.cpp | 22 +++++++++----------- src/Storages/StorageMongoDBSocketFactory.h | 4 ++-- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/Storages/StorageMongoDBSocketFactory.cpp b/src/Storages/StorageMongoDBSocketFactory.cpp index ac323a50252..12e554f26ba 100644 --- a/src/Storages/StorageMongoDBSocketFactory.cpp +++ b/src/Storages/StorageMongoDBSocketFactory.cpp @@ -11,12 +11,6 @@ # include #endif -#ifdef __clang__ -# pragma clang diagnostic ignored "-Wunused-parameter" -#else -# pragma GCC diagnostic ignored "-Wunused-parameter" -#endif - namespace DB { @@ -24,25 +18,29 @@ namespace DB Poco::Net::StreamSocket StorageMongoDBSocketFactory::createSocket(const std::string & host, int port, Poco::Timespan connectTimeout, bool secure) { #if USE_SSL - return secure ? createSecureSocket(host, port) : createPlainSocket(host, port); + return secure ? createSecureSocket(host, port, connectTimeout) : createPlainSocket(host, port, connectTimeout); #else - return createPlainSocket(host, port); + return createPlainSocket(host, port, connectTimeout); #endif } -Poco::Net::StreamSocket StorageMongoDBSocketFactory::createPlainSocket(const std::string & host, int port) +Poco::Net::StreamSocket StorageMongoDBSocketFactory::createPlainSocket(const std::string & host, int port, Poco::Timespan connectTimeout) { Poco::Net::SocketAddress address(host, port); - Poco::Net::StreamSocket socket(address); + Poco::Net::StreamSocket socket; + + socket.connect(address, connectTimeout); return socket; } #if USE_SSL -Poco::Net::StreamSocket StorageMongoDBSocketFactory::createSecureSocket(const std::string & host, int port) +Poco::Net::StreamSocket StorageMongoDBSocketFactory::createSecureSocket(const std::string & host, int port, Poco::Timespan connectTimeout) { Poco::Net::SocketAddress address(host, port); - Poco::Net::SecureStreamSocket socket(address, host); + Poco::Net::SecureStreamSocket socket; + + socket.connect(address, connectTimeout); return socket; } diff --git a/src/Storages/StorageMongoDBSocketFactory.h b/src/Storages/StorageMongoDBSocketFactory.h index 91b05698401..af091bf4f92 100644 --- a/src/Storages/StorageMongoDBSocketFactory.h +++ b/src/Storages/StorageMongoDBSocketFactory.h @@ -12,8 +12,8 @@ public: virtual Poco::Net::StreamSocket createSocket(const std::string & host, int port, Poco::Timespan connectTimeout, bool secure) override; private: - Poco::Net::StreamSocket createPlainSocket(const std::string & host, int port); - Poco::Net::StreamSocket createSecureSocket(const std::string & host, int port); + Poco::Net::StreamSocket createPlainSocket(const std::string & host, int port, Poco::Timespan connectTimeout); + Poco::Net::StreamSocket createSecureSocket(const std::string & host, int port, Poco::Timespan connectTimeout); }; } From 3a393042a2405543826041693fca4114bf80e573 Mon Sep 17 00:00:00 2001 From: OmarBazaraa Date: Wed, 28 Jul 2021 15:28:30 +0000 Subject: [PATCH 038/198] Fix --- src/Storages/StorageMongoDB.cpp | 2 +- src/Storages/StorageMongoDBSocketFactory.cpp | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index 5e758225b44..fd9c02700e9 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -46,7 +46,7 @@ StorageMongoDB::StorageMongoDB( , username(username_) , password(password_) , options(options_) - , uri("mongodb://" + username_ + ":" + password_ + "@" + host_ + ":" + std::to_string(port_) + "/" + database_name_ + "?" + options_) + , uri("mongodb://" + host_ + ":" + std::to_string(port_) + "/" + database_name_ + "?" + options_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); diff --git a/src/Storages/StorageMongoDBSocketFactory.cpp b/src/Storages/StorageMongoDBSocketFactory.cpp index 12e554f26ba..955507fc2bc 100644 --- a/src/Storages/StorageMongoDBSocketFactory.cpp +++ b/src/Storages/StorageMongoDBSocketFactory.cpp @@ -17,11 +17,7 @@ namespace DB Poco::Net::StreamSocket StorageMongoDBSocketFactory::createSocket(const std::string & host, int port, Poco::Timespan connectTimeout, bool secure) { -#if USE_SSL return secure ? createSecureSocket(host, port, connectTimeout) : createPlainSocket(host, port, connectTimeout); -#else - return createPlainSocket(host, port, connectTimeout); -#endif } Poco::Net::StreamSocket StorageMongoDBSocketFactory::createPlainSocket(const std::string & host, int port, Poco::Timespan connectTimeout) @@ -34,16 +30,19 @@ Poco::Net::StreamSocket StorageMongoDBSocketFactory::createPlainSocket(const std return socket; } -#if USE_SSL + Poco::Net::StreamSocket StorageMongoDBSocketFactory::createSecureSocket(const std::string & host, int port, Poco::Timespan connectTimeout) { +#if USE_SSL Poco::Net::SocketAddress address(host, port); Poco::Net::SecureStreamSocket socket; socket.connect(address, connectTimeout); return socket; -} +#else + return createPlainSocket(host, port, connectTimeout); #endif +} } From a665cd330873ece732a1ebca7592dd6cfb32c2cd Mon Sep 17 00:00:00 2001 From: OmarBazaraa Date: Thu, 29 Jul 2021 08:38:39 +0000 Subject: [PATCH 039/198] Fixes --- src/Storages/StorageMongoDB.cpp | 2 +- src/Storages/StorageMongoDBSocketFactory.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index fd9c02700e9..1fd58a293dc 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -125,7 +125,7 @@ void registerStorageMongoDB(StorageFactory & factory) const String & username = engine_args[3]->as().value.safeGet(); const String & password = engine_args[4]->as().value.safeGet(); - String options = ""; + String options; if (engine_args.size() >= 6) options = engine_args[5]->as().value.safeGet(); diff --git a/src/Storages/StorageMongoDBSocketFactory.h b/src/Storages/StorageMongoDBSocketFactory.h index af091bf4f92..5fc423c63cb 100644 --- a/src/Storages/StorageMongoDBSocketFactory.h +++ b/src/Storages/StorageMongoDBSocketFactory.h @@ -12,8 +12,8 @@ public: virtual Poco::Net::StreamSocket createSocket(const std::string & host, int port, Poco::Timespan connectTimeout, bool secure) override; private: - Poco::Net::StreamSocket createPlainSocket(const std::string & host, int port, Poco::Timespan connectTimeout); - Poco::Net::StreamSocket createSecureSocket(const std::string & host, int port, Poco::Timespan connectTimeout); + static Poco::Net::StreamSocket createPlainSocket(const std::string & host, int port, Poco::Timespan connectTimeout); + static Poco::Net::StreamSocket createSecureSocket(const std::string & host, int port, Poco::Timespan connectTimeout); }; } From 17570868bdfd79f68045b62c0ce90aea6b2cbf5b Mon Sep 17 00:00:00 2001 From: Mikhail Date: Thu, 29 Jul 2021 19:17:48 +0300 Subject: [PATCH 040/198] some corrections and translation --- .../statements/select/order-by.md | 2 +- .../statements/select/order-by.md | 35 +++++++++++-------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index 09f061b6ca4..19cdee0dace 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -364,7 +364,7 @@ Result: └────────────┴────────────┴──────────┘ ``` -Field `d1` does not fill and use default value cause we do not have repeated values for `d2` value, and sequence for `d1` can’t be properly calculated. +Field `d1` does not fill in and use the default value cause we do not have repeated values for `d2` value, and sequence for `d1` can’t be properly calculated. The following query with a changed field in `ORDER BY`: diff --git a/docs/ru/sql-reference/statements/select/order-by.md b/docs/ru/sql-reference/statements/select/order-by.md index cb49d167b13..530837d8346 100644 --- a/docs/ru/sql-reference/statements/select/order-by.md +++ b/docs/ru/sql-reference/statements/select/order-by.md @@ -271,8 +271,8 @@ SELECT * FROM collate_test ORDER BY s ASC COLLATE 'en'; Этот модификатор также может быть скобинирован с модификатором [LIMIT ... WITH TIES](../../../sql-reference/statements/select/limit.md#limit-with-ties) -`WITH FILL` модификатор может быть установлен после `ORDER BY expr` с опциональными параметрами `FROM expr`, `TO expr` и `STEP expr`. -Все пропущенные значнеия для колонки `expr` будут заполненые значениями соответсвующими предполагаемой последовательности значений колонки, другие колонки будут заполнены значенями по умолчанию. +Модификатор `WITH FILL` может быть установлен после `ORDER BY expr` с опциональными параметрами `FROM expr`, `TO expr` и `STEP expr`. +Все пропущенные значения для колонки `expr` будут заполнены значениями, соответствующими предполагаемой последовательности значений колонки, другие колонки будут заполнены значенями по умолчанию. Используйте следующую конструкцию для заполнения нескольких колонок с модификатором `WITH FILL` с необязательными параметрами после каждого имени поля в секции `ORDER BY`. @@ -280,22 +280,22 @@ SELECT * FROM collate_test ORDER BY s ASC COLLATE 'en'; ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr] ``` -`WITH FILL` может быть применене только к полям с числовыми (все разновидности float, int, decimal) или временными (все разновидности Date, DateTime) типами. +`WITH FILL` может быть применен к полям с числовыми (все разновидности float, int, decimal) или временными (все разновидности Date, DateTime) типами. В случае применения к полям типа `String`, недостающие значения заполняются пустой строкой. Когда не определен `FROM const_expr`, последовательность заполнения использует минимальное значение поля `expr` из `ORDER BY`. Когда не определен `TO const_expr`, последовательность заполнения использует максимальное значение поля `expr` из `ORDER BY`. -Когда `STEP const_numeric_expr` определен, тогда `const_numeric_expr` интерпретируется `как есть` для числовых типов, как `дни` для типа Date и как `секунды` для типа DateTime. +Когда `STEP const_numeric_expr` определен, `const_numeric_expr` интерпретируется `как есть` для числовых типов, как `дни` для типа Date и как `секунды` для типа DateTime. Когда `STEP const_numeric_expr` не указан, тогда используется `1.0` для числовых типов, `1 день` для типа Date и `1 секунда` для типа DateTime. - -Для примера, следующий запрос +Пример запроса без использования `WITH FILL`: ```sql SELECT n, source FROM ( SELECT toFloat32(number % 10) AS n, 'original' AS source FROM numbers(10) WHERE number % 3 = 1 -) ORDER BY n +) ORDER BY n; ``` -возвращает +Результат: + ```text ┌─n─┬─source───┐ │ 1 │ original │ @@ -304,7 +304,8 @@ SELECT n, source FROM ( └───┴──────────┘ ``` -но после применения модификатора `WITH FILL` +Тот же запрос после применения модификатора `WITH FILL`: + ```sql SELECT n, source FROM ( SELECT toFloat32(number % 10) AS n, 'original' AS source @@ -312,7 +313,8 @@ SELECT n, source FROM ( ) ORDER BY n WITH FILL FROM 0 TO 5.51 STEP 0.5 ``` -возвращает +Результат: + ```text ┌───n─┬─source───┐ │ 0 │ │ @@ -331,9 +333,10 @@ SELECT n, source FROM ( └─────┴──────────┘ ``` -Для случая когда у нас есть несколько полей `ORDER BY field2 WITH FILL, field1 WITH FILL` порядок заполнения будет следовать порядку полей в секции `ORDER BY`. +Для случая с несколькими полями `ORDER BY field2 WITH FILL, field1 WITH FILL` порядок заполнения будет следовать порядку полей в секции `ORDER BY`. Пример: + ```sql SELECT toDate((number * 10) * 86400) AS d1, @@ -346,7 +349,8 @@ ORDER BY d1 WITH FILL STEP 5; ``` -возвращает +Результат: + ```text ┌───d1───────┬───d2───────┬─source───┐ │ 1970-01-11 │ 1970-01-02 │ original │ @@ -359,9 +363,9 @@ ORDER BY └────────────┴────────────┴──────────┘ ``` -Поле `d1` не заполняет и используется значение по умолчанию поскольку у нас нет повторяющихся значения для `d2` поэтому мы не можем правильно рассчитать последователность заполнения для`d1`. +Поле `d1` не заполняется и использует значение по умолчанию. Поскольку у нас нет повторяющихся значений для `d2`, мы не можем правильно рассчитать последователность заполнения для `d1`. -Cледующий запрос (с измененым порядком в ORDER BY) +Cледующий запрос (с измененым порядком в ORDER BY): ```sql SELECT toDate((number * 10) * 86400) AS d1, @@ -374,7 +378,8 @@ ORDER BY d2 WITH FILL; ``` -возвращает +Результат: + ```text ┌───d1───────┬───d2───────┬─source───┐ │ 1970-01-11 │ 1970-01-02 │ original │ From e4f3518a211fb640b215ded5b572ff9d6ffafaa6 Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Thu, 29 Jul 2021 19:58:24 +0300 Subject: [PATCH 041/198] Update docs/ru/sql-reference/statements/select/order-by.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/statements/select/order-by.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/select/order-by.md b/docs/ru/sql-reference/statements/select/order-by.md index 530837d8346..f8eb5450365 100644 --- a/docs/ru/sql-reference/statements/select/order-by.md +++ b/docs/ru/sql-reference/statements/select/order-by.md @@ -272,7 +272,7 @@ SELECT * FROM collate_test ORDER BY s ASC COLLATE 'en'; Этот модификатор также может быть скобинирован с модификатором [LIMIT ... WITH TIES](../../../sql-reference/statements/select/limit.md#limit-with-ties) Модификатор `WITH FILL` может быть установлен после `ORDER BY expr` с опциональными параметрами `FROM expr`, `TO expr` и `STEP expr`. -Все пропущенные значения для колонки `expr` будут заполнены значениями, соответствующими предполагаемой последовательности значений колонки, другие колонки будут заполнены значенями по умолчанию. +Все пропущенные значения для колонки `expr` будут заполнены значениями, соответствующими предполагаемой последовательности значений колонки, другие колонки будут заполнены значениями по умолчанию. Используйте следующую конструкцию для заполнения нескольких колонок с модификатором `WITH FILL` с необязательными параметрами после каждого имени поля в секции `ORDER BY`. From adca166266bb0e9269efd9ff02ce5e1b9aa6b0ae Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Thu, 29 Jul 2021 19:58:47 +0300 Subject: [PATCH 042/198] Update docs/ru/sql-reference/statements/select/order-by.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/statements/select/order-by.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/select/order-by.md b/docs/ru/sql-reference/statements/select/order-by.md index f8eb5450365..ba54df5e858 100644 --- a/docs/ru/sql-reference/statements/select/order-by.md +++ b/docs/ru/sql-reference/statements/select/order-by.md @@ -280,7 +280,7 @@ SELECT * FROM collate_test ORDER BY s ASC COLLATE 'en'; ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr] ``` -`WITH FILL` может быть применен к полям с числовыми (все разновидности float, int, decimal) или временными (все разновидности Date, DateTime) типами. В случае применения к полям типа `String`, недостающие значения заполняются пустой строкой. +`WITH FILL` может быть применен к полям с числовыми (все разновидности float, int, decimal) или временными (все разновидности Date, DateTime) типами. В случае применения к полям типа `String` недостающие значения заполняются пустой строкой. Когда не определен `FROM const_expr`, последовательность заполнения использует минимальное значение поля `expr` из `ORDER BY`. Когда не определен `TO const_expr`, последовательность заполнения использует максимальное значение поля `expr` из `ORDER BY`. Когда `STEP const_numeric_expr` определен, `const_numeric_expr` интерпретируется `как есть` для числовых типов, как `дни` для типа Date и как `секунды` для типа DateTime. From bfab2630d1cd27533500b5091c7d34dc586cca2a Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Thu, 29 Jul 2021 19:58:58 +0300 Subject: [PATCH 043/198] Update docs/ru/sql-reference/statements/select/order-by.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/statements/select/order-by.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/select/order-by.md b/docs/ru/sql-reference/statements/select/order-by.md index ba54df5e858..a7ba44ec6e1 100644 --- a/docs/ru/sql-reference/statements/select/order-by.md +++ b/docs/ru/sql-reference/statements/select/order-by.md @@ -333,7 +333,7 @@ SELECT n, source FROM ( └─────┴──────────┘ ``` -Для случая с несколькими полями `ORDER BY field2 WITH FILL, field1 WITH FILL` порядок заполнения будет следовать порядку полей в секции `ORDER BY`. +Для случая с несколькими полями `ORDER BY field2 WITH FILL, field1 WITH FILL` порядок заполнения будет соответствовать порядку полей в секции `ORDER BY`. Пример: From 9bbc39f6d1f86af8d703015a2b8441616bbe7c8d Mon Sep 17 00:00:00 2001 From: Filatenkov Artur <58165623+FArthur-cmd@users.noreply.github.com> Date: Thu, 29 Jul 2021 21:02:49 +0300 Subject: [PATCH 044/198] Update order-by.md --- docs/ru/sql-reference/statements/select/order-by.md | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/docs/ru/sql-reference/statements/select/order-by.md b/docs/ru/sql-reference/statements/select/order-by.md index 1fbb0c64eb4..d7d2e9c7574 100644 --- a/docs/ru/sql-reference/statements/select/order-by.md +++ b/docs/ru/sql-reference/statements/select/order-by.md @@ -274,7 +274,6 @@ SELECT * FROM collate_test ORDER BY s ASC COLLATE 'en'; Модификатор `WITH FILL` может быть установлен после `ORDER BY expr` с опциональными параметрами `FROM expr`, `TO expr` и `STEP expr`. Все пропущенные значения для колонки `expr` будут заполнены значениями, соответствующими предполагаемой последовательности значений колонки, другие колонки будут заполнены значениями по умолчанию. - Используйте следующую конструкцию для заполнения нескольких колонок с модификатором `WITH FILL` с необязательными параметрами после каждого имени поля в секции `ORDER BY`. ```sql @@ -284,7 +283,7 @@ ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_ `WITH FILL` может быть применен к полям с числовыми (все разновидности float, int, decimal) или временными (все разновидности Date, DateTime) типами. В случае применения к полям типа `String` недостающие значения заполняются пустой строкой. Когда не определен `FROM const_expr`, последовательность заполнения использует минимальное значение поля `expr` из `ORDER BY`. Когда не определен `TO const_expr`, последовательность заполнения использует максимальное значение поля `expr` из `ORDER BY`. -Когда `STEP const_numeric_expr` определен, `const_numeric_expr` интерпретируется `как есть` для числовых типов, как `дни` для типа Date и как `секунды` для типа DateTime. +Когда `STEP const_numeric_expr` определен, `const_numeric_expr` интерпретируется "как есть" для числовых типов, как "дни" для типа `Date` и как "секунды" для типа `DateTime`. Когда `STEP const_numeric_expr` не указан, тогда используется `1.0` для числовых типов, `1 день` для типа Date и `1 секунда` для типа DateTime. @@ -297,7 +296,6 @@ SELECT n, source FROM ( ``` Результат: - ```text ┌─n─┬─source───┐ │ 1 │ original │ @@ -307,7 +305,6 @@ SELECT n, source FROM ( ``` Тот же запрос после применения модификатора `WITH FILL`: - ```sql SELECT n, source FROM ( SELECT toFloat32(number % 10) AS n, 'original' AS source @@ -338,12 +335,10 @@ SELECT n, source FROM ( Для случая с несколькими полями `ORDER BY field2 WITH FILL, field1 WITH FILL` порядок заполнения будет соответствовать порядку полей в секции `ORDER BY`. Пример: - ```sql SELECT toDate((number * 10) * 86400) AS d1, toDate(number * 86400) AS d2, - 'original' AS source FROM numbers(10) WHERE (number % 3) = 1 @@ -353,7 +348,6 @@ ORDER BY ``` Результат: - ```text ┌───d1───────┬───d2───────┬─source───┐ │ 1970-01-11 │ 1970-01-02 │ original │ @@ -368,7 +362,6 @@ ORDER BY Поле `d1` не заполняется и использует значение по умолчанию. Поскольку у нас нет повторяющихся значений для `d2`, мы не можем правильно рассчитать последователность заполнения для `d1`. - Cледующий запрос (с измененым порядком в ORDER BY): ```sql SELECT @@ -383,7 +376,6 @@ ORDER BY ``` Результат: - ```text ┌───d1───────┬───d2───────┬─source───┐ │ 1970-01-11 │ 1970-01-02 │ original │ From f5ffc604d7fd83d677fe3065aecdbbfa70fd291e Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 30 Jul 2021 13:32:05 +0300 Subject: [PATCH 045/198] Updated libunwind --- contrib/libunwind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libunwind b/contrib/libunwind index 6b816d2fba3..c6d4f18a5f5 160000 --- a/contrib/libunwind +++ b/contrib/libunwind @@ -1 +1 @@ -Subproject commit 6b816d2fba3991f8fd6aaec17d92f68947eab667 +Subproject commit c6d4f18a5f5e41d8d737b2ae1adba764ba5aac89 From a4e9a56e8e2bc786edd78158650ceb36f63155bb Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Fri, 30 Jul 2021 20:57:32 +0200 Subject: [PATCH 046/198] Make changes by review on #26663 PR --- src/Functions/array/mapPopulateSeries.cpp | 80 +++++++++++-------- ...01925_map_populate_series_on_map.reference | 30 +++++++ .../01925_map_populate_series_on_map.sql | 1 + .../tests/array_tuple_map.py | 22 +++-- 4 files changed, 91 insertions(+), 42 deletions(-) diff --git a/src/Functions/array/mapPopulateSeries.cpp b/src/Functions/array/mapPopulateSeries.cpp index f8bcbf8d451..51e436e8022 100644 --- a/src/Functions/array/mapPopulateSeries.cpp +++ b/src/Functions/array/mapPopulateSeries.cpp @@ -40,7 +40,7 @@ private: if (!(which_key.isInt() || which_key.isUInt())) { throw Exception( - "Keys for " + getName() + " should be of integer type (signed or unsigned)", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Keys for {} function should be of integer type (signed or unsigned)", getName()); } if (max_key_type) @@ -49,27 +49,28 @@ private: if (which_max_key.isNullable()) throw Exception( - "Max key argument in arguments of function " + getName() + " can not be Nullable", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Max key argument in arguments of function " + getName() + " can not be Nullable"); if (key_type->getTypeId() != max_key_type->getTypeId()) - throw Exception("Max key type in " + getName() + " should be same as keys type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Max key type in {} should be same as keys type", getName()); } } DataTypePtr getReturnTypeForTuple(const DataTypes & arguments) const { if (arguments.size() < 2) - throw Exception(getName() + " accepts at least two arrays for key and value", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} accepts at least two arrays for key and value", getName()); if (arguments.size() > 3) - throw Exception("too many arguments in " + getName() + " call", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Too many arguments in {} call", getName()); const DataTypeArray * key_array_type = checkAndGetDataType(arguments[0].get()); const DataTypeArray * val_array_type = checkAndGetDataType(arguments[1].get()); if (!key_array_type || !val_array_type) - throw Exception(getName() + " accepts two arrays for key and value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {} accepts two arrays for key and value", getName()); const auto & key_type = key_array_type->getNestedType(); @@ -89,7 +90,7 @@ private: else if (arguments.size() == 2) this->checkTypes(map->getKeyType(), arguments[1]); else - throw Exception("too many arguments in " + getName() + " call", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Too many arguments in {} call", getName()); return std::make_shared(map->getKeyType(), map->getValueType()); } @@ -97,14 +98,18 @@ private: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (arguments.empty()) - throw Exception(getName() + " accepts at least one map", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, getName() + " accepts at least one map or two arrays"); if (arguments[0]->getTypeId() == TypeIndex::Array) return getReturnTypeForTuple(arguments); else if (arguments[0]->getTypeId() == TypeIndex::Map) return getReturnTypeForMap(arguments); else - throw Exception(getName() + " only accepts maps", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Function {} only accepts one map or arrays, but got {}", + getName(), + arguments[0]->getName()); } // Struct holds input and output columns references, @@ -146,7 +151,8 @@ private: { const ColumnConst * const_array = checkAndGetColumnConst(key_column); if (!const_array) - throw Exception("Expected array column, found " + key_column->getName(), ErrorCodes::ILLEGAL_COLUMN); + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, "Expected array column in function {}, found {}", getName(), key_column->getName()); in_keys_array = checkAndGetColumn(const_array->getDataColumnPtr().get()); key_is_const = true; @@ -158,7 +164,8 @@ private: { const ColumnConst * const_array = checkAndGetColumnConst(val_column); if (!const_array) - throw Exception("Expected array column, found " + val_column->getName(), ErrorCodes::ILLEGAL_COLUMN); + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, "Expected array column in function {}, found {}", getName(), val_column->getName()); in_values_array = checkAndGetColumn(const_array->getDataColumnPtr().get()); val_is_const = true; @@ -166,7 +173,7 @@ private: if (!in_keys_array || !in_values_array) /* something went wrong */ - throw Exception("Illegal columns in arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal columns in arguments of function " + getName()); const auto & in_keys_data = assert_cast &>(in_keys_array->getData()).getData(); const auto & in_values_data = assert_cast &>(in_values_array->getData()).getData(); @@ -230,8 +237,8 @@ private: { MutableColumnPtr res_column = res_type->createColumn(); bool max_key_is_const = false; - auto inout = res_column->getDataType() == TypeIndex::Tuple ? getInOutDataFromArrays(res_column, arg_columns) - : getInOutDataFromMap(res_column, arg_columns); + auto columns = res_column->getDataType() == TypeIndex::Tuple ? getInOutDataFromArrays(res_column, arg_columns) + : getInOutDataFromMap(res_column, arg_columns); KeyType max_key_const{0}; @@ -246,36 +253,39 @@ private: std::map res_map; //Iterate through two arrays and fill result values. - for (size_t row = 0; row < inout.row_count; ++row) + for (size_t row = 0; row < columns.row_count; ++row) { - size_t key_offset = 0, val_offset = 0, items_count = inout.in_key_offsets[0], val_array_size = inout.in_val_offsets[0]; + size_t key_offset = 0, val_offset = 0, items_count = columns.in_key_offsets[0], val_array_size = columns.in_val_offsets[0]; res_map.clear(); - if (!inout.key_is_const) + if (!columns.key_is_const) { - key_offset = row > 0 ? inout.in_key_offsets[row - 1] : 0; - items_count = inout.in_key_offsets[row] - key_offset; + key_offset = row > 0 ? columns.in_key_offsets[row - 1] : 0; + items_count = columns.in_key_offsets[row] - key_offset; } - if (!inout.val_is_const) + if (!columns.val_is_const) { - val_offset = row > 0 ? inout.in_val_offsets[row - 1] : 0; - val_array_size = inout.in_val_offsets[row] - val_offset; + val_offset = row > 0 ? columns.in_val_offsets[row - 1] : 0; + val_array_size = columns.in_val_offsets[row] - val_offset; } if (items_count != val_array_size) - throw Exception("Key and value array should have same amount of elements", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Key and value array should have same amount of elements in function {}", + getName()); if (items_count == 0) { - inout.out_keys_offsets.push_back(offset); + columns.out_keys_offsets.push_back(offset); continue; } for (size_t i = 0; i < items_count; ++i) { - res_map.insert({inout.in_keys_data[key_offset + i], inout.in_vals_data[val_offset + i]}); + res_map.insert({columns.in_keys_data[key_offset + i], columns.in_vals_data[val_offset + i]}); } auto min_key = res_map.begin()->first; @@ -296,7 +306,7 @@ private: /* no need to add anything, max key is less that first key */ if (max_key < min_key) { - inout.out_keys_offsets.push_back(offset); + columns.out_keys_offsets.push_back(offset); continue; } } @@ -309,16 +319,16 @@ private: KeyType key; for (key = min_key;; ++key) { - inout.out_keys_data.push_back(key); + columns.out_keys_data.push_back(key); auto it = res_map.find(key); if (it != res_map.end()) { - inout.out_vals_data.push_back(it->second); + columns.out_vals_data.push_back(it->second); } else { - inout.out_vals_data.push_back(0); + columns.out_vals_data.push_back(0); } ++offset; @@ -326,11 +336,11 @@ private: break; } - inout.out_keys_offsets.push_back(offset); + columns.out_keys_offsets.push_back(offset); } - if (inout.out_vals_offsets) - inout.out_vals_offsets->insert(inout.out_keys_offsets.begin(), inout.out_keys_offsets.end()); + if (columns.out_vals_offsets) + columns.out_vals_offsets->insert(columns.out_keys_offsets.begin(), columns.out_keys_offsets.end()); return res_column; } @@ -365,7 +375,7 @@ private: case TypeIndex::UInt256: return execute2(arg_columns, max_key_column, res_type); default: - throw Exception("Illegal columns in arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal columns in arguments of function " + getName()); } } @@ -431,7 +441,7 @@ private: case TypeIndex::UInt256: return execute1(arg_columns, max_key_column, res_type, val_type); default: - throw Exception("Illegal columns in arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal columns in arguments of function " + getName()); } } }; diff --git a/tests/queries/0_stateless/01925_map_populate_series_on_map.reference b/tests/queries/0_stateless/01925_map_populate_series_on_map.reference index 74ed35f0638..235a227f548 100644 --- a/tests/queries/0_stateless/01925_map_populate_series_on_map.reference +++ b/tests/queries/0_stateless/01925_map_populate_series_on_map.reference @@ -1,37 +1,67 @@ +-- { echo } +drop table if exists map_test; +set allow_experimental_map_type = 1; +create table map_test engine=TinyLog() as (select (number + 1) as n, map(1, 1, number,2) as m from numbers(1, 5)); +select mapPopulateSeries(m) from map_test; {1:1} {1:1,2:2} {1:1,2:0,3:2} {1:1,2:0,3:0,4:2} {1:1,2:0,3:0,4:0,5:2} +select mapPopulateSeries(m, toUInt64(3)) from map_test; {1:1,2:0,3:0} {1:1,2:2,3:0} {1:1,2:0,3:2} {1:1,2:0,3:0} {1:1,2:0,3:0} +select mapPopulateSeries(m, toUInt64(10)) from map_test; {1:1,2:0,3:0,4:0,5:0,6:0,7:0,8:0,9:0,10:0} {1:1,2:2,3:0,4:0,5:0,6:0,7:0,8:0,9:0,10:0} {1:1,2:0,3:2,4:0,5:0,6:0,7:0,8:0,9:0,10:0} {1:1,2:0,3:0,4:2,5:0,6:0,7:0,8:0,9:0,10:0} {1:1,2:0,3:0,4:0,5:2,6:0,7:0,8:0,9:0,10:0} +select mapPopulateSeries(m, 1000) from map_test; -- { serverError 43 } +select mapPopulateSeries(m, n) from map_test; {1:1,2:0} {1:1,2:2,3:0} {1:1,2:0,3:2,4:0} {1:1,2:0,3:0,4:2,5:0} {1:1,2:0,3:0,4:0,5:2,6:0} +drop table map_test; +select mapPopulateSeries(map(toUInt8(1), toUInt8(1), 2, 1)) as res, toTypeName(res); {1:1,2:1} Map(UInt8,UInt8) +select mapPopulateSeries(map(toUInt16(1), toUInt16(1), 2, 1)) as res, toTypeName(res); {1:1,2:1} Map(UInt16,UInt16) +select mapPopulateSeries(map(toUInt32(1), toUInt32(1), 2, 1)) as res, toTypeName(res); {1:1,2:1} Map(UInt32,UInt32) +select mapPopulateSeries(map(toUInt64(1), toUInt64(1), 2, 1)) as res, toTypeName(res); {1:1,2:1} Map(UInt64,UInt64) +select mapPopulateSeries(map(toUInt128(1), toUInt128(1), 2, 1)) as res, toTypeName(res); {1:1,2:1} Map(UInt128,UInt128) +select mapPopulateSeries(map(toUInt256(1), toUInt256(1), 2, 1)) as res, toTypeName(res); {1:1,2:1} Map(UInt256,UInt256) +select mapPopulateSeries(map(toInt8(1), toInt8(1), 2, 1)) as res, toTypeName(res); {1:1,2:1} Map(Int16,Int16) +select mapPopulateSeries(map(toInt16(1), toInt16(1), 2, 1)) as res, toTypeName(res); {1:1,2:1} Map(Int16,Int16) +select mapPopulateSeries(map(toInt32(1), toInt32(1), 2, 1)) as res, toTypeName(res); {1:1,2:1} Map(Int32,Int32) +select mapPopulateSeries(map(toInt64(1), toInt64(1), 2, 1)) as res, toTypeName(res); {1:1,2:1} Map(Int64,Int64) +select mapPopulateSeries(map(toInt128(1), toInt128(1), 2, 1)) as res, toTypeName(res); {1:1,2:1} Map(Int128,Int128) +select mapPopulateSeries(map(toInt256(1), toInt256(1), 2, 1)) as res, toTypeName(res); {1:1,2:1} Map(Int256,Int256) +select mapPopulateSeries(map(toInt8(-10), toInt8(1), 2, 1)) as res, toTypeName(res); {-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int16,Int16) +select mapPopulateSeries(map(toInt16(-10), toInt16(1), 2, 1)) as res, toTypeName(res); {-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int16,Int16) +select mapPopulateSeries(map(toInt32(-10), toInt32(1), 2, 1)) as res, toTypeName(res); {-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int32,Int32) +select mapPopulateSeries(map(toInt64(-10), toInt64(1), 2, 1)) as res, toTypeName(res); {-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int64,Int64) +select mapPopulateSeries(map(toInt64(-10), toInt64(1), 2, 1), toInt64(-5)) as res, toTypeName(res); {-10:1,-9:0,-8:0,-7:0,-6:0,-5:0} Map(Int64,Int64) +select mapPopulateSeries(); -- { serverError 42 } +select mapPopulateSeries('asdf'); -- { serverError 43 } +select mapPopulateSeries(map('1', 1, '2', 1)) as res, toTypeName(res); -- { serverError 43 } diff --git a/tests/queries/0_stateless/01925_map_populate_series_on_map.sql b/tests/queries/0_stateless/01925_map_populate_series_on_map.sql index f45543a8dba..ac78280ec1d 100644 --- a/tests/queries/0_stateless/01925_map_populate_series_on_map.sql +++ b/tests/queries/0_stateless/01925_map_populate_series_on_map.sql @@ -1,3 +1,4 @@ +-- { echo } drop table if exists map_test; set allow_experimental_map_type = 1; create table map_test engine=TinyLog() as (select (number + 1) as n, map(1, 1, number,2) as m from numbers(1, 5)); diff --git a/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py b/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py index 938beabfff4..04150d945f6 100644 --- a/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py +++ b/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py @@ -357,7 +357,7 @@ def map_func(self, data_type, node=None): exitcode, message = 0, None if data_type.startswith("Decimal"): - exitcode, message = 43, "Exception:" + exitcode, message = 43, "Exception:" node.query(sql, exitcode=exitcode, message=message) execute_query(f"""SELECT * FROM {table_name} ORDER BY a ASC""") @@ -393,9 +393,13 @@ def map_func(self, data_type, node=None): execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") with Scenario(f"mapPopulateSeries with {data_type}"): - node.query(f"SELECT mapPopulateSeries([1,2,3], [{to_data_type(data_type,1)}," - f"{to_data_type(data_type,2)}, {to_data_type(data_type,3)}], 5)", - exitcode = 44, message='Exception:') + sql = (f"SELECT mapPopulateSeries([1,2,3], [{to_data_type(data_type,1)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,3)}], 5)") + + exitcode, message = 0, None + if data_type.startswith("Decimal"): + exitcode, message = 44, "Exception:" + node.query(sql, exitcode=exitcode, message=message) with Scenario(f"mapPopulateSeries with {data_type} on a table"): table_name = get_table_name() @@ -403,9 +407,13 @@ def map_func(self, data_type, node=None): table(name = table_name, data_type = f'Tuple(Array({data_type}), Array({data_type}))') with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT mapPopulateSeries([1,2,3]," - f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}, {to_data_type(data_type,3)}], 5)", - exitcode = 44, message='Exception:') + sql = (f"INSERT INTO {table_name} SELECT mapPopulateSeries([1,2,3],", + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}, {to_data_type(data_type,3)}], 5)") + + exitcode, message = 0, None + if data_type.startswith("Decimal"): + exitcode, message = 44, "Exception:" + node.query(sql, exitcode=exitcode, message=message) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") From 66631ca6801528f947a809a832fb9706a3767d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=87=E5=BA=B7?= Date: Sun, 1 Aug 2021 07:51:24 +0800 Subject: [PATCH 047/198] add column default_database --- docs/en/operations/settings/settings-users.md | 2 +- docs/en/sql-reference/statements/create/user.md | 1 + src/Storages/System/StorageSystemUsers.cpp | 10 ++++++++-- ...2001_add_default_database_to_system_users.reference | 1 + .../02001_add_default_database_to_system_users.sql | 3 +++ 5 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/02001_add_default_database_to_system_users.reference create mode 100644 tests/queries/0_stateless/02001_add_default_database_to_system_users.sql diff --git a/docs/en/operations/settings/settings-users.md b/docs/en/operations/settings/settings-users.md index ee834dca98a..2c8315ad069 100644 --- a/docs/en/operations/settings/settings-users.md +++ b/docs/en/operations/settings/settings-users.md @@ -28,7 +28,7 @@ Structure of the `users` section: profile_name default - + default diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index ea275b9a2f8..dfa065f5d0a 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -15,6 +15,7 @@ CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1] [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']}] [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [DEFAULT ROLE role [,...]] + [DEFAULT DATABASE database | NONE] [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...] ``` diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index 90b0a914d58..a48e12a1476 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -50,6 +50,7 @@ NamesAndTypesList StorageSystemUsers::getNamesAndTypes() {"grantees_any", std::make_shared()}, {"grantees_list", std::make_shared(std::make_shared())}, {"grantees_except", std::make_shared(std::make_shared())}, + {"default_database", std::make_shared()}, }; return names_and_types; } @@ -85,6 +86,7 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte auto & column_grantees_list_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_grantees_except = assert_cast(assert_cast(*res_columns[column_index]).getData()); auto & column_grantees_except_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_default_database = assert_cast(*res_columns[column_index++]); auto add_row = [&](const String & name, const UUID & id, @@ -92,7 +94,8 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte const Authentication & authentication, const AllowedClientHosts & allowed_hosts, const RolesOrUsersSet & default_roles, - const RolesOrUsersSet & grantees) + const RolesOrUsersSet & grantees, + const String default_database) { column_name.insertData(name.data(), name.length()); column_id.push_back(id.toUnderType()); @@ -180,6 +183,8 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte for (const auto & except_name : grantees_ast->except_names) column_grantees_except.insertData(except_name.data(), except_name.length()); column_grantees_except_offsets.push_back(column_grantees_except.size()); + + column_default_database.insertData(default_database.data(),default_database.length()); }; for (const auto & id : ids) @@ -192,7 +197,8 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte if (!storage) continue; - add_row(user->getName(), id, storage->getStorageName(), user->authentication, user->allowed_client_hosts, user->default_roles, user->grantees); + add_row(user->getName(), id, storage->getStorageName(), user->authentication, user->allowed_client_hosts, + user->default_roles, user->grantees, user->default_database); } } diff --git a/tests/queries/0_stateless/02001_add_default_database_to_system_users.reference b/tests/queries/0_stateless/02001_add_default_database_to_system_users.reference new file mode 100644 index 00000000000..bec3a35ee8b --- /dev/null +++ b/tests/queries/0_stateless/02001_add_default_database_to_system_users.reference @@ -0,0 +1 @@ +system diff --git a/tests/queries/0_stateless/02001_add_default_database_to_system_users.sql b/tests/queries/0_stateless/02001_add_default_database_to_system_users.sql new file mode 100644 index 00000000000..2952f3fd23e --- /dev/null +++ b/tests/queries/0_stateless/02001_add_default_database_to_system_users.sql @@ -0,0 +1,3 @@ +create user if not exists u_02001 default database system; +select default_database from system.users where name = 'u_02001'; +drop user if exists u_02001; \ No newline at end of file From 80f33244d4011ed9b6588de91cee202d063b8bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=87=E5=BA=B7?= Date: Sun, 1 Aug 2021 08:12:37 +0800 Subject: [PATCH 048/198] add column default_database --- .../0_stateless/02001_add_default_database_to_system_users.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02001_add_default_database_to_system_users.sql b/tests/queries/0_stateless/02001_add_default_database_to_system_users.sql index 2952f3fd23e..b006f9acb22 100644 --- a/tests/queries/0_stateless/02001_add_default_database_to_system_users.sql +++ b/tests/queries/0_stateless/02001_add_default_database_to_system_users.sql @@ -1,3 +1,3 @@ create user if not exists u_02001 default database system; select default_database from system.users where name = 'u_02001'; -drop user if exists u_02001; \ No newline at end of file +drop user if exists u_02001; From 130253e3b9b0b28bfdbc5d2c076b2c381f1d3c7b Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 30 Jul 2021 14:20:57 +0000 Subject: [PATCH 049/198] Fix bridge-server interaction in case of metadata inconsistency --- programs/library-bridge/HandlerFactory.cpp | 4 +- programs/library-bridge/Handlers.cpp | 146 +++++++++++++----- programs/library-bridge/Handlers.h | 15 +- .../SharedLibraryHandlerFactory.cpp | 20 +-- .../SharedLibraryHandlerFactory.h | 4 +- src/Bridge/IBridgeHelper.cpp | 15 -- src/Bridge/IBridgeHelper.h | 8 +- src/Bridge/LibraryBridgeHelper.cpp | 112 ++++++++++++-- src/Bridge/LibraryBridgeHelper.h | 25 ++- src/Bridge/XDBCBridgeHelper.h | 37 +++-- src/Dictionaries/LibraryDictionarySource.cpp | 27 +++- tests/integration/test_library_bridge/test.py | 65 +++++++- 12 files changed, 358 insertions(+), 120 deletions(-) diff --git a/programs/library-bridge/HandlerFactory.cpp b/programs/library-bridge/HandlerFactory.cpp index 9f53a24156f..3e31f298fca 100644 --- a/programs/library-bridge/HandlerFactory.cpp +++ b/programs/library-bridge/HandlerFactory.cpp @@ -12,8 +12,8 @@ namespace DB Poco::URI uri{request.getURI()}; LOG_DEBUG(log, "Request URI: {}", uri.toString()); - if (uri == "/ping" && request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET) - return std::make_unique(keep_alive_timeout); + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET) + return std::make_unique(keep_alive_timeout, getContext()); if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) return std::make_unique(keep_alive_timeout, getContext()); diff --git a/programs/library-bridge/Handlers.cpp b/programs/library-bridge/Handlers.cpp index 89219d1b910..7e79830a7b2 100644 --- a/programs/library-bridge/Handlers.cpp +++ b/programs/library-bridge/Handlers.cpp @@ -19,6 +19,16 @@ namespace DB { namespace { + void processError(HTTPServerResponse & response, const std::string & message) + { + response.setStatusAndReason(HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); + + if (!response.sent()) + *response.send() << message << std::endl; + + LOG_WARNING(&Poco::Logger::get("LibraryBridge"), message); + } + std::shared_ptr parseColumns(std::string && column_string) { auto sample_block = std::make_shared(); @@ -30,13 +40,13 @@ namespace return sample_block; } - std::vector parseIdsFromBinary(const std::string & ids_string) - { - ReadBufferFromString buf(ids_string); - std::vector ids; - readVectorBinary(ids, buf); - return ids; - } + // std::vector parseIdsFromBinary(const std::string & ids_string) + // { + // ReadBufferFromString buf(ids_string); + // std::vector ids; + // readVectorBinary(ids, buf); + // return ids; + // } std::vector parseNamesFromBinary(const std::string & names_string) { @@ -67,13 +77,36 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe std::string method = params.get("method"); std::string dictionary_id = params.get("dictionary_id"); - LOG_TRACE(log, "Library method: '{}', dictionary id: {}", method, dictionary_id); + LOG_TRACE(log, "Library method: '{}', dictionary id: {}", method, dictionary_id); WriteBufferFromHTTPServerResponse out(response, request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, keep_alive_timeout); try { - if (method == "libNew") + bool lib_new = (method == "libNew"); + if (method == "libClone") + { + if (!params.has("from_dictionary_id")) + { + processError(response, "No 'from_dictionary_id' in request URL"); + return; + } + + std::string from_dictionary_id = params.get("from_dictionary_id"); + bool cloned = false; + cloned = SharedLibraryHandlerFactory::instance().clone(from_dictionary_id, dictionary_id); + + if (cloned) + { + writeStringBinary("1", out); + } + else + { + LOG_TRACE(log, "Cannot clone from dictionary with id: {}, will call libNew instead"); + lib_new = true; + } + } + if (lib_new) { auto & read_buf = request.getStream(); params.read(read_buf); @@ -92,6 +125,8 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe std::string library_path = params.get("library_path"); const auto & settings_string = params.get("library_settings"); + + LOG_DEBUG(log, "Parsing library settings from binary string"); std::vector library_settings = parseNamesFromBinary(settings_string); /// Needed for library dictionary @@ -102,6 +137,8 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe } const auto & attributes_string = params.get("attributes_names"); + + LOG_DEBUG(log, "Parsing attributes names from binary string"); std::vector attributes_names = parseNamesFromBinary(attributes_string); /// Needed to parse block from binary string format @@ -140,54 +177,71 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe SharedLibraryHandlerFactory::instance().create(dictionary_id, library_path, library_settings, sample_block_with_nulls, attributes_names); writeStringBinary("1", out); } - else if (method == "libClone") - { - if (!params.has("from_dictionary_id")) - { - processError(response, "No 'from_dictionary_id' in request URL"); - return; - } - - std::string from_dictionary_id = params.get("from_dictionary_id"); - LOG_TRACE(log, "Calling libClone from {} to {}", from_dictionary_id, dictionary_id); - SharedLibraryHandlerFactory::instance().clone(from_dictionary_id, dictionary_id); - writeStringBinary("1", out); - } else if (method == "libDelete") { - SharedLibraryHandlerFactory::instance().remove(dictionary_id); + auto deleted = SharedLibraryHandlerFactory::instance().remove(dictionary_id); + + /// Do not throw, a warning is ok. + if (!deleted) + LOG_WARNING(log, "Cannot delete library for with dictionary id: {}, because such id was not found.", dictionary_id); + writeStringBinary("1", out); } else if (method == "isModified") { auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); + if (!library_handler) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found dictionary with id: {}", dictionary_id); + bool res = library_handler->isModified(); writeStringBinary(std::to_string(res), out); } else if (method == "supportsSelectiveLoad") { auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); + if (!library_handler) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found dictionary with id: {}", dictionary_id); + bool res = library_handler->supportsSelectiveLoad(); writeStringBinary(std::to_string(res), out); } else if (method == "loadAll") { auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); + if (!library_handler) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found dictionary with id: {}", dictionary_id); + const auto & sample_block = library_handler->getSampleBlock(); auto input = library_handler->loadAll(); + LOG_DEBUG(log, "Started sending result data for dictionary id: {}", dictionary_id); BlockOutputStreamPtr output = FormatFactory::instance().getOutputStream(FORMAT, out, sample_block, getContext()); copyData(*input, *output); } else if (method == "loadIds") { + LOG_DEBUG(log, "Getting diciontary ids for dictionary with id: {}", dictionary_id); String ids_string; readString(ids_string, request.getStream()); - std::vector ids = parseIdsFromBinary(ids_string); + + Strings ids_strs; + splitInto<'-'>(ids_strs, ids_string); + std::vector ids; + for (const auto & id : ids_strs) + ids.push_back(parse(id)); + if (ids.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Received no ids"); + + // std::vector ids = parseIdsFromBinary(ids_string); auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); + if (!library_handler) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found dictionary with id: {}", dictionary_id); + const auto & sample_block = library_handler->getSampleBlock(); auto input = library_handler->loadIds(ids); + + LOG_DEBUG(log, "Started sending result data for dictionary id: {}", dictionary_id); BlockOutputStreamPtr output = FormatFactory::instance().getOutputStream(FORMAT, out, sample_block, getContext()); copyData(*input, *output); } @@ -219,8 +273,13 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe auto block = reader->read(); auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); + if (!library_handler) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found dictionary with id: {}", dictionary_id); + const auto & sample_block = library_handler->getSampleBlock(); auto input = library_handler->loadKeys(block.getColumns()); + + LOG_DEBUG(log, "Started sending result data for dictionary id: {}", dictionary_id); BlockOutputStreamPtr output = FormatFactory::instance().getOutputStream(FORMAT, out, sample_block, getContext()); copyData(*input, *output); } @@ -228,8 +287,9 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe catch (...) { auto message = getCurrentExceptionMessage(true); - response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR, message); // can't call process_error, because of too soon response sending + LOG_ERROR(log, "Failed to process request for dictionary_id: {}. Error: {}", dictionary_id, message); + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR, message); // can't call process_error, because of too soon response sending try { writeStringBinary(message, out); @@ -239,8 +299,6 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe { tryLogCurrentException(log); } - - tryLogCurrentException(log); } try @@ -254,24 +312,30 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe } -void LibraryRequestHandler::processError(HTTPServerResponse & response, const std::string & message) -{ - response.setStatusAndReason(HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); - - if (!response.sent()) - *response.send() << message << std::endl; - - LOG_WARNING(log, message); -} - - -void PingHandler::handleRequest(HTTPServerRequest & /* request */, HTTPServerResponse & response) +void PingHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { try { + LOG_TRACE(log, "Request URI: {}", request.getURI()); + HTMLForm params(getContext()->getSettingsRef(), request); + + if (!params.has("dictionary_id")) + { + processError(response, "No 'dictionary_id in request URL"); + return; + } + + std::string dictionary_id = params.get("dictionary_id"); + auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); + String res; + if (library_handler) + res = "dictionary=1"; + else + res = "dictionary=0"; + setResponseDefaultHeaders(response, keep_alive_timeout); - const char * data = "Ok.\n"; - response.sendBuffer(data, strlen(data)); + LOG_TRACE(log, "Senging ping response: {} (dictionary id: {})", res, dictionary_id); + response.sendBuffer(res.data(), res.size()); } catch (...) { diff --git a/programs/library-bridge/Handlers.h b/programs/library-bridge/Handlers.h index dac61d3a735..a8e4ba0dd2a 100644 --- a/programs/library-bridge/Handlers.h +++ b/programs/library-bridge/Handlers.h @@ -22,8 +22,7 @@ class LibraryRequestHandler : public HTTPRequestHandler, WithContext public: LibraryRequestHandler( - size_t keep_alive_timeout_, - ContextPtr context_) + size_t keep_alive_timeout_, ContextPtr context_) : WithContext(context_) , log(&Poco::Logger::get("LibraryRequestHandler")) , keep_alive_timeout(keep_alive_timeout_) @@ -35,18 +34,18 @@ public: private: static constexpr inline auto FORMAT = "RowBinary"; - void processError(HTTPServerResponse & response, const std::string & message); - Poco::Logger * log; size_t keep_alive_timeout; }; -class PingHandler : public HTTPRequestHandler +class PingHandler : public HTTPRequestHandler, WithContext { public: - explicit PingHandler(size_t keep_alive_timeout_) - : keep_alive_timeout(keep_alive_timeout_) + explicit PingHandler(size_t keep_alive_timeout_, ContextPtr context_) + : WithContext(context_) + , keep_alive_timeout(keep_alive_timeout_) + , log(&Poco::Logger::get("LibraryRequestHandler")) { } @@ -54,6 +53,8 @@ public: private: const size_t keep_alive_timeout; + Poco::Logger * log; + }; } diff --git a/programs/library-bridge/SharedLibraryHandlerFactory.cpp b/programs/library-bridge/SharedLibraryHandlerFactory.cpp index 270e07a1046..900981191c8 100644 --- a/programs/library-bridge/SharedLibraryHandlerFactory.cpp +++ b/programs/library-bridge/SharedLibraryHandlerFactory.cpp @@ -18,7 +18,7 @@ SharedLibraryHandlerPtr SharedLibraryHandlerFactory::get(const std::string & dic if (library_handler != library_handlers.end()) return library_handler->second; - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not found dictionary with id: {}", dictionary_id); + return nullptr; } @@ -30,32 +30,32 @@ void SharedLibraryHandlerFactory::create( const std::vector & attributes_names) { std::lock_guard lock(mutex); - library_handlers[dictionary_id] = std::make_shared(library_path, library_settings, sample_block, attributes_names); + if (!library_handlers.count(dictionary_id)) + library_handlers.emplace(std::make_pair(dictionary_id, std::make_shared(library_path, library_settings, sample_block, attributes_names))); + else + LOG_WARNING(&Poco::Logger::get("SharedLibraryHandlerFactory"), "Library handler with dictionary id {} already exists", dictionary_id); } -void SharedLibraryHandlerFactory::clone(const std::string & from_dictionary_id, const std::string & to_dictionary_id) +bool SharedLibraryHandlerFactory::clone(const std::string & from_dictionary_id, const std::string & to_dictionary_id) { std::lock_guard lock(mutex); auto from_library_handler = library_handlers.find(from_dictionary_id); - /// This is not supposed to happen as libClone is called from copy constructor of LibraryDictionarySource - /// object, and shared library handler of from_dictionary is removed only in its destructor. - /// And if for from_dictionary there was no shared library handler, it would have received and exception in - /// its constructor, so no libClone would be made from it. if (from_library_handler == library_handlers.end()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "No shared library handler found"); + return false; /// libClone method will be called in copy constructor library_handlers[to_dictionary_id] = std::make_shared(*from_library_handler->second); + return true; } -void SharedLibraryHandlerFactory::remove(const std::string & dictionary_id) +bool SharedLibraryHandlerFactory::remove(const std::string & dictionary_id) { std::lock_guard lock(mutex); /// libDelete is called in destructor. - library_handlers.erase(dictionary_id); + return library_handlers.erase(dictionary_id); } diff --git a/programs/library-bridge/SharedLibraryHandlerFactory.h b/programs/library-bridge/SharedLibraryHandlerFactory.h index 473d90618a2..115cc78ae52 100644 --- a/programs/library-bridge/SharedLibraryHandlerFactory.h +++ b/programs/library-bridge/SharedLibraryHandlerFactory.h @@ -24,9 +24,9 @@ public: const Block & sample_block, const std::vector & attributes_names); - void clone(const std::string & from_dictionary_id, const std::string & to_dictionary_id); + bool clone(const std::string & from_dictionary_id, const std::string & to_dictionary_id); - void remove(const std::string & dictionary_id); + bool remove(const std::string & dictionary_id); private: /// map: dict_id -> sharedLibraryHandler diff --git a/src/Bridge/IBridgeHelper.cpp b/src/Bridge/IBridgeHelper.cpp index b6f3446d0a6..a90688cc8b5 100644 --- a/src/Bridge/IBridgeHelper.cpp +++ b/src/Bridge/IBridgeHelper.cpp @@ -33,21 +33,6 @@ Poco::URI IBridgeHelper::getPingURI() const } -bool IBridgeHelper::checkBridgeIsRunning() const -{ - try - { - ReadWriteBufferFromHTTP buf( - getPingURI(), Poco::Net::HTTPRequest::HTTP_GET, {}, ConnectionTimeouts::getHTTPTimeouts(getContext())); - return checkString(PING_OK_ANSWER, buf); - } - catch (...) - { - return false; - } -} - - void IBridgeHelper::startBridgeSync() const { if (!checkBridgeIsRunning()) diff --git a/src/Bridge/IBridgeHelper.h b/src/Bridge/IBridgeHelper.h index caaf031b7d8..6a9c19da68f 100644 --- a/src/Bridge/IBridgeHelper.h +++ b/src/Bridge/IBridgeHelper.h @@ -28,16 +28,18 @@ public: static const inline std::string MAIN_METHOD = Poco::Net::HTTPRequest::HTTP_POST; explicit IBridgeHelper(ContextPtr context_) : WithContext(context_) {} - virtual ~IBridgeHelper() = default; - void startBridgeSync() const; + virtual ~IBridgeHelper() = default; Poco::URI getMainURI() const; Poco::URI getPingURI() const; + void startBridgeSync() const; protected: + virtual bool checkBridgeIsRunning() const = 0; + /// clickhouse-odbc-bridge, clickhouse-library-bridge virtual String serviceAlias() const = 0; @@ -61,8 +63,6 @@ protected: private: - bool checkBridgeIsRunning() const; - std::unique_ptr startBridgeCommand() const; }; diff --git a/src/Bridge/LibraryBridgeHelper.cpp b/src/Bridge/LibraryBridgeHelper.cpp index b13be0aba29..6f976200153 100644 --- a/src/Bridge/LibraryBridgeHelper.cpp +++ b/src/Bridge/LibraryBridgeHelper.cpp @@ -23,12 +23,14 @@ namespace DB LibraryBridgeHelper::LibraryBridgeHelper( ContextPtr context_, const Block & sample_block_, - const Field & dictionary_id_) + const Field & dictionary_id_, + const LibraryInitData & library_data_) : IBridgeHelper(context_->getGlobalContext()) , log(&Poco::Logger::get("LibraryBridgeHelper")) , sample_block(sample_block_) , config(context_->getConfigRef()) , http_timeout(context_->getGlobalContext()->getSettingsRef().http_receive_timeout.value) + , library_data(library_data_) , dictionary_id(dictionary_id_) { bridge_port = config.getUInt("library_bridge.port", DEFAULT_PORT); @@ -61,26 +63,85 @@ void LibraryBridgeHelper::startBridge(std::unique_ptr cmd) const } -bool LibraryBridgeHelper::initLibrary(const std::string & library_path, const std::string library_settings, const std::string attributes_names) +bool LibraryBridgeHelper::checkBridgeIsRunning() const { - startBridgeSync(); - auto uri = createRequestURI(LIB_NEW_METHOD); + String result; + try + { + ReadWriteBufferFromHTTP buf(createRequestURI(PING), Poco::Net::HTTPRequest::HTTP_GET, {}, ConnectionTimeouts::getHTTPTimeouts(getContext())); + readString(result, buf); + } + catch (...) + { + return false; + } + /* + * When pinging bridge we also pass current dicionary_id. The bridge will check if there is such + * dictionary. It is possible that such dictionary_id is not present only in two cases: + * 1. It is dictionary source creation and initialization of library handler on bridge side did not happen yet. + * 2. Bridge crashed or restarted for some reason while server did not. + **/ + static constexpr auto dictionary_check = "dictionary="; + if (result.size() != (std::strlen(dictionary_check) + 1)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {}. Check bridge and server have the same version.", + result, std::strlen(dictionary_check)); + + UInt8 dictionary_id_exists; + auto parsed = tryParse(dictionary_id_exists, result.substr(std::strlen(dictionary_check))); + if (!parsed || (dictionary_id_exists != 0 && dictionary_id_exists != 1)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {} ({}). Check bridge and server have the same version.", + result, parsed ? toString(dictionary_id_exists) : "failed to parse"); + + if (dictionary_id_exists && !library_initialized) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Library was not initialized, but bridge responded to already have dictionary id: {}", dictionary_id); + + if (!dictionary_id_exists && library_initialized) + { + LOG_WARNING(log, "Library bridge does not have library handler with dictionaty id: {}. It will be reinitialized.", dictionary_id); + try + { + if (!initLibrary(false)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Failed to reinitialize library handler on bridge side for dictionary with id: {}", dictionary_id); + } + catch (...) + { + tryLogCurrentException(log); + return false; + } + } + + return true; +} + + +ReadWriteBufferFromHTTP::OutStreamCallback LibraryBridgeHelper::getInitLibraryCallback() const +{ /// Sample block must contain null values WriteBufferFromOwnString out; auto output_stream = getContext()->getOutputStream(LibraryBridgeHelper::DEFAULT_FORMAT, out, sample_block); formatBlock(output_stream, sample_block); auto block_string = out.str(); - auto out_stream_callback = [library_path, library_settings, attributes_names, block_string, this](std::ostream & os) + return [block_string, this](std::ostream & os) { - os << "library_path=" << escapeForFileName(library_path) << "&"; - os << "library_settings=" << escapeForFileName(library_settings) << "&"; - os << "attributes_names=" << escapeForFileName(attributes_names) << "&"; + os << "library_path=" << escapeForFileName(library_data.library_path) << "&"; + os << "library_settings=" << escapeForFileName(library_data.library_settings) << "&"; + os << "attributes_names=" << escapeForFileName(library_data.dict_attributes) << "&"; os << "sample_block=" << escapeForFileName(sample_block.getNamesAndTypesList().toString()) << "&"; os << "null_values=" << escapeForFileName(block_string); }; - return executeRequest(uri, out_stream_callback); +} + + +bool LibraryBridgeHelper::initLibrary(bool check_bridge) const +{ + /// Do not check if we call initLibrary from checkBridgeSync. + if (check_bridge) + startBridgeSync(); + auto uri = createRequestURI(LIB_NEW_METHOD); + return executeRequest(uri, getInitLibraryCallback()); } @@ -89,15 +150,20 @@ bool LibraryBridgeHelper::cloneLibrary(const Field & other_dictionary_id) startBridgeSync(); auto uri = createRequestURI(LIB_CLONE_METHOD); uri.addQueryParameter("from_dictionary_id", toString(other_dictionary_id)); - return executeRequest(uri); + return executeRequest(uri, getInitLibraryCallback()); } bool LibraryBridgeHelper::removeLibrary() { - startBridgeSync(); - auto uri = createRequestURI(LIB_DELETE_METHOD); - return executeRequest(uri); + /// Do not force bridge restart if it is not running in case of removeLibrary + /// because in this case after restart it will not have this dictionaty id in memory anyway. + if (checkBridgeIsRunning()) + { + auto uri = createRequestURI(LIB_DELETE_METHOD); + return executeRequest(uri); + } + return true; } @@ -125,11 +191,25 @@ BlockInputStreamPtr LibraryBridgeHelper::loadAll() } -BlockInputStreamPtr LibraryBridgeHelper::loadIds(const std::string ids_string) +BlockInputStreamPtr LibraryBridgeHelper::loadIds(const std::string, const std::vector ids) { startBridgeSync(); auto uri = createRequestURI(LOAD_IDS_METHOD); - return loadBase(uri, [ids_string](std::ostream & os) { os << ids_string; }); + + uri.addQueryParameter("ids_num", toString(ids.size())); + String ids_str; + for (const auto & id : ids) + { + if (!ids_str.empty()) + ids_str += '-'; + ids_str += toString(id); + } + + uri.addQueryParameter("ids", ids_str); + std::cerr << "\n\nLibraryBridgeHelper: " << toString(dictionary_id) << " , ids_num: " << ids.size() << " , ids: " << ids_str << std::endl << std::endl; + LOG_ERROR(log, "dictionary_id: {}, ids_num: {}, ids: {}", dictionary_id, ids.size(), ids_str); + + return loadBase(uri, [ids_str](std::ostream & os) { os << ids_str; }); } @@ -149,7 +229,7 @@ BlockInputStreamPtr LibraryBridgeHelper::loadKeys(const Block & requested_block) } -bool LibraryBridgeHelper::executeRequest(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback) +bool LibraryBridgeHelper::executeRequest(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback) const { ReadWriteBufferFromHTTP buf( uri, diff --git a/src/Bridge/LibraryBridgeHelper.h b/src/Bridge/LibraryBridgeHelper.h index 12fe0c33363..815fce72b33 100644 --- a/src/Bridge/LibraryBridgeHelper.h +++ b/src/Bridge/LibraryBridgeHelper.h @@ -15,11 +15,18 @@ class LibraryBridgeHelper : public IBridgeHelper { public: + struct LibraryInitData + { + String library_path; + String library_settings; + String dict_attributes; + }; + static constexpr inline size_t DEFAULT_PORT = 9012; - LibraryBridgeHelper(ContextPtr context_, const Block & sample_block, const Field & dictionary_id_); + LibraryBridgeHelper(ContextPtr context_, const Block & sample_block, const Field & dictionary_id_, const LibraryInitData & library_data_); - bool initLibrary(const std::string & library_path, std::string library_settings, std::string attributes_names); + bool initLibrary(bool check_bridge = true) const; bool cloneLibrary(const Field & other_dictionary_id); @@ -31,16 +38,21 @@ public: BlockInputStreamPtr loadAll(); - BlockInputStreamPtr loadIds(std::string ids_string); + BlockInputStreamPtr loadIds(std::string ids_string, const std::vector ids); BlockInputStreamPtr loadKeys(const Block & requested_block); BlockInputStreamPtr loadBase(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = {}); - bool executeRequest(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = {}); + bool executeRequest(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = {}) const; + LibraryInitData getLibraryData() const { return library_data; } + + void setInitialized() { library_initialized = true; } protected: + bool checkBridgeIsRunning() const override; + void startBridge(std::unique_ptr cmd) const override; String serviceAlias() const override { return "clickhouse-library-bridge"; } @@ -61,6 +73,8 @@ protected: Poco::URI createBaseURI() const override; + ReadWriteBufferFromHTTP::OutStreamCallback getInitLibraryCallback() const; + private: static constexpr inline auto LIB_NEW_METHOD = "libNew"; static constexpr inline auto LIB_CLONE_METHOD = "libClone"; @@ -69,6 +83,7 @@ private: static constexpr inline auto LOAD_IDS_METHOD = "loadIds"; static constexpr inline auto LOAD_KEYS_METHOD = "loadKeys"; static constexpr inline auto IS_MODIFIED_METHOD = "isModified"; + static constexpr inline auto PING = "ping"; static constexpr inline auto SUPPORTS_SELECTIVE_LOAD_METHOD = "supportsSelectiveLoad"; Poco::URI createRequestURI(const String & method) const; @@ -78,9 +93,11 @@ private: const Poco::Util::AbstractConfiguration & config; const Poco::Timespan http_timeout; + LibraryInitData library_data; Field dictionary_id; std::string bridge_host; size_t bridge_port; + bool library_initialized = false; }; } diff --git a/src/Bridge/XDBCBridgeHelper.h b/src/Bridge/XDBCBridgeHelper.h index 8be4c194962..13f0f528c8f 100644 --- a/src/Bridge/XDBCBridgeHelper.h +++ b/src/Bridge/XDBCBridgeHelper.h @@ -60,20 +60,33 @@ public: static constexpr inline auto SCHEMA_ALLOWED_HANDLER = "/schema_allowed"; XDBCBridgeHelper( - ContextPtr context_, - Poco::Timespan http_timeout_, - const std::string & connection_string_) - : IXDBCBridgeHelper(context_->getGlobalContext()) - , log(&Poco::Logger::get(BridgeHelperMixin::getName() + "BridgeHelper")) - , connection_string(connection_string_) - , http_timeout(http_timeout_) - , config(context_->getGlobalContext()->getConfigRef()) -{ - bridge_host = config.getString(BridgeHelperMixin::configPrefix() + ".host", DEFAULT_HOST); - bridge_port = config.getUInt(BridgeHelperMixin::configPrefix() + ".port", DEFAULT_PORT); -} + ContextPtr context_, + Poco::Timespan http_timeout_, + const std::string & connection_string_) + : IXDBCBridgeHelper(context_->getGlobalContext()) + , log(&Poco::Logger::get(BridgeHelperMixin::getName() + "BridgeHelper")) + , connection_string(connection_string_) + , http_timeout(http_timeout_) + , config(context_->getGlobalContext()->getConfigRef()) + { + bridge_host = config.getString(BridgeHelperMixin::configPrefix() + ".host", DEFAULT_HOST); + bridge_port = config.getUInt(BridgeHelperMixin::configPrefix() + ".port", DEFAULT_PORT); + } protected: + bool checkBridgeIsRunning() const override + { + try + { + ReadWriteBufferFromHTTP buf(getPingURI(), Poco::Net::HTTPRequest::HTTP_GET, {}, ConnectionTimeouts::getHTTPTimeouts(getContext())); + return checkString(PING_OK_ANSWER, buf); + } + catch (...) + { + return false; + } + } + auto getConnectionString() const { return connection_string; } String getName() const override { return BridgeHelperMixin::getName(); } diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index 0b8b52a2d67..d5a63735643 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -48,17 +48,34 @@ LibraryDictionarySource::LibraryDictionarySource( throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "LibraryDictionarySource: Can't load library {}: file doesn't exist", path); description.init(sample_block); - bridge_helper = std::make_shared(context, description.sample_block, dictionary_id); - auto res = bridge_helper->initLibrary(path, getLibrarySettingsString(config, config_prefix + ".settings"), getDictAttributesString()); + + LibraryBridgeHelper::LibraryInitData library_data + { + .library_path = path, + .library_settings = getLibrarySettingsString(config, config_prefix + ".settings"), + .dict_attributes = getDictAttributesString() + }; + bridge_helper = std::make_shared(context, description.sample_block, dictionary_id, library_data); + auto res = bridge_helper->initLibrary(); if (!res) throw Exception(ErrorCodes::EXTERNAL_LIBRARY_ERROR, "Failed to create shared library from path: {}", path); + else + bridge_helper->setInitialized(); } LibraryDictionarySource::~LibraryDictionarySource() { - bridge_helper->removeLibrary(); + try + { + bridge_helper->removeLibrary(); + } + catch (...) + { + tryLogCurrentException("LibraryDictionarySource"); + } + } @@ -72,7 +89,7 @@ LibraryDictionarySource::LibraryDictionarySource(const LibraryDictionarySource & , context(other.context) , description{other.description} { - bridge_helper = std::make_shared(context, description.sample_block, dictionary_id); + bridge_helper = std::make_shared(context, description.sample_block, dictionary_id, other.bridge_helper->getLibraryData()); bridge_helper->cloneLibrary(other.dictionary_id); } @@ -99,7 +116,7 @@ BlockInputStreamPtr LibraryDictionarySource::loadAll() BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector & ids) { LOG_TRACE(log, "loadIds {} size = {}", toString(), ids.size()); - return bridge_helper->loadIds(getDictIdsString(ids)); + return bridge_helper->loadIds(getDictIdsString(ids), ids); } diff --git a/tests/integration/test_library_bridge/test.py b/tests/integration/test_library_bridge/test.py index ba44918bd60..dc15f6a8430 100644 --- a/tests/integration/test_library_bridge/test.py +++ b/tests/integration/test_library_bridge/test.py @@ -8,8 +8,23 @@ from helpers.cluster import ClickHouseCluster, run_and_check cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance', - dictionaries=['configs/dictionaries/dict1.xml'], - main_configs=['configs/config.d/config.xml']) + dictionaries=['configs/dictionaries/dict1.xml'], main_configs=['configs/config.d/config.xml'], stay_alive=True) + + +def create_dict_simple(): + instance.query('DROP DICTIONARY IF EXISTS lib_dict_c') + instance.query(''' + CREATE DICTIONARY lib_dict_c (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) + PRIMARY KEY key SOURCE(library(PATH '/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so')) + LAYOUT(CACHE( + SIZE_IN_CELLS 10000000 + BLOCK_SIZE 4096 + FILE_SIZE 16777216 + READ_BUFFER_SIZE 1048576 + MAX_STORED_KEYS 1048576)) + LIFETIME(2) ; + ''') + @pytest.fixture(scope="module") def ch_cluster(): @@ -160,6 +175,52 @@ def test_null_values(ch_cluster): assert(result == expected) +def test_recover_after_bridge_crash(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + create_dict_simple() + + result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(0));''') + assert(result.strip() == '100') + result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') + assert(result.strip() == '101') + + instance.exec_in_container(['bash', '-c', 'kill -9 `pidof clickhouse-library-bridge`'], user='root') + instance.query('SYSTEM RELOAD DICTIONARY lib_dict_c') + + result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(0));''') + assert(result.strip() == '100') + result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') + assert(result.strip() == '101') + + instance.exec_in_container(['bash', '-c', 'kill -9 `pidof clickhouse-library-bridge`'], user='root') + instance.query('DROP DICTIONARY lib_dict_c') + + +def test_server_restart_bridge_might_be_stil_alive(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + create_dict_simple() + + result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') + assert(result.strip() == '101') + + instance.restart_clickhouse() + + result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') + assert(result.strip() == '101') + + instance.exec_in_container(['bash', '-c', 'kill -9 `pidof clickhouse-library-bridge`'], user='root') + instance.restart_clickhouse() + + result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') + assert(result.strip() == '101') + + instance.query('DROP DICTIONARY lib_dict_c') + + if __name__ == '__main__': cluster.start() input("Cluster created, press any key to destroy...") From 9c6a8b00593e8a9ad431e682b547c326704fc749 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 1 Aug 2021 08:51:40 +0000 Subject: [PATCH 050/198] Restore previous ids passing --- programs/library-bridge/Handlers.cpp | 28 +++++------- src/Bridge/LibraryBridgeHelper.cpp | 17 +------- tests/integration/test_library_bridge/test.py | 43 +++++++++++++++++++ 3 files changed, 55 insertions(+), 33 deletions(-) diff --git a/programs/library-bridge/Handlers.cpp b/programs/library-bridge/Handlers.cpp index 7e79830a7b2..0b49a801ca3 100644 --- a/programs/library-bridge/Handlers.cpp +++ b/programs/library-bridge/Handlers.cpp @@ -40,13 +40,12 @@ namespace return sample_block; } - // std::vector parseIdsFromBinary(const std::string & ids_string) - // { - // ReadBufferFromString buf(ids_string); - // std::vector ids; - // readVectorBinary(ids, buf); - // return ids; - // } + std::vector parseIdsFromBinary(ReadBuffer & buf) + { + std::vector ids; + readVectorBinary(ids, buf); + return ids; + } std::vector parseNamesFromBinary(const std::string & names_string) { @@ -212,6 +211,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found dictionary with id: {}", dictionary_id); const auto & sample_block = library_handler->getSampleBlock(); + LOG_DEBUG(log, "Calling loadAll() for dictionary id: {}", dictionary_id); auto input = library_handler->loadAll(); LOG_DEBUG(log, "Started sending result data for dictionary id: {}", dictionary_id); @@ -222,23 +222,14 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe { LOG_DEBUG(log, "Getting diciontary ids for dictionary with id: {}", dictionary_id); String ids_string; - readString(ids_string, request.getStream()); - - Strings ids_strs; - splitInto<'-'>(ids_strs, ids_string); - std::vector ids; - for (const auto & id : ids_strs) - ids.push_back(parse(id)); - if (ids.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Received no ids"); - - // std::vector ids = parseIdsFromBinary(ids_string); + std::vector ids = parseIdsFromBinary(request.getStream()); auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); if (!library_handler) throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found dictionary with id: {}", dictionary_id); const auto & sample_block = library_handler->getSampleBlock(); + LOG_DEBUG(log, "Calling loadIds() for dictionary id: {}", dictionary_id); auto input = library_handler->loadIds(ids); LOG_DEBUG(log, "Started sending result data for dictionary id: {}", dictionary_id); @@ -277,6 +268,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found dictionary with id: {}", dictionary_id); const auto & sample_block = library_handler->getSampleBlock(); + LOG_DEBUG(log, "Calling loadKeys() for dictionary id: {}", dictionary_id); auto input = library_handler->loadKeys(block.getColumns()); LOG_DEBUG(log, "Started sending result data for dictionary id: {}", dictionary_id); diff --git a/src/Bridge/LibraryBridgeHelper.cpp b/src/Bridge/LibraryBridgeHelper.cpp index 6f976200153..9cdc2c7864a 100644 --- a/src/Bridge/LibraryBridgeHelper.cpp +++ b/src/Bridge/LibraryBridgeHelper.cpp @@ -191,24 +191,11 @@ BlockInputStreamPtr LibraryBridgeHelper::loadAll() } -BlockInputStreamPtr LibraryBridgeHelper::loadIds(const std::string, const std::vector ids) +BlockInputStreamPtr LibraryBridgeHelper::loadIds(const std::string ids_str, const std::vector ids) { startBridgeSync(); auto uri = createRequestURI(LOAD_IDS_METHOD); - - uri.addQueryParameter("ids_num", toString(ids.size())); - String ids_str; - for (const auto & id : ids) - { - if (!ids_str.empty()) - ids_str += '-'; - ids_str += toString(id); - } - - uri.addQueryParameter("ids", ids_str); - std::cerr << "\n\nLibraryBridgeHelper: " << toString(dictionary_id) << " , ids_num: " << ids.size() << " , ids: " << ids_str << std::endl << std::endl; - LOG_ERROR(log, "dictionary_id: {}, ids_num: {}, ids: {}", dictionary_id, ids.size(), ids_str); - + uri.addQueryParameter("ids_num", toString(ids.size())); /// Not used parameter, but helpful return loadBase(uri, [ids_str](std::ostream & os) { os << ids_str; }); } diff --git a/tests/integration/test_library_bridge/test.py b/tests/integration/test_library_bridge/test.py index dc15f6a8430..0b01627a9c8 100644 --- a/tests/integration/test_library_bridge/test.py +++ b/tests/integration/test_library_bridge/test.py @@ -113,6 +113,10 @@ def test_load_ids(ch_cluster): result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(0));''') assert(result.strip() == '100') + + # Just check bridge is ok with a large vector of random ids + instance.query('''select number, dictGet(lib_dict_c, 'value1', toUInt64(rand())) from numbers(5000);''') + result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') assert(result.strip() == '101') instance.query('DROP DICTIONARY lib_dict_c') @@ -221,6 +225,45 @@ def test_server_restart_bridge_might_be_stil_alive(ch_cluster): instance.query('DROP DICTIONARY lib_dict_c') +def test_bridge_dies_with_parent(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + if instance.is_built_with_address_sanitizer(): + pytest.skip("Leak sanitizer falsely reports about a leak of 16 bytes in clickhouse-odbc-bridge") + + create_dict_simple() + result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') + assert(result.strip() == '101') + + clickhouse_pid = instance.get_process_pid("clickhouse server") + bridge_pid = instance.get_process_pid("library-bridge") + assert clickhouse_pid is not None + assert bridge_pid is not None + + while clickhouse_pid is not None: + try: + instance.exec_in_container(["kill", str(clickhouse_pid)], privileged=True, user='root') + except: + pass + clickhouse_pid = instance.get_process_pid("clickhouse server") + time.sleep(1) + + for i in range(30): + time.sleep(1) + bridge_pid = instance.get_process_pid("library-bridge") + if bridge_pid is None: + break + + if bridge_pid: + out = instance.exec_in_container(["gdb", "-p", str(bridge_pid), "--ex", "thread apply all bt", "--ex", "q"], + privileged=True, user='root') + logging.debug(f"Bridge is running, gdb output:\n{out}") + + assert clickhouse_pid is None + assert bridge_pid is None + instance.start_clickhouse(20) + + if __name__ == '__main__': cluster.start() input("Cluster created, press any key to destroy...") From 2bdb97d5e01f901ca585114c786d6c445d1a9df8 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 1 Aug 2021 09:56:48 +0000 Subject: [PATCH 051/198] Some small changes --- src/Bridge/LibraryBridgeHelper.cpp | 35 +++++++++++++++---- src/Bridge/LibraryBridgeHelper.h | 4 ++- src/Dictionaries/LibraryDictionarySource.cpp | 27 +++++++------- src/Dictionaries/LibraryDictionarySource.h | 2 -- tests/integration/test_library_bridge/test.py | 2 +- 5 files changed, 45 insertions(+), 25 deletions(-) diff --git a/src/Bridge/LibraryBridgeHelper.cpp b/src/Bridge/LibraryBridgeHelper.cpp index 9cdc2c7864a..b39e3f01903 100644 --- a/src/Bridge/LibraryBridgeHelper.cpp +++ b/src/Bridge/LibraryBridgeHelper.cpp @@ -1,6 +1,5 @@ #include "LibraryBridgeHelper.h" -#include #include #include #include @@ -8,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -20,6 +21,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int EXTERNAL_LIBRARY_ERROR; + extern const int LOGICAL_ERROR; +} + LibraryBridgeHelper::LibraryBridgeHelper( ContextPtr context_, const Block & sample_block_, @@ -93,23 +100,29 @@ bool LibraryBridgeHelper::checkBridgeIsRunning() const throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {} ({}). Check bridge and server have the same version.", result, parsed ? toString(dictionary_id_exists) : "failed to parse"); + LOG_TRACE(log, "dictionary_id: {}, dictionary_id_exists on bridge side: {}, library confirmed to be initialized on server side: {}", + toString(dictionary_id), toString(dictionary_id_exists), library_initialized); + if (dictionary_id_exists && !library_initialized) throw Exception(ErrorCodes::LOGICAL_ERROR, "Library was not initialized, but bridge responded to already have dictionary id: {}", dictionary_id); if (!dictionary_id_exists && library_initialized) { LOG_WARNING(log, "Library bridge does not have library handler with dictionaty id: {}. It will be reinitialized.", dictionary_id); + bool reinitialized = false; try { - if (!initLibrary(false)) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Failed to reinitialize library handler on bridge side for dictionary with id: {}", dictionary_id); + reinitialized = initLibrary(false); } catch (...) { tryLogCurrentException(log); return false; } + + if (!reinitialized) + throw Exception(ErrorCodes::EXTERNAL_LIBRARY_ERROR, + "Failed to reinitialize library handler on bridge side for dictionary with id: {}", dictionary_id); } return true; @@ -191,12 +204,13 @@ BlockInputStreamPtr LibraryBridgeHelper::loadAll() } -BlockInputStreamPtr LibraryBridgeHelper::loadIds(const std::string ids_str, const std::vector ids) +BlockInputStreamPtr LibraryBridgeHelper::loadIds(const std::vector & ids) { startBridgeSync(); auto uri = createRequestURI(LOAD_IDS_METHOD); uri.addQueryParameter("ids_num", toString(ids.size())); /// Not used parameter, but helpful - return loadBase(uri, [ids_str](std::ostream & os) { os << ids_str; }); + auto ids_string = getDictIdsString(ids); + return loadBase(uri, [ids_string](std::ostream & os) { os << ids_string; }); } @@ -246,4 +260,13 @@ BlockInputStreamPtr LibraryBridgeHelper::loadBase(const Poco::URI & uri, ReadWri return std::make_shared>(input_stream, std::move(read_buf_ptr)); } + +String LibraryBridgeHelper::getDictIdsString(const std::vector & ids) +{ + WriteBufferFromOwnString out; + writeVectorBinary(ids, out); + return out.str(); +} + + } diff --git a/src/Bridge/LibraryBridgeHelper.h b/src/Bridge/LibraryBridgeHelper.h index 815fce72b33..e4fe8e06965 100644 --- a/src/Bridge/LibraryBridgeHelper.h +++ b/src/Bridge/LibraryBridgeHelper.h @@ -38,7 +38,7 @@ public: BlockInputStreamPtr loadAll(); - BlockInputStreamPtr loadIds(std::string ids_string, const std::vector ids); + BlockInputStreamPtr loadIds(const std::vector & ids); BlockInputStreamPtr loadKeys(const Block & requested_block); @@ -88,6 +88,8 @@ private: Poco::URI createRequestURI(const String & method) const; + static String getDictIdsString(const std::vector & ids); + Poco::Logger * log; const Block sample_block; const Poco::Util::AbstractConfiguration & config; diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index d5a63735643..273f9d3c3b4 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -55,13 +55,14 @@ LibraryDictionarySource::LibraryDictionarySource( .library_settings = getLibrarySettingsString(config, config_prefix + ".settings"), .dict_attributes = getDictAttributesString() }; - bridge_helper = std::make_shared(context, description.sample_block, dictionary_id, library_data); - auto res = bridge_helper->initLibrary(); - if (!res) - throw Exception(ErrorCodes::EXTERNAL_LIBRARY_ERROR, "Failed to create shared library from path: {}", path); - else + bridge_helper = std::make_shared(context, description.sample_block, dictionary_id, library_data); + + bool initialized = bridge_helper->initLibrary(); + if (initialized) bridge_helper->setInitialized(); + else + throw Exception(ErrorCodes::EXTERNAL_LIBRARY_ERROR, "Failed to create shared library from path: {}", path); } @@ -90,7 +91,11 @@ LibraryDictionarySource::LibraryDictionarySource(const LibraryDictionarySource & , description{other.description} { bridge_helper = std::make_shared(context, description.sample_block, dictionary_id, other.bridge_helper->getLibraryData()); - bridge_helper->cloneLibrary(other.dictionary_id); + bool cloned = bridge_helper->cloneLibrary(other.dictionary_id); + if (cloned) + bridge_helper->setInitialized(); + else + throw Exception(ErrorCodes::EXTERNAL_LIBRARY_ERROR, "Failed to clone library"); } @@ -116,7 +121,7 @@ BlockInputStreamPtr LibraryDictionarySource::loadAll() BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector & ids) { LOG_TRACE(log, "loadIds {} size = {}", toString(), ids.size()); - return bridge_helper->loadIds(getDictIdsString(ids), ids); + return bridge_helper->loadIds(ids); } @@ -164,14 +169,6 @@ String LibraryDictionarySource::getLibrarySettingsString(const Poco::Util::Abstr } -String LibraryDictionarySource::getDictIdsString(const std::vector & ids) -{ - WriteBufferFromOwnString out; - writeVectorBinary(ids, out); - return out.str(); -} - - String LibraryDictionarySource::getDictAttributesString() { std::vector attributes_names(dict_struct.attributes.size()); diff --git a/src/Dictionaries/LibraryDictionarySource.h b/src/Dictionaries/LibraryDictionarySource.h index 88e133666e6..18a2d1b97b2 100644 --- a/src/Dictionaries/LibraryDictionarySource.h +++ b/src/Dictionaries/LibraryDictionarySource.h @@ -70,8 +70,6 @@ public: std::string toString() const override; private: - static String getDictIdsString(const std::vector & ids); - String getDictAttributesString(); static String getLibrarySettingsString(const Poco::Util::AbstractConfiguration & config, const std::string & config_root); diff --git a/tests/integration/test_library_bridge/test.py b/tests/integration/test_library_bridge/test.py index 0b01627a9c8..4bef197f8bd 100644 --- a/tests/integration/test_library_bridge/test.py +++ b/tests/integration/test_library_bridge/test.py @@ -115,7 +115,7 @@ def test_load_ids(ch_cluster): assert(result.strip() == '100') # Just check bridge is ok with a large vector of random ids - instance.query('''select number, dictGet(lib_dict_c, 'value1', toUInt64(rand())) from numbers(5000);''') + instance.query('''select number, dictGet(lib_dict_c, 'value1', toUInt64(rand())) from numbers(1000);''') result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') assert(result.strip() == '101') From 10d860acc37b21a1095d555cab27fb33217671e2 Mon Sep 17 00:00:00 2001 From: OmarBazaraa Date: Sun, 1 Aug 2021 14:26:11 +0000 Subject: [PATCH 052/198] Empty commit From 15f4ff9d6edac5e305241f7aabc067ef7ddb81b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=87=E5=BA=B7?= Date: Mon, 2 Aug 2021 08:29:28 +0800 Subject: [PATCH 053/198] skip parallel test --- tests/queries/skip_list.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 3eede6783f8..46bc1d8ae01 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -525,6 +525,7 @@ "01902_table_function_merge_db_repr", "01946_test_zstd_decompression_with_escape_sequence_at_the_end_of_buffer", "01946_test_wrong_host_name_access", - "01213_alter_rename_with_default_zookeeper" /// Warning: Removing leftovers from table. + "01213_alter_rename_with_default_zookeeper", /// Warning: Removing leftovers from table. + "02001_add_default_database_to_system_users" ///create user ] } From 583c2d67cbdbcfd34aa4594281dfd4ee00220dd4 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 2 Aug 2021 09:32:23 +0300 Subject: [PATCH 054/198] Update DiskSelector.cpp --- src/Disks/DiskSelector.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Disks/DiskSelector.cpp b/src/Disks/DiskSelector.cpp index 1bf71fb2efd..1e081a59701 100644 --- a/src/Disks/DiskSelector.cpp +++ b/src/Disks/DiskSelector.cpp @@ -75,7 +75,6 @@ DiskSelectorPtr DiskSelector::updateFromConfig( old_disks_minus_new_disks.erase(disk_name); - // TODO: Implement updateFromConfigIfChanged for DiskS3 (DiskLocal already done) result->getDisksMap().find(disk_name)->second->updateFromConfigIfChanged(config, disk_config_prefix, context); } } From 9007ddfe227133263ec1666ff3b87ff8fb59a2c9 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 2 Aug 2021 09:34:39 +0300 Subject: [PATCH 055/198] Update DiskLocal.h --- src/Disks/DiskLocal.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index d682952bd4f..714d4683590 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -17,7 +17,6 @@ namespace ErrorCodes class DiskLocalReservation; -class Context; class DiskLocal : public IDisk { public: From 322870acd29561e1415897a3ae5a43de0172543d Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 2 Aug 2021 09:36:44 +0300 Subject: [PATCH 056/198] Update IDisk.h --- src/Disks/IDisk.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 229e03e3bbf..0f902dd8338 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -35,7 +35,6 @@ using Reservations = std::vector; class ReadBufferFromFileBase; class WriteBufferFromFileBase; -class Context; class MMappedFileCache; /** @@ -232,7 +231,7 @@ public: /// Required for remote disk to ensure that replica has access to data written by other node virtual bool checkUniqueId(const String & id) const { return exists(id); } - /// Reload config if config changed + /// Reload config if it was changed virtual void updateFromConfigIfChanged(const Poco::Util::AbstractConfiguration & /* config */, const String & /* config_prefix */, ContextPtr /* context */) { } From b25dfdd6af0b2ea5749938731b2e3ee2d4ca52cd Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 2 Aug 2021 08:15:21 +0000 Subject: [PATCH 057/198] Use method which was added for s3 instead --- src/Disks/DiskLocal.cpp | 8 +++++--- src/Disks/DiskLocal.h | 4 +--- src/Disks/DiskSelector.cpp | 2 -- src/Disks/IDisk.h | 5 ----- 4 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index edebb6ff71e..b4d7f74f10f 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -367,15 +367,17 @@ SyncGuardPtr DiskLocal::getDirectorySyncGuard(const String & path) const return std::make_unique(fs::path(disk_path) / path); } -void DiskLocal::updateFromConfigIfChanged(const Poco::Util::AbstractConfiguration & config, - const String & config_prefix, ContextPtr context) + +void DiskLocal::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap &) { String new_disk_path; UInt64 new_keep_free_space_bytes; + loadDiskLocalConfig(name, config, config_prefix, context, new_disk_path, new_keep_free_space_bytes); if (disk_path != new_disk_path) - throw Exception("Disk path can't update from config " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + throw Exception("Disk path can't be updated from config " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + if (keep_free_space_bytes != new_keep_free_space_bytes) keep_free_space_bytes = new_keep_free_space_bytes; } diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index 714d4683590..0cf0cf39bbc 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -105,9 +105,7 @@ public: SyncGuardPtr getDirectorySyncGuard(const String & path) const override; - void updateFromConfigIfChanged(const Poco::Util::AbstractConfiguration & config, - const String & config_prefix, - ContextPtr context) override; + void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap &) override; private: bool tryReserve(UInt64 bytes); diff --git a/src/Disks/DiskSelector.cpp b/src/Disks/DiskSelector.cpp index 1e081a59701..bc7810479c5 100644 --- a/src/Disks/DiskSelector.cpp +++ b/src/Disks/DiskSelector.cpp @@ -74,8 +74,6 @@ DiskSelectorPtr DiskSelector::updateFromConfig( disk->applyNewSettings(config, context, disk_config_prefix, result->getDisksMap()); old_disks_minus_new_disks.erase(disk_name); - - result->getDisksMap().find(disk_name)->second->updateFromConfigIfChanged(config, disk_config_prefix, context); } } diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 0f902dd8338..61c805961ae 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -231,11 +231,6 @@ public: /// Required for remote disk to ensure that replica has access to data written by other node virtual bool checkUniqueId(const String & id) const { return exists(id); } - /// Reload config if it was changed - virtual void updateFromConfigIfChanged(const Poco::Util::AbstractConfiguration & /* config */, - const String & /* config_prefix */, - ContextPtr /* context */) { } - /// Invoked on partitions freeze query. virtual void onFreeze(const String &) { } From df0d9be04b4c32e4a21744c8f667e53f44b7e6fa Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 2 Aug 2021 12:59:13 +0300 Subject: [PATCH 058/198] Update docs/en/sql-reference/functions/tuple-map-functions.md --- docs/en/sql-reference/functions/tuple-map-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index 908d3e1bbd3..34935a10d8c 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -197,7 +197,7 @@ or Query with mapped arrays: -``` sql +```sql select mapPopulateSeries([1,2,4], [11,22,44], 5) as res, toTypeName(res) as type; ``` From 7364f35da6d4cd87b1c9ef5757c386b2c99358ac Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 2 Aug 2021 13:49:56 +0300 Subject: [PATCH 059/198] Add alwaysReturnsEmptySet for Merge Join --- src/Interpreters/MergeJoin.h | 2 ++ .../01015_empty_in_inner_right_join.reference | 17 +++++++++++++++++ ...l => 01015_empty_in_inner_right_join.sql.j2} | 6 ++++++ 3 files changed, 25 insertions(+) rename tests/queries/0_stateless/{01015_empty_in_inner_right_join.sql => 01015_empty_in_inner_right_join.sql.j2} (96%) diff --git a/src/Interpreters/MergeJoin.h b/src/Interpreters/MergeJoin.h index 11e5dc86dc2..844c730de4f 100644 --- a/src/Interpreters/MergeJoin.h +++ b/src/Interpreters/MergeJoin.h @@ -32,6 +32,8 @@ public: size_t getTotalRowCount() const override { return right_blocks.row_count; } size_t getTotalByteCount() const override { return right_blocks.bytes; } + /// Has to be called only after setTotals()/mergeRightBlocks() + bool alwaysReturnsEmptySet() const override { return (is_right || is_inner) && min_max_right_blocks.empty(); } BlockInputStreamPtr createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const override; diff --git a/tests/queries/0_stateless/01015_empty_in_inner_right_join.reference b/tests/queries/0_stateless/01015_empty_in_inner_right_join.reference index f41483b25da..1aa2f963bd7 100644 --- a/tests/queries/0_stateless/01015_empty_in_inner_right_join.reference +++ b/tests/queries/0_stateless/01015_empty_in_inner_right_join.reference @@ -15,3 +15,20 @@ multiple sets INNER JOIN empty set AND IN non-empty set 0 multiple sets INNER JOIN non-empty set AND IN non-empty set 1 IN empty set equals 0 10 IN empty set sum if 10 +IN empty set 0 +IN non-empty set 1 +NOT IN empty set 10 +INNER JOIN empty set 0 +INNER JOIN non-empty set 1 +RIGHT JOIN empty set 0 +RIGHT JOIN non-empty set 1 +LEFT JOIN empty set 10 +LEFT JOIN non-empty set 10 +multiple sets IN empty set OR IN non-empty set 1 +multiple sets IN empty set OR NOT IN non-empty set 9 +multiple sets NOT IN empty set AND IN non-empty set 1 +multiple sets INNER JOIN empty set AND IN empty set 0 +multiple sets INNER JOIN empty set AND IN non-empty set 0 +multiple sets INNER JOIN non-empty set AND IN non-empty set 1 +IN empty set equals 0 10 +IN empty set sum if 10 diff --git a/tests/queries/0_stateless/01015_empty_in_inner_right_join.sql b/tests/queries/0_stateless/01015_empty_in_inner_right_join.sql.j2 similarity index 96% rename from tests/queries/0_stateless/01015_empty_in_inner_right_join.sql rename to tests/queries/0_stateless/01015_empty_in_inner_right_join.sql.j2 index e0d06a7e3b6..6c13654598e 100644 --- a/tests/queries/0_stateless/01015_empty_in_inner_right_join.sql +++ b/tests/queries/0_stateless/01015_empty_in_inner_right_join.sql.j2 @@ -1,5 +1,9 @@ SET joined_subquery_requires_alias = 0; +{% for join_algorithm in ['partial_merge', 'hash'] -%} + +SET join_algorithm = '{{ join_algorithm }}'; + SELECT 'IN empty set',count() FROM system.numbers WHERE number IN (SELECT toUInt64(1) WHERE 0); SELECT 'IN non-empty set',count() FROM (SELECT number FROM system.numbers LIMIT 10) t1 WHERE t1.number IN (SELECT toUInt64(1) WHERE 1); SELECT 'NOT IN empty set',count() FROM (SELECT number FROM system.numbers LIMIT 10) WHERE number NOT IN (SELECT toUInt64(1) WHERE 0); @@ -22,3 +26,5 @@ SELECT 'multiple sets INNER JOIN non-empty set AND IN non-empty set',count() FRO SELECT 'IN empty set equals 0', count() FROM numbers(10) WHERE (number IN (SELECT toUInt64(1) WHERE 0)) = 0; SELECT 'IN empty set sum if', sum(if(number IN (SELECT toUInt64(1) WHERE 0), 2, 1)) FROM numbers(10); + +{% endfor -%} From 593b7e9bbb0f0492e0e50b228ca39eff355b53fc Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 29 Jul 2021 14:50:48 +0300 Subject: [PATCH 060/198] Do not throw exception if unknwon column met while join rewriting --- src/Interpreters/JoinToSubqueryTransformVisitor.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 8595d3a4179..eabdeaefc04 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -551,8 +551,6 @@ std::vector normalizeColumnNamesExtractNeeded( else needed_columns[*table_pos].no_clashes.emplace(ident->shortName()); } - else if (!got_alias) - throw Exception("Unknown column name '" + ident->name() + "'", ErrorCodes::UNKNOWN_IDENTIFIER); } return needed_columns; From fc9d72e75dd4176b73e712f7b1db47ca3a371d20 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 2 Aug 2021 15:03:55 +0300 Subject: [PATCH 061/198] fix performance of short queries with large number of columns --- src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp | 4 ++-- src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h | 2 +- src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp | 2 +- src/Storages/MergeTree/MergeTreeReadPool.cpp | 2 +- src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp | 2 +- src/Storages/MergeTree/MergeTreeSelectProcessor.cpp | 1 + src/Storages/MergeTree/MergeTreeSelectProcessor.h | 3 +++ 7 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 0ebf5ec33d7..c91d60c5de7 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -468,7 +468,7 @@ Block MergeTreeBaseSelectProcessor::transformHeader( std::unique_ptr MergeTreeBaseSelectProcessor::getSizePredictor( const MergeTreeData::DataPartPtr & data_part, const MergeTreeReadTaskColumns & task_columns, - const StorageMetadataPtr & metadata_snapshot) + const Block & sample_block) { const auto & required_column_names = task_columns.columns.getNames(); const auto & required_pre_column_names = task_columns.pre_columns.getNames(); @@ -476,7 +476,7 @@ std::unique_ptr MergeTreeBaseSelectProcessor::getSi complete_column_names.insert(required_pre_column_names.begin(), required_pre_column_names.end()); return std::make_unique( - data_part, Names(complete_column_names.begin(), complete_column_names.end()), metadata_snapshot->getSampleBlock()); + data_part, Names(complete_column_names.begin(), complete_column_names.end()), sample_block); } MergeTreeBaseSelectProcessor::~MergeTreeBaseSelectProcessor() = default; diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h index 3f6a841f780..d102e4f07a4 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h @@ -40,7 +40,7 @@ public: static std::unique_ptr getSizePredictor( const MergeTreeData::DataPartPtr & data_part, const MergeTreeReadTaskColumns & task_columns, - const StorageMetadataPtr & metadata_snapshot); + const Block & sample_block); protected: Chunk generate() final; diff --git a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp index b56430003b1..70fea1414e6 100644 --- a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp @@ -18,7 +18,7 @@ try } auto size_predictor = (preferred_block_size_bytes == 0) ? nullptr - : getSizePredictor(data_part, task_columns, metadata_snapshot); + : getSizePredictor(data_part, task_columns, sample_block); MarkRanges mark_ranges_for_task; /// If we need to read few rows, set one range per task to reduce number of read data. diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index 30a6edf9f92..b18f22c3ab1 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -231,7 +231,7 @@ std::vector MergeTreeReadPool::fillPerPartInfo( auto task_columns = getReadTaskColumns(data, metadata_snapshot, part.data_part, column_names, prewhere_info, check_columns); auto size_predictor = !predict_block_size_bytes ? nullptr - : MergeTreeBaseSelectProcessor::getSizePredictor(part.data_part, task_columns, metadata_snapshot); + : MergeTreeBaseSelectProcessor::getSizePredictor(part.data_part, task_columns, sample_block); per_part_size_predictor.emplace_back(std::move(size_predictor)); diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index 7ab71bedb4a..5edf322cd6a 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -27,7 +27,7 @@ try all_mark_ranges.pop_back(); auto size_predictor = (preferred_block_size_bytes == 0) ? nullptr - : getSizePredictor(data_part, task_columns, metadata_snapshot); + : getSizePredictor(data_part, task_columns, sample_block); task = std::make_unique( data_part, mark_ranges_for_task, part_index_in_query, ordered_names, column_name_set, diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 440b47b12cf..63f3fc639e1 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -31,6 +31,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( reader_settings_, use_uncompressed_cache_, virt_column_names_}, required_columns{std::move(required_columns_)}, data_part{owned_data_part_}, + sample_block(metadata_snapshot_->getSampleBlock()), all_mark_ranges(std::move(mark_ranges_)), part_index_in_query(part_index_in_query_), has_limit_below_one_block(has_limit_below_one_block_), diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 13662c087e8..20a102c6ff6 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -51,6 +51,9 @@ protected: /// Data part will not be removed if the pointer owns it MergeTreeData::DataPartPtr data_part; + /// Cache getSampleBlock call, which might be heavy. + Block sample_block; + /// Mark ranges we should read (in ascending order) MarkRanges all_mark_ranges; /// Value of _part_index virtual column (used only in SelectExecutor) From 5d2aec78c3b97794473f20e735ce2c6a6bcab434 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 2 Aug 2021 15:08:14 +0300 Subject: [PATCH 062/198] Add testcase for multiple joins and unknwon identifier in where section --- .../0_stateless/00820_multiple_joins.reference | 1 + tests/queries/0_stateless/00820_multiple_joins.sql | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/tests/queries/0_stateless/00820_multiple_joins.reference b/tests/queries/0_stateless/00820_multiple_joins.reference index 6d317230813..f9d9a664784 100644 --- a/tests/queries/0_stateless/00820_multiple_joins.reference +++ b/tests/queries/0_stateless/00820_multiple_joins.reference @@ -48,3 +48,4 @@ 6 6 60 60 66 66 120 12 12 120 120 132 132 240 18 18 180 180 198 198 360 +1 diff --git a/tests/queries/0_stateless/00820_multiple_joins.sql b/tests/queries/0_stateless/00820_multiple_joins.sql index 890d003a124..df82a199337 100644 --- a/tests/queries/0_stateless/00820_multiple_joins.sql +++ b/tests/queries/0_stateless/00820_multiple_joins.sql @@ -2,6 +2,7 @@ DROP TABLE IF EXISTS table1; DROP TABLE IF EXISTS table2; DROP TABLE IF EXISTS table3; DROP TABLE IF EXISTS table5; +DROP TABLE IF EXISTS table_set; CREATE TABLE table1 (a UInt32) ENGINE = Memory; CREATE TABLE table2 (a UInt32, b UInt32) ENGINE = Memory; @@ -76,6 +77,18 @@ join table3 as t3 on t2_b = t3_b; --join table2 as t2 on t1_t2_x = t2.a --join table3 as t3 on t1_t3_x = t2_t3_x; + +CREATE TABLE table_set ( x UInt32 ) ENGINE = Set; +INSERT INTO table_set VALUES (0), (1), (2); + +select count() +from table1 as t1 +join table2 as t2 on t1.a = t2.a +join table3 as t3 on t2.b = t3.b +join table5 as t5 on t3.c = t5.c +WHERE t1.a in table_set; + +DROP TABLE table_set; DROP TABLE table1; DROP TABLE table2; DROP TABLE table3; From d2440dca86da346d66ad88fb302a66e429b695e5 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 2 Aug 2021 15:18:05 +0300 Subject: [PATCH 063/198] Updated cache dictionaries documentation --- .../external-dicts-dict-layout.md | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index c69dc4224e6..81b3df6742f 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -275,12 +275,17 @@ The dictionary is stored in a cache that has a fixed number of cells. These cell When searching for a dictionary, the cache is searched first. For each block of data, all keys that are not found in the cache or are outdated are requested from the source using `SELECT attrs... FROM db.table WHERE id IN (k1, k2, ...)`. The received data is then written to the cache. -For cache dictionaries, the expiration [lifetime](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md) of data in the cache can be set. If more time than `lifetime` has passed since loading the data in a cell, the cell’s value is not used, and it is re-requested the next time it needs to be used. +If keys are not found in dictionary, then update cache task is created and added into update queue. Update queue properties can be controlled with settings `max_update_queue_size`, `update_queue_push_timeout_milliseconds`, `query_wait_timeout_milliseconds`, `max_threads_for_updates`. + +For cache dictionaries, the expiration [lifetime](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md) of data in the cache can be set. If more time than `lifetime` has passed since loading the data in a cell, the cell’s value is not used and key becomes expired, and it is re-requested the next time it needs to be used this behaviour can be configured with setting `allow_read_expired_keys`. This is the least effective of all the ways to store dictionaries. The speed of the cache depends strongly on correct settings and the usage scenario. A cache type dictionary performs well only when the hit rates are high enough (recommended 99% and higher). You can view the average hit rate in the `system.dictionaries` table. +If setting `allow_read_expired_keys` is set to 1, by default 0. Then dictionary can support asynchronous updates. If client requests keys and all of them are in cache, but some of them are expired, then dictionary will return expired keys for client and request them asynchronously from source. + To improve cache performance, use a subquery with `LIMIT`, and call the function with the dictionary externally. -Supported [sources](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md): MySQL, ClickHouse, executable, HTTP. +Supported [sources](../../../ +sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md): MySQL, ClickHouse, executable, HTTP. Example of settings: @@ -289,6 +294,16 @@ Example of settings: 1000000000 + + 0 + + 100000 + + 10 + + 60000 + + 4 ``` @@ -315,7 +330,7 @@ This type of storage is for use with composite [keys](../../../sql-reference/dic ### ssd_cache {#ssd-cache} -Similar to `cache`, but stores data on SSD and index in RAM. +Similar to `cache`, but stores data on SSD and index in RAM. All cache dictionary settings related to update queue can also be applied to SSD cache dictionaries. ``` xml From d9db3dcff87961bf87e28cfb78bcb4a3543b3732 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Mon, 2 Aug 2021 12:32:45 +0000 Subject: [PATCH 064/198] Improve docs --- .../en/sql-reference/functions/nlp-functions.md | 17 ++++++++++++----- .../ru/sql-reference/functions/nlp-functions.md | 13 ++++++++++--- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/docs/en/sql-reference/functions/nlp-functions.md b/docs/en/sql-reference/functions/nlp-functions.md index 2d5a09c0897..5edaf8486b9 100644 --- a/docs/en/sql-reference/functions/nlp-functions.md +++ b/docs/en/sql-reference/functions/nlp-functions.md @@ -3,11 +3,14 @@ toc_priority: 67 toc_title: NLP --- -# Natural Language Processing functions {#nlp-functions} +# [experimental] Natural Language Processing functions {#nlp-functions} + +!!! warning "Warning" + This is an experimental feature that is currently in development and is not ready for general use. It will change in unpredictable backwards-incompatible ways in the future releases. Set `allow_experimental_nlp_functions = 1` to enable it. ## stem {#stem} -Performs stemming on a previously tokenized text. +Performs stemming on a given word. **Syntax** @@ -38,7 +41,7 @@ Result: ## lemmatize {#lemmatize} -Performs lemmatization on a given word. +Performs lemmatization on a given word. Needs dictionaries to operate. Dictionaries can be obtained [here](https://github.com/vpodpecan/lemmagen3/tree/master/src/lemmagen3/models). **Syntax** @@ -79,7 +82,11 @@ Configuration: ## synonyms {#synonyms} -Finds synonyms to a given word. +Finds synonyms to a given word. There are two types of synonym extensions: `plain` and `wordnet`. + +With `plain` extension type we need to provide a path to simple text file, where each line corresponds to certain synonym set. Words in this line must be separated with space or tab characters. + +With `wordnet` extension type we need to provide a path to directory with WordNet thesaurus in it. Thesaurus must contain WordNet sense index. **Syntax** @@ -89,7 +96,7 @@ synonyms('extension_name', word) **Arguments** -- `extension_name` — Name of the extention in which search will be performed. [String](../../sql-reference/data-types/string.md#string). +- `extension_name` — Name of the extension in which search will be performed. [String](../../sql-reference/data-types/string.md#string). - `word` — Word that will be searched in extension. [String](../../sql-reference/data-types/string.md#string). **Examples** diff --git a/docs/ru/sql-reference/functions/nlp-functions.md b/docs/ru/sql-reference/functions/nlp-functions.md index 582b5c93b93..58c4eb86e35 100644 --- a/docs/ru/sql-reference/functions/nlp-functions.md +++ b/docs/ru/sql-reference/functions/nlp-functions.md @@ -3,7 +3,10 @@ toc_priority: 67 toc_title: NLP --- -# Функции для работы с ествественным языком {#nlp-functions} +# [экспериментально] Функции для работы с ествественным языком {#nlp-functions} + +!!! warning "Предупреждение" + Сейчас использование функций для работы с ествественным языком является экспериментальной возможностью. Чтобы использовать данные функции, включите настройку `allow_experimental_nlp_functions = 1`. ## stem {#stem} @@ -38,7 +41,7 @@ Result: ## lemmatize {#lemmatize} -Данная функция проводит лемматизацию для заданного слова. +Данная функция проводит лемматизацию для заданного слова. Для работы лемматизатора необходимы словари, которые можно найти [здесь](https://github.com/vpodpecan/lemmagen3/tree/master/src/lemmagen3/models). **Синтаксис** @@ -79,7 +82,11 @@ SELECT lemmatize('en', 'wolves'); ## synonyms {#synonyms} -Находит синонимы к заданному слову. +Находит синонимы к заданному слову. Представлены два типа расширений словарей: `plain` и `wordnet`. + +Для работы расширения типа `plain` необходимо указать путь до простого текстового файла, где каждая строка соотвествует одному набору синонимов. Слова в данной строке должны быть разделены с помощью пробела или знака табуляции. + +Для работы расширения типа `plain` необходимо указать путь до WordNet тезауруса. Тезаурус должен содержать WordNet sense index. **Синтаксис** From 8577938efef1186290080d256e98eff0664c01df Mon Sep 17 00:00:00 2001 From: OmarBazaraa Date: Mon, 2 Aug 2021 12:34:50 +0000 Subject: [PATCH 065/198] Throw exception in case SSL is not enabled --- src/Storages/StorageMongoDBSocketFactory.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageMongoDBSocketFactory.cpp b/src/Storages/StorageMongoDBSocketFactory.cpp index 955507fc2bc..d80088865b9 100644 --- a/src/Storages/StorageMongoDBSocketFactory.cpp +++ b/src/Storages/StorageMongoDBSocketFactory.cpp @@ -1,5 +1,7 @@ #include "StorageMongoDBSocketFactory.h" +#include + #if !defined(ARCADIA_BUILD) # include #endif @@ -15,6 +17,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME; +} + Poco::Net::StreamSocket StorageMongoDBSocketFactory::createSocket(const std::string & host, int port, Poco::Timespan connectTimeout, bool secure) { return secure ? createSecureSocket(host, port, connectTimeout) : createPlainSocket(host, port, connectTimeout); @@ -41,7 +48,7 @@ Poco::Net::StreamSocket StorageMongoDBSocketFactory::createSecureSocket(const st return socket; #else - return createPlainSocket(host, port, connectTimeout); + throw Exception("SSL is not enabled at build time.", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); #endif } From 91928fdf5a79e7eb4f05a477c1382a77f5ab1c77 Mon Sep 17 00:00:00 2001 From: OmarBazaraa Date: Mon, 2 Aug 2021 13:11:49 +0000 Subject: [PATCH 066/198] Fix --- src/Storages/StorageMongoDBSocketFactory.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Storages/StorageMongoDBSocketFactory.cpp b/src/Storages/StorageMongoDBSocketFactory.cpp index d80088865b9..5f8fcab369f 100644 --- a/src/Storages/StorageMongoDBSocketFactory.cpp +++ b/src/Storages/StorageMongoDBSocketFactory.cpp @@ -3,6 +3,7 @@ #include #if !defined(ARCADIA_BUILD) +# include "config_core.h" # include #endif @@ -13,6 +14,14 @@ # include #endif +#ifdef __clang__ +# pragma clang diagnostic ignored "-Wunused-parameter" +#endif + +#if USE_EMBEDDED_COMPILER +# pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + namespace DB { From 42a8bb6872c57af82ec3d68972b98562dd2132b0 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 30 Jul 2021 19:34:18 +0300 Subject: [PATCH 067/198] fix assertions in Replicated database --- src/Databases/DatabaseReplicated.cpp | 34 ++++++++++++------- src/Databases/DatabaseReplicatedWorker.cpp | 11 +++--- src/Interpreters/ClientInfo.h | 2 ++ src/Interpreters/DDLTask.cpp | 1 + src/Interpreters/DDLTask.h | 2 +- src/Interpreters/InterpreterAlterQuery.cpp | 5 +-- src/Interpreters/InterpreterCreateQuery.cpp | 4 +-- src/Interpreters/InterpreterDropQuery.cpp | 3 +- src/Interpreters/InterpreterRenameQuery.cpp | 2 +- src/Storages/AlterCommands.cpp | 5 +++ src/Storages/AlterCommands.h | 5 ++- src/Storages/StorageReplicatedMergeTree.cpp | 11 ++++++ ...keeper_merge_tree_alter_settings.reference | 30 ++++++++-------- ...80_zookeeper_merge_tree_alter_settings.sql | 8 ++--- ...mutation_stuck_after_replace_partition.sql | 1 + .../01175_distributed_ddl_output_mode_long.sh | 2 +- .../01294_system_distributed_on_cluster.sql | 1 + .../0_stateless/01415_sticking_mutations.sh | 3 +- tests/queries/skip_list.json | 15 ++++---- 19 files changed, 92 insertions(+), 53 deletions(-) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 2c433c79033..ac6c525af68 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -40,6 +40,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; extern const int INCORRECT_QUERY; extern const int ALL_CONNECTION_TRIES_FAILED; + extern const int NO_ACTIVE_REPLICAS; } static constexpr const char * DROPPED_MARK = "DROPPED"; @@ -137,7 +138,9 @@ ClusterPtr DatabaseReplicated::getClusterImpl() const Coordination::Stat stat; hosts = zookeeper->getChildren(zookeeper_path + "/replicas", &stat); if (hosts.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "No hosts found"); + throw Exception(ErrorCodes::NO_ACTIVE_REPLICAS, "No replicas of database {} found. " + "It's possible if the first replica is not fully created yet " + "or if the last replica was just dropped or due to logical error", database_name); Int32 cversion = stat.cversion; std::sort(hosts.begin(), hosts.end()); @@ -514,6 +517,19 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep } } + auto make_query_context = [this, current_zookeeper]() + { + auto query_context = Context::createCopy(getContext()); + query_context->makeQueryContext(); + query_context->getClientInfo().query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; + query_context->getClientInfo().is_replicated_database_internal = true; + query_context->setCurrentDatabase(database_name); + query_context->setCurrentQueryId(""); + auto txn = std::make_shared(current_zookeeper, zookeeper_path, false); + query_context->initZooKeeperMetadataTransaction(txn); + return query_context; + }; + String db_name = getDatabaseName(); String to_db_name = getDatabaseName() + BROKEN_TABLES_SUFFIX; if (total_tables * db_settings.max_broken_tables_ratio < tables_to_detach.size()) @@ -548,7 +564,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep dropped_dictionaries += table->isDictionary(); table->flushAndShutdown(); - DatabaseAtomic::dropTable(getContext(), table_name, true); + DatabaseAtomic::dropTable(make_query_context(), table_name, true); } else { @@ -558,7 +574,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep assert(db_name < to_db_name); DDLGuardPtr to_table_guard = DatabaseCatalog::instance().getDDLGuard(to_db_name, to_name); auto to_db_ptr = DatabaseCatalog::instance().getDatabase(to_db_name); - DatabaseAtomic::renameTable(getContext(), table_name, *to_db_ptr, to_name, false, false); + DatabaseAtomic::renameTable(make_query_context(), table_name, *to_db_ptr, to_name, false, false); ++moved_tables; } } @@ -577,7 +593,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep /// TODO Maybe we should do it in two steps: rename all tables to temporary names and then rename them to actual names? DDLGuardPtr table_guard = DatabaseCatalog::instance().getDDLGuard(db_name, std::min(from, to)); DDLGuardPtr to_table_guard = DatabaseCatalog::instance().getDDLGuard(db_name, std::max(from, to)); - DatabaseAtomic::renameTable(getContext(), from, *this, to, false, false); + DatabaseAtomic::renameTable(make_query_context(), from, *this, to, false, false); } for (const auto & id : dropped_tables) @@ -592,15 +608,9 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep } auto query_ast = parseQueryFromMetadataInZooKeeper(name_and_meta.first, name_and_meta.second); - - auto query_context = Context::createCopy(getContext()); - query_context->makeQueryContext(); - query_context->getClientInfo().query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; - query_context->setCurrentDatabase(database_name); - query_context->setCurrentQueryId(""); // generate random query_id - LOG_INFO(log, "Executing {}", serializeAST(*query_ast)); - InterpreterCreateQuery(query_ast, query_context).execute(); + auto create_query_context = make_query_context(); + InterpreterCreateQuery(query_ast, create_query_context).execute(); } current_zookeeper->set(replica_path + "/log_ptr", toString(max_log_ptr)); diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index eb7e65e1b70..365a5d02816 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -60,12 +60,13 @@ void DatabaseReplicatedDDLWorker::initializeReplication() /// Check if we need to recover replica. /// Invariant: replica is lost if it's log_ptr value is less then max_log_ptr - logs_to_keep. - String log_ptr_str = current_zookeeper->get(database->replica_path + "/log_ptr"); + auto zookeeper = getAndSetZooKeeper(); + String log_ptr_str = zookeeper->get(database->replica_path + "/log_ptr"); UInt32 our_log_ptr = parse(log_ptr_str); - UInt32 max_log_ptr = parse(current_zookeeper->get(database->zookeeper_path + "/max_log_ptr")); - logs_to_keep = parse(current_zookeeper->get(database->zookeeper_path + "/logs_to_keep")); + UInt32 max_log_ptr = parse(zookeeper->get(database->zookeeper_path + "/max_log_ptr")); + logs_to_keep = parse(zookeeper->get(database->zookeeper_path + "/logs_to_keep")); if (our_log_ptr == 0 || our_log_ptr + logs_to_keep < max_log_ptr) - database->recoverLostReplica(current_zookeeper, our_log_ptr, max_log_ptr); + database->recoverLostReplica(zookeeper, our_log_ptr, max_log_ptr); else last_skipped_entry_name.emplace(DDLTaskBase::getLogEntryName(our_log_ptr)); } @@ -198,7 +199,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na } } - UInt32 our_log_ptr = parse(current_zookeeper->get(fs::path(database->replica_path) / "log_ptr")); + UInt32 our_log_ptr = parse(zookeeper->get(fs::path(database->replica_path) / "log_ptr")); UInt32 entry_num = DatabaseReplicatedTask::getLogEntryNumber(entry_name); if (entry_num <= our_log_ptr) diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index d6158a2d7d5..7c169e6ebb5 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -100,6 +100,8 @@ public: UInt64 distributed_depth = 0; + bool is_replicated_database_internal = false; + bool empty() const { return query_kind == QueryKind::NO_QUERY; } /** Serialization and deserialization. diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index 4fb44738d8d..8bebc2fb442 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -359,6 +359,7 @@ ContextMutablePtr DatabaseReplicatedTask::makeQueryContext(ContextPtr from_conte { auto query_context = DDLTaskBase::makeQueryContext(from_context, zookeeper); query_context->getClientInfo().query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; + query_context->getClientInfo().is_replicated_database_internal = true; query_context->setCurrentDatabase(database->getDatabaseName()); auto txn = std::make_shared(zookeeper, database->zookeeper_path, is_initial_query); diff --git a/src/Interpreters/DDLTask.h b/src/Interpreters/DDLTask.h index 703d691a358..41ab9ec8058 100644 --- a/src/Interpreters/DDLTask.h +++ b/src/Interpreters/DDLTask.h @@ -196,7 +196,7 @@ public: void commit(); - ~ZooKeeperMetadataTransaction() { assert(isExecuted() || std::uncaught_exceptions()); } + ~ZooKeeperMetadataTransaction() { assert(isExecuted() || std::uncaught_exceptions() || ops.empty()); } }; ClusterPtr tryGetReplicatedDatabaseCluster(const String & cluster_name); diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 6f0af049d05..76e7afb7009 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -54,7 +54,7 @@ BlockIO InterpreterAlterQuery::execute() DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); if (typeid_cast(database.get()) - && getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY) + && !getContext()->getClientInfo().is_replicated_database_internal) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); @@ -100,7 +100,8 @@ BlockIO InterpreterAlterQuery::execute() if (typeid_cast(database.get())) { int command_types_count = !mutation_commands.empty() + !partition_commands.empty() + !live_view_commands.empty() + !alter_commands.empty(); - if (1 < command_types_count) + bool mixed_settings_amd_metadata_alter = alter_commands.hasSettingsAlterCommand() && !alter_commands.isSettingsAlter(); + if (1 < command_types_count || mixed_settings_amd_metadata_alter) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "For Replicated databases it's not allowed " "to execute ALTERs of different types in single query"); } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 1f7e239b504..b3cbfdcd035 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -856,7 +856,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.table); if (auto* ptr = typeid_cast(database.get()); - ptr && getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY) + ptr && !getContext()->getClientInfo().is_replicated_database_internal) { create.database = database_name; guard->releaseTableLock(); @@ -950,7 +950,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) auto guard = DatabaseCatalog::instance().getDDLGuard(create.database, create.table); if (auto * ptr = typeid_cast(database.get()); - ptr && getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY) + ptr && !getContext()->getClientInfo().is_replicated_database_internal) { assertOrSetUUID(create, database); guard->releaseTableLock(); diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index d5f6c3260c3..0e15c6be27c 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -133,7 +133,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ASTDropQuery & query, DatabaseP /// Prevents recursive drop from drop database query. The original query must specify a table. bool is_drop_or_detach_database = query_ptr->as()->table.empty(); bool is_replicated_ddl_query = typeid_cast(database.get()) && - getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && + !getContext()->getClientInfo().is_replicated_database_internal && !is_drop_or_detach_database; AccessFlags drop_storage; @@ -426,6 +426,7 @@ void InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind kind, ContextPtr if (auto txn = current_context->getZooKeeperMetadataTransaction()) { /// For Replicated database + drop_context->getClientInfo().is_replicated_database_internal = true; drop_context->setQueryContext(std::const_pointer_cast(current_context)); drop_context->initZooKeeperMetadataTransaction(txn, true); } diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp index 515559ad903..373953e7530 100644 --- a/src/Interpreters/InterpreterRenameQuery.cpp +++ b/src/Interpreters/InterpreterRenameQuery.cpp @@ -81,7 +81,7 @@ BlockIO InterpreterRenameQuery::executeToTables(const ASTRenameQuery & rename, c DatabasePtr database = database_catalog.getDatabase(elem.from_database_name); if (typeid_cast(database.get()) - && getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY) + && !getContext()->getClientInfo().is_replicated_database_internal) { if (1 < descriptions.size()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Database {} is Replicated, " diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 9e9510b51a4..1bb8e6bb9b0 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -1276,6 +1276,11 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, ContextPt validateColumnsDefaultsAndGetSampleBlock(default_expr_list, all_columns.getAll(), context); } +bool AlterCommands::hasSettingsAlterCommand() const +{ + return std::any_of(begin(), end(), [](const AlterCommand & c) { return c.isSettingsAlter(); }); +} + bool AlterCommands::isSettingsAlter() const { return std::all_of(begin(), end(), [](const AlterCommand & c) { return c.isSettingsAlter(); }); diff --git a/src/Storages/AlterCommands.h b/src/Storages/AlterCommands.h index 6987de68f9c..60f4ad7d552 100644 --- a/src/Storages/AlterCommands.h +++ b/src/Storages/AlterCommands.h @@ -195,9 +195,12 @@ public: void apply(StorageInMemoryMetadata & metadata, ContextPtr context) const; /// At least one command modify settings. + bool hasSettingsAlterCommand() const; + + /// All commands modify settings only. bool isSettingsAlter() const; - /// At least one command modify comments. + /// All commands modify comments only. bool isCommentAlter() const; /// Return mutation commands which some storages may execute as part of diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8b6267943bd..0049f28e568 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4972,6 +4972,8 @@ void StorageReplicatedMergeTree::alter( if (auto txn = query_context->getZooKeeperMetadataTransaction()) { + /// It would be better to clone ops instead of moving, so we could retry on ZBADVERSION, + /// but clone() is not implemented for Coordination::Request. txn->moveOpsTo(ops); /// NOTE: IDatabase::alterTable(...) is called when executing ALTER_METADATA queue entry without query context, /// so we have to update metadata of DatabaseReplicated here. @@ -5015,6 +5017,11 @@ void StorageReplicatedMergeTree::alter( throw Exception("Metadata on replica is not up to date with common metadata in Zookeeper. Cannot alter", ErrorCodes::CANNOT_ASSIGN_ALTER); + /// Cannot retry automatically, because some zookeeper ops were lost on the first attempt. Will retry on DDLWorker-level. + if (query_context->getZooKeeperMetadataTransaction()) + throw Exception("Cannot execute alter, because mutations version was suddenly changed due to concurrent alter", + ErrorCodes::CANNOT_ASSIGN_ALTER); + continue; } else @@ -6007,6 +6014,10 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, Conte } else if (rc == Coordination::Error::ZBADVERSION) { + /// Cannot retry automatically, because some zookeeper ops were lost on the first attempt. Will retry on DDLWorker-level. + if (query_context->getZooKeeperMetadataTransaction()) + throw Exception("Cannot execute alter, because mutations version was suddenly changed due to concurrent alter", + ErrorCodes::CANNOT_ASSIGN_ALTER); LOG_TRACE(log, "Version conflict when trying to create a mutation node, retrying..."); continue; } diff --git a/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference b/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference index 357d1bef78d..811facbac40 100644 --- a/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference +++ b/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference @@ -1,21 +1,21 @@ -CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 -CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 4 4 4 4 6 6 -CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1 -CREATE TABLE default.replicated_table_for_alter2\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_alter\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 -CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64,\n `Data` String,\n `Data2` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1, check_delay_period = 15 -CREATE TABLE default.replicated_table_for_alter2\n(\n `id` UInt64,\n `Data` String,\n `Data2` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_alter\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 -CREATE TABLE default.replicated_table_for_reset_setting1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_reset_setting\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 -CREATE TABLE default.replicated_table_for_reset_setting2\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_reset_setting\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192 -CREATE TABLE default.replicated_table_for_reset_setting1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_reset_setting\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 -CREATE TABLE default.replicated_table_for_reset_setting1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_reset_setting\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192, merge_with_ttl_timeout = 100 -CREATE TABLE default.replicated_table_for_reset_setting2\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_reset_setting\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192, merge_with_ttl_timeout = 200 -CREATE TABLE default.replicated_table_for_reset_setting1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_reset_setting\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192, merge_with_ttl_timeout = 100 -CREATE TABLE default.replicated_table_for_reset_setting2\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_reset_setting\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192, merge_with_ttl_timeout = 200 -CREATE TABLE default.replicated_table_for_reset_setting1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_reset_setting\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 -CREATE TABLE default.replicated_table_for_reset_setting2\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980/replicated_table_for_reset_setting\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1 +CREATE TABLE default.replicated_table_for_alter2\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_alter\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64,\n `Data` String,\n `Data2` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1, check_delay_period = 15 +CREATE TABLE default.replicated_table_for_alter2\n(\n `id` UInt64,\n `Data` String,\n `Data2` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_alter\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.replicated_table_for_reset_setting1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_reset_setting\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.replicated_table_for_reset_setting2\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_reset_setting\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.replicated_table_for_reset_setting1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_reset_setting\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.replicated_table_for_reset_setting1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_reset_setting\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192, merge_with_ttl_timeout = 100 +CREATE TABLE default.replicated_table_for_reset_setting2\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_reset_setting\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192, merge_with_ttl_timeout = 200 +CREATE TABLE default.replicated_table_for_reset_setting1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_reset_setting\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192, merge_with_ttl_timeout = 100 +CREATE TABLE default.replicated_table_for_reset_setting2\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_reset_setting\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192, merge_with_ttl_timeout = 200 +CREATE TABLE default.replicated_table_for_reset_setting1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_reset_setting\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.replicated_table_for_reset_setting2\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00980_default/replicated_table_for_reset_setting\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql b/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql index 6ad8860227d..25516f577ec 100644 --- a/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql +++ b/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql @@ -6,12 +6,12 @@ SET replication_alter_partitions_sync = 2; CREATE TABLE replicated_table_for_alter1 ( id UInt64, Data String -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00980/replicated_table_for_alter', '1') ORDER BY id; +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00980_{database}/replicated_table_for_alter', '1') ORDER BY id; CREATE TABLE replicated_table_for_alter2 ( id UInt64, Data String -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00980/replicated_table_for_alter', '2') ORDER BY id; +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00980_{database}/replicated_table_for_alter', '2') ORDER BY id; SHOW CREATE TABLE replicated_table_for_alter1; @@ -76,12 +76,12 @@ SET replication_alter_partitions_sync = 2; CREATE TABLE replicated_table_for_reset_setting1 ( id UInt64, Data String -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00980/replicated_table_for_reset_setting', '1') ORDER BY id; +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00980_{database}/replicated_table_for_reset_setting', '1') ORDER BY id; CREATE TABLE replicated_table_for_reset_setting2 ( id UInt64, Data String -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00980/replicated_table_for_reset_setting', '2') ORDER BY id; +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00980_{database}/replicated_table_for_reset_setting', '2') ORDER BY id; SHOW CREATE TABLE replicated_table_for_reset_setting1; SHOW CREATE TABLE replicated_table_for_reset_setting2; diff --git a/tests/queries/0_stateless/01149_zookeeper_mutation_stuck_after_replace_partition.sql b/tests/queries/0_stateless/01149_zookeeper_mutation_stuck_after_replace_partition.sql index 18508635cf9..ca8f70b3cf4 100644 --- a/tests/queries/0_stateless/01149_zookeeper_mutation_stuck_after_replace_partition.sql +++ b/tests/queries/0_stateless/01149_zookeeper_mutation_stuck_after_replace_partition.sql @@ -1,3 +1,4 @@ +set send_logs_level='error'; drop table if exists mt; drop table if exists rmt sync; diff --git a/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh b/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh index 7347fc5d626..483979d00db 100755 --- a/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh +++ b/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh @@ -14,7 +14,7 @@ function run_until_out_contains() PATTERN=$1 shift - while true + for _ in {1..20} do "$@" > "${CLICKHOUSE_TMP}/out" 2>&1 if grep -q "$PATTERN" "${CLICKHOUSE_TMP}/out" diff --git a/tests/queries/0_stateless/01294_system_distributed_on_cluster.sql b/tests/queries/0_stateless/01294_system_distributed_on_cluster.sql index d56bddba3c6..525974e78ba 100644 --- a/tests/queries/0_stateless/01294_system_distributed_on_cluster.sql +++ b/tests/queries/0_stateless/01294_system_distributed_on_cluster.sql @@ -3,6 +3,7 @@ -- quirk for ON CLUSTER does not uses currentDatabase() drop database if exists db_01294; create database db_01294; +set distributed_ddl_output_mode='throw'; drop table if exists db_01294.dist_01294; create table db_01294.dist_01294 as system.one engine=Distributed(test_shard_localhost, system, one); diff --git a/tests/queries/0_stateless/01415_sticking_mutations.sh b/tests/queries/0_stateless/01415_sticking_mutations.sh index 9bd0a6eeebf..2e86b6d972d 100755 --- a/tests/queries/0_stateless/01415_sticking_mutations.sh +++ b/tests/queries/0_stateless/01415_sticking_mutations.sh @@ -33,9 +33,10 @@ function check_sticky_mutations() query_result=$($CLICKHOUSE_CLIENT --query="$check_query" 2>&1) - while [ "$query_result" == "0" ] + for _ in {1..50} do query_result=$($CLICKHOUSE_CLIENT --query="$check_query" 2>&1) + if ! [ "$query_result" == "0" ]; then break; fi sleep 0.5 done ##### wait mutation to start ##### diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index bb1442b1ff9..f4aeffdc50d 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -119,16 +119,14 @@ "memory_tracking", "memory_usage", "live_view", - "01181_db_atomic_drop_on_cluster", - "01175_distributed_ddl_output_mode", - "01415_sticking_mutations", - "00980_zookeeper_merge_tree_alter_settings", "01148_zookeeper_path_macros_unfolding", - "01294_system_distributed_on_cluster", "01269_create_with_null", "01451_replicated_detach_drop_and_quorum", "01188_attach_table_from_path", - "01149_zookeeper_mutation_stuck_after_replace_partition", + /// ON CLUSTER is not allowed + "01181_db_atomic_drop_on_cluster", + "01175_distributed_ddl_output_mode", + "01415_sticking_mutations", /// user_files "01721_engine_file_truncate_on_insert", /// Fails due to additional replicas or shards @@ -158,6 +156,8 @@ "00152_insert_different_granularity", "00054_merge_tree_partitions", "01781_merge_tree_deduplication", + "00980_zookeeper_merge_tree_alter_settings", + "00980_merge_alter_settings", /// Old syntax is not allowed "01062_alter_on_mutataion_zookeeper", "00925_zookeeper_empty_replicated_merge_tree_optimize_final", @@ -175,10 +175,11 @@ /// Does not support renaming of multiple tables in single query "00634_rename_view", "00140_rename", + /// Different query_id + "01943_query_id_check", /// Requires investigation "00953_zookeeper_suetin_deduplication_bug", "01783_http_chunk_size", - "01943_query_id_check", "00166_explain_estimate" ], "polymorphic-parts": [ From d5a5ba8f0d64875d42ad115642d48dd552ebde45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Aug 2021 15:19:40 +0200 Subject: [PATCH 068/198] Stateless tests: Keep an DNS error free log --- .../queries/0_stateless/00965_shard_unresolvable_addresses.sql | 3 +++ .../0_stateless/01083_expressions_in_engine_arguments.sql | 3 +++ .../0_stateless/01372_remote_table_function_empty_table.sql | 3 +++ 3 files changed, 9 insertions(+) diff --git a/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql b/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql index 6d57cd0447d..7026466fbab 100644 --- a/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql +++ b/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql @@ -1,2 +1,5 @@ SELECT count() FROM remote('127.0.0.1,localhos', system.one); -- { serverError 198 } SELECT count() FROM remote('127.0.0.1|localhos', system.one); + +-- Clear cache to avoid future errors in the logs +SYSTEM DROP DNS CACHE diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql index e8ea12f9f9f..4a3ac513e36 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql @@ -68,3 +68,6 @@ INSERT INTO buffer VALUES (1); -- | |-> file (1) -- |-> remote(127.0.0.2) --> ... SELECT sum(n) from rich_syntax; + +-- Clear cache to avoid future errors in the logs +SYSTEM DROP DNS CACHE diff --git a/tests/queries/0_stateless/01372_remote_table_function_empty_table.sql b/tests/queries/0_stateless/01372_remote_table_function_empty_table.sql index 698c323d73f..4153dc632f3 100644 --- a/tests/queries/0_stateless/01372_remote_table_function_empty_table.sql +++ b/tests/queries/0_stateless/01372_remote_table_function_empty_table.sql @@ -1 +1,4 @@ SELECT * FROM remote('127..2', 'a.'); -- { serverError 36 } + +-- Clear cache to avoid future errors in the logs +SYSTEM DROP DNS CACHE From 294695bb7dc3fde2f8343d85e5ea7c0b960da545 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 2 Aug 2021 07:18:51 +0000 Subject: [PATCH 069/198] Review fixes --- programs/library-bridge/HandlerFactory.cpp | 2 +- programs/library-bridge/Handlers.cpp | 24 ++++++++------ programs/library-bridge/Handlers.h | 4 +-- .../SharedLibraryHandlerFactory.cpp | 6 ---- src/Bridge/IBridgeHelper.cpp | 8 ++--- src/Bridge/IBridgeHelper.h | 7 +++-- src/Bridge/LibraryBridgeHelper.cpp | 31 ++++++++++--------- src/Bridge/LibraryBridgeHelper.h | 6 ++-- src/Bridge/XDBCBridgeHelper.h | 2 +- src/Dictionaries/LibraryDictionarySource.cpp | 13 +++----- src/Dictionaries/LibraryDictionarySource.h | 2 +- .../configs/config.d/config.xml | 4 +++ 12 files changed, 56 insertions(+), 53 deletions(-) diff --git a/programs/library-bridge/HandlerFactory.cpp b/programs/library-bridge/HandlerFactory.cpp index 3e31f298fca..43087082c46 100644 --- a/programs/library-bridge/HandlerFactory.cpp +++ b/programs/library-bridge/HandlerFactory.cpp @@ -13,7 +13,7 @@ namespace DB LOG_DEBUG(log, "Request URI: {}", uri.toString()); if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET) - return std::make_unique(keep_alive_timeout, getContext()); + return std::make_unique(keep_alive_timeout, getContext()); if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) return std::make_unique(keep_alive_timeout, getContext()); diff --git a/programs/library-bridge/Handlers.cpp b/programs/library-bridge/Handlers.cpp index 0b49a801ca3..2b6d0057bb2 100644 --- a/programs/library-bridge/Handlers.cpp +++ b/programs/library-bridge/Handlers.cpp @@ -17,6 +17,12 @@ namespace DB { + +namespace ErrorCodes +{ + extern const int BAD_REQUEST_PARAMETER; +} + namespace { void processError(HTTPServerResponse & response, const std::string & message) @@ -190,7 +196,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe { auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); if (!library_handler) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found dictionary with id: {}", dictionary_id); + throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id); bool res = library_handler->isModified(); writeStringBinary(std::to_string(res), out); @@ -199,7 +205,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe { auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); if (!library_handler) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found dictionary with id: {}", dictionary_id); + throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id); bool res = library_handler->supportsSelectiveLoad(); writeStringBinary(std::to_string(res), out); @@ -208,7 +214,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe { auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); if (!library_handler) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found dictionary with id: {}", dictionary_id); + throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id); const auto & sample_block = library_handler->getSampleBlock(); LOG_DEBUG(log, "Calling loadAll() for dictionary id: {}", dictionary_id); @@ -226,7 +232,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); if (!library_handler) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found dictionary with id: {}", dictionary_id); + throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id); const auto & sample_block = library_handler->getSampleBlock(); LOG_DEBUG(log, "Calling loadIds() for dictionary id: {}", dictionary_id); @@ -265,7 +271,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); if (!library_handler) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found dictionary with id: {}", dictionary_id); + throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id); const auto & sample_block = library_handler->getSampleBlock(); LOG_DEBUG(log, "Calling loadKeys() for dictionary id: {}", dictionary_id); @@ -304,7 +310,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe } -void PingHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) +void LibraryExistsHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { try { @@ -313,7 +319,7 @@ void PingHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse if (!params.has("dictionary_id")) { - processError(response, "No 'dictionary_id in request URL"); + processError(response, "No 'dictionary_id' in request URL"); return; } @@ -321,9 +327,9 @@ void PingHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); String res; if (library_handler) - res = "dictionary=1"; + res = "1"; else - res = "dictionary=0"; + res = "0"; setResponseDefaultHeaders(response, keep_alive_timeout); LOG_TRACE(log, "Senging ping response: {} (dictionary id: {})", res, dictionary_id); diff --git a/programs/library-bridge/Handlers.h b/programs/library-bridge/Handlers.h index a8e4ba0dd2a..58af24b06d1 100644 --- a/programs/library-bridge/Handlers.h +++ b/programs/library-bridge/Handlers.h @@ -39,10 +39,10 @@ private: }; -class PingHandler : public HTTPRequestHandler, WithContext +class LibraryExistsHandler : public HTTPRequestHandler, WithContext { public: - explicit PingHandler(size_t keep_alive_timeout_, ContextPtr context_) + explicit LibraryExistsHandler(size_t keep_alive_timeout_, ContextPtr context_) : WithContext(context_) , keep_alive_timeout(keep_alive_timeout_) , log(&Poco::Logger::get("LibraryRequestHandler")) diff --git a/programs/library-bridge/SharedLibraryHandlerFactory.cpp b/programs/library-bridge/SharedLibraryHandlerFactory.cpp index 900981191c8..a9358ca552a 100644 --- a/programs/library-bridge/SharedLibraryHandlerFactory.cpp +++ b/programs/library-bridge/SharedLibraryHandlerFactory.cpp @@ -4,12 +4,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; - extern const int LOGICAL_ERROR; -} - SharedLibraryHandlerPtr SharedLibraryHandlerFactory::get(const std::string & dictionary_id) { std::lock_guard lock(mutex); diff --git a/src/Bridge/IBridgeHelper.cpp b/src/Bridge/IBridgeHelper.cpp index a90688cc8b5..5c884a2ca3d 100644 --- a/src/Bridge/IBridgeHelper.cpp +++ b/src/Bridge/IBridgeHelper.cpp @@ -33,9 +33,9 @@ Poco::URI IBridgeHelper::getPingURI() const } -void IBridgeHelper::startBridgeSync() const +void IBridgeHelper::startBridgeSync() { - if (!checkBridgeIsRunning()) + if (!bridgeHandShake()) { LOG_TRACE(getLog(), "{} is not running, will try to start it", serviceAlias()); startBridge(startBridgeCommand()); @@ -49,7 +49,7 @@ void IBridgeHelper::startBridgeSync() const ++counter; LOG_TRACE(getLog(), "Checking {} is running, try {}", serviceAlias(), counter); - if (checkBridgeIsRunning()) + if (bridgeHandShake()) { started = true; break; @@ -66,7 +66,7 @@ void IBridgeHelper::startBridgeSync() const } -std::unique_ptr IBridgeHelper::startBridgeCommand() const +std::unique_ptr IBridgeHelper::startBridgeCommand() { if (startBridgeManually()) throw Exception(serviceAlias() + " is not running. Please, start it manually", ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING); diff --git a/src/Bridge/IBridgeHelper.h b/src/Bridge/IBridgeHelper.h index 6a9c19da68f..0537658663d 100644 --- a/src/Bridge/IBridgeHelper.h +++ b/src/Bridge/IBridgeHelper.h @@ -35,10 +35,11 @@ public: Poco::URI getPingURI() const; - void startBridgeSync() const; + void startBridgeSync(); protected: - virtual bool checkBridgeIsRunning() const = 0; + /// Check bridge is running. Can also check something else in the mean time. + virtual bool bridgeHandShake() = 0; /// clickhouse-odbc-bridge, clickhouse-library-bridge virtual String serviceAlias() const = 0; @@ -63,7 +64,7 @@ protected: private: - std::unique_ptr startBridgeCommand() const; + std::unique_ptr startBridgeCommand(); }; } diff --git a/src/Bridge/LibraryBridgeHelper.cpp b/src/Bridge/LibraryBridgeHelper.cpp index b39e3f01903..9341ddda922 100644 --- a/src/Bridge/LibraryBridgeHelper.cpp +++ b/src/Bridge/LibraryBridgeHelper.cpp @@ -70,7 +70,7 @@ void LibraryBridgeHelper::startBridge(std::unique_ptr cmd) const } -bool LibraryBridgeHelper::checkBridgeIsRunning() const +bool LibraryBridgeHelper::bridgeHandShake() { String result; try @@ -89,13 +89,11 @@ bool LibraryBridgeHelper::checkBridgeIsRunning() const * 1. It is dictionary source creation and initialization of library handler on bridge side did not happen yet. * 2. Bridge crashed or restarted for some reason while server did not. **/ - static constexpr auto dictionary_check = "dictionary="; - if (result.size() != (std::strlen(dictionary_check) + 1)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {}. Check bridge and server have the same version.", - result, std::strlen(dictionary_check)); + if (result.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {}. Check bridge and server have the same version.", result); UInt8 dictionary_id_exists; - auto parsed = tryParse(dictionary_id_exists, result.substr(std::strlen(dictionary_check))); + auto parsed = tryParse(dictionary_id_exists, result); if (!parsed || (dictionary_id_exists != 0 && dictionary_id_exists != 1)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {} ({}). Check bridge and server have the same version.", result, parsed ? toString(dictionary_id_exists) : "failed to parse"); @@ -106,13 +104,16 @@ bool LibraryBridgeHelper::checkBridgeIsRunning() const if (dictionary_id_exists && !library_initialized) throw Exception(ErrorCodes::LOGICAL_ERROR, "Library was not initialized, but bridge responded to already have dictionary id: {}", dictionary_id); + /// Here we want to say bridge to recreate a new library handler for current dictionary, + /// because it responded to have lost it, but we know that it has already been created. (It is a direct result of bridge crash). if (!dictionary_id_exists && library_initialized) { LOG_WARNING(log, "Library bridge does not have library handler with dictionaty id: {}. It will be reinitialized.", dictionary_id); bool reinitialized = false; try { - reinitialized = initLibrary(false); + auto uri = createRequestURI(LIB_NEW_METHOD); + reinitialized = executeRequest(uri, getInitLibraryCallback()); } catch (...) { @@ -148,13 +149,12 @@ ReadWriteBufferFromHTTP::OutStreamCallback LibraryBridgeHelper::getInitLibraryCa } -bool LibraryBridgeHelper::initLibrary(bool check_bridge) const +bool LibraryBridgeHelper::initLibrary() { - /// Do not check if we call initLibrary from checkBridgeSync. - if (check_bridge) - startBridgeSync(); + startBridgeSync(); auto uri = createRequestURI(LIB_NEW_METHOD); - return executeRequest(uri, getInitLibraryCallback()); + library_initialized = executeRequest(uri, getInitLibraryCallback()); + return library_initialized; } @@ -163,7 +163,10 @@ bool LibraryBridgeHelper::cloneLibrary(const Field & other_dictionary_id) startBridgeSync(); auto uri = createRequestURI(LIB_CLONE_METHOD); uri.addQueryParameter("from_dictionary_id", toString(other_dictionary_id)); - return executeRequest(uri, getInitLibraryCallback()); + /// We also pass initialization settings in order to create a library handler + /// in case from_dictionary_id does not exist in bridge side (possible in case of bridge crash). + library_initialized = executeRequest(uri, getInitLibraryCallback()); + return library_initialized; } @@ -171,7 +174,7 @@ bool LibraryBridgeHelper::removeLibrary() { /// Do not force bridge restart if it is not running in case of removeLibrary /// because in this case after restart it will not have this dictionaty id in memory anyway. - if (checkBridgeIsRunning()) + if (bridgeHandShake()) { auto uri = createRequestURI(LIB_DELETE_METHOD); return executeRequest(uri); diff --git a/src/Bridge/LibraryBridgeHelper.h b/src/Bridge/LibraryBridgeHelper.h index e4fe8e06965..6615162efe7 100644 --- a/src/Bridge/LibraryBridgeHelper.h +++ b/src/Bridge/LibraryBridgeHelper.h @@ -26,7 +26,7 @@ public: LibraryBridgeHelper(ContextPtr context_, const Block & sample_block, const Field & dictionary_id_, const LibraryInitData & library_data_); - bool initLibrary(bool check_bridge = true) const; + bool initLibrary(); bool cloneLibrary(const Field & other_dictionary_id); @@ -48,10 +48,8 @@ public: LibraryInitData getLibraryData() const { return library_data; } - void setInitialized() { library_initialized = true; } - protected: - bool checkBridgeIsRunning() const override; + bool bridgeHandShake() override; void startBridge(std::unique_ptr cmd) const override; diff --git a/src/Bridge/XDBCBridgeHelper.h b/src/Bridge/XDBCBridgeHelper.h index 13f0f528c8f..b3613c381d0 100644 --- a/src/Bridge/XDBCBridgeHelper.h +++ b/src/Bridge/XDBCBridgeHelper.h @@ -74,7 +74,7 @@ public: } protected: - bool checkBridgeIsRunning() const override + bool bridgeHandShake() override { try { diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index 273f9d3c3b4..4747d3f1216 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -41,6 +41,9 @@ LibraryDictionarySource::LibraryDictionarySource( , sample_block{sample_block_} , context(Context::createCopy(context_)) { + if (fs::path(path).is_relative()) + path = fs::canonical(path); + if (created_from_ddl && !pathStartsWith(path, context->getDictionariesLibPath())) throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", path, context->getDictionariesLibPath()); @@ -58,10 +61,7 @@ LibraryDictionarySource::LibraryDictionarySource( bridge_helper = std::make_shared(context, description.sample_block, dictionary_id, library_data); - bool initialized = bridge_helper->initLibrary(); - if (initialized) - bridge_helper->setInitialized(); - else + if (!bridge_helper->initLibrary()) throw Exception(ErrorCodes::EXTERNAL_LIBRARY_ERROR, "Failed to create shared library from path: {}", path); } @@ -91,10 +91,7 @@ LibraryDictionarySource::LibraryDictionarySource(const LibraryDictionarySource & , description{other.description} { bridge_helper = std::make_shared(context, description.sample_block, dictionary_id, other.bridge_helper->getLibraryData()); - bool cloned = bridge_helper->cloneLibrary(other.dictionary_id); - if (cloned) - bridge_helper->setInitialized(); - else + if (!bridge_helper->cloneLibrary(other.dictionary_id)) throw Exception(ErrorCodes::EXTERNAL_LIBRARY_ERROR, "Failed to clone library"); } diff --git a/src/Dictionaries/LibraryDictionarySource.h b/src/Dictionaries/LibraryDictionarySource.h index 18a2d1b97b2..b5cf4f68b05 100644 --- a/src/Dictionaries/LibraryDictionarySource.h +++ b/src/Dictionaries/LibraryDictionarySource.h @@ -80,7 +80,7 @@ private: const DictionaryStructure dict_struct; const std::string config_prefix; - const std::string path; + std::string path; const Field dictionary_id; Block sample_block; diff --git a/tests/integration/test_library_bridge/configs/config.d/config.xml b/tests/integration/test_library_bridge/configs/config.d/config.xml index 9bea75fbb6f..7811c1e26b7 100644 --- a/tests/integration/test_library_bridge/configs/config.d/config.xml +++ b/tests/integration/test_library_bridge/configs/config.d/config.xml @@ -8,5 +8,9 @@ 10 /var/log/clickhouse-server/stderr.log /var/log/clickhouse-server/stdout.log + + /var/log/clickhouse-server/clickhouse-library-bridge.log + /var/log/clickhouse-server/clickhouse-library-bridge.err.log + trace From 7a2e7117656afc68ab6c6ae8ef41b0a46644d94a Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 2 Aug 2021 17:48:53 +0300 Subject: [PATCH 070/198] Try update AMQP-CPP --- contrib/AMQP-CPP | 2 +- contrib/amqpcpp-cmake/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/AMQP-CPP b/contrib/AMQP-CPP index 03781aaff0f..dda64f5debc 160000 --- a/contrib/AMQP-CPP +++ b/contrib/AMQP-CPP @@ -1 +1 @@ -Subproject commit 03781aaff0f10ef41f902b8cf865fe0067180c10 +Subproject commit dda64f5debceb15ad7e521af3e7d78e5d929b3d6 diff --git a/contrib/amqpcpp-cmake/CMakeLists.txt b/contrib/amqpcpp-cmake/CMakeLists.txt index 4e8342af125..05b2c3849a2 100644 --- a/contrib/amqpcpp-cmake/CMakeLists.txt +++ b/contrib/amqpcpp-cmake/CMakeLists.txt @@ -10,7 +10,7 @@ set (SRCS "${LIBRARY_DIR}/src/deferredconsumer.cpp" "${LIBRARY_DIR}/src/deferredextreceiver.cpp" "${LIBRARY_DIR}/src/deferredget.cpp" - "${LIBRARY_DIR}/src/deferredpublisher.cpp" + "${LIBRARY_DIR}/src/deferredrecall.cpp" "${LIBRARY_DIR}/src/deferredreceiver.cpp" "${LIBRARY_DIR}/src/field.cpp" "${LIBRARY_DIR}/src/flags.cpp" From 71e1c82f8772e6d1878739f7ad8c356ad909f2b8 Mon Sep 17 00:00:00 2001 From: OmarBazaraa Date: Mon, 2 Aug 2021 15:16:39 +0000 Subject: [PATCH 071/198] Refactor --- src/Storages/StorageMongoDBSocketFactory.cpp | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/Storages/StorageMongoDBSocketFactory.cpp b/src/Storages/StorageMongoDBSocketFactory.cpp index 5f8fcab369f..9a2b120e9ed 100644 --- a/src/Storages/StorageMongoDBSocketFactory.cpp +++ b/src/Storages/StorageMongoDBSocketFactory.cpp @@ -3,7 +3,6 @@ #include #if !defined(ARCADIA_BUILD) -# include "config_core.h" # include #endif @@ -14,14 +13,6 @@ # include #endif -#ifdef __clang__ -# pragma clang diagnostic ignored "-Wunused-parameter" -#endif - -#if USE_EMBEDDED_COMPILER -# pragma GCC diagnostic ignored "-Wunused-parameter" -#endif - namespace DB { @@ -47,7 +38,7 @@ Poco::Net::StreamSocket StorageMongoDBSocketFactory::createPlainSocket(const std } -Poco::Net::StreamSocket StorageMongoDBSocketFactory::createSecureSocket(const std::string & host, int port, Poco::Timespan connectTimeout) +Poco::Net::StreamSocket StorageMongoDBSocketFactory::createSecureSocket(const std::string & host [[maybe_unused]], int port [[maybe_unused]], Poco::Timespan connectTimeout [[maybe_unused]]) { #if USE_SSL Poco::Net::SocketAddress address(host, port); From 0f6fff47d35ccbcd3bccf2a5df76096d55cebe20 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> Date: Mon, 2 Aug 2021 18:54:24 +0300 Subject: [PATCH 072/198] Apply suggestions from code review Co-authored-by: Alexey Boykov <33257111+mathalex@users.noreply.github.com> --- docs/en/sql-reference/functions/nlp-functions.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/sql-reference/functions/nlp-functions.md b/docs/en/sql-reference/functions/nlp-functions.md index 5edaf8486b9..1654771574b 100644 --- a/docs/en/sql-reference/functions/nlp-functions.md +++ b/docs/en/sql-reference/functions/nlp-functions.md @@ -6,7 +6,7 @@ toc_title: NLP # [experimental] Natural Language Processing functions {#nlp-functions} !!! warning "Warning" - This is an experimental feature that is currently in development and is not ready for general use. It will change in unpredictable backwards-incompatible ways in the future releases. Set `allow_experimental_nlp_functions = 1` to enable it. + This is an experimental feature that is currently in development and is not ready for general use. It will change in unpredictable backwards-incompatible ways in future releases. Set `allow_experimental_nlp_functions = 1` to enable it. ## stem {#stem} @@ -41,7 +41,7 @@ Result: ## lemmatize {#lemmatize} -Performs lemmatization on a given word. Needs dictionaries to operate. Dictionaries can be obtained [here](https://github.com/vpodpecan/lemmagen3/tree/master/src/lemmagen3/models). +Performs lemmatization on a given word. Needs dictionaries to operate, which can be obtained [here](https://github.com/vpodpecan/lemmagen3/tree/master/src/lemmagen3/models). **Syntax** @@ -84,9 +84,9 @@ Configuration: Finds synonyms to a given word. There are two types of synonym extensions: `plain` and `wordnet`. -With `plain` extension type we need to provide a path to simple text file, where each line corresponds to certain synonym set. Words in this line must be separated with space or tab characters. +With the `plain` extension type we need to provide a path to a simple text file, where each line corresponds to a certain synonym set. Words in this line must be separated with space or tab characters. -With `wordnet` extension type we need to provide a path to directory with WordNet thesaurus in it. Thesaurus must contain WordNet sense index. +With the `wordnet` extension type we need to provide a path to a directory with WordNet thesaurus in it. Thesaurus must contain a WordNet sense index. **Syntax** @@ -129,4 +129,4 @@ Configuration: en/ -``` \ No newline at end of file +``` From c645dc9d7d7314429b1f895600a3a0da619af719 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 2 Aug 2021 19:23:13 +0300 Subject: [PATCH 073/198] fix --- contrib/amqpcpp-cmake/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/amqpcpp-cmake/CMakeLists.txt b/contrib/amqpcpp-cmake/CMakeLists.txt index 05b2c3849a2..5637db4cf41 100644 --- a/contrib/amqpcpp-cmake/CMakeLists.txt +++ b/contrib/amqpcpp-cmake/CMakeLists.txt @@ -15,6 +15,7 @@ set (SRCS "${LIBRARY_DIR}/src/field.cpp" "${LIBRARY_DIR}/src/flags.cpp" "${LIBRARY_DIR}/src/linux_tcp/openssl.cpp" + "${LIBRARY_DIR}/src/linux_tcp/sslerrorprinter.cpp" "${LIBRARY_DIR}/src/linux_tcp/tcpconnection.cpp" "${LIBRARY_DIR}/src/inbuffer.cpp" "${LIBRARY_DIR}/src/receivedframe.cpp" From 31d027b8d0790a04938f2b6a6de1fbb47b8b6331 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 2 Aug 2021 19:33:15 +0300 Subject: [PATCH 074/198] Try update arrow --- contrib/arrow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/arrow b/contrib/arrow index debf751a129..078e21bad34 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit debf751a129bdda9ff4d1e895e08957ff77000a1 +Subproject commit 078e21bad344747b7656ef2d7a4f7410a0a303eb From 36c6d7e1c03c9700fa8f879e530a19957cb2272f Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Mon, 2 Aug 2021 20:13:35 +0300 Subject: [PATCH 075/198] Update docs/en/sql-reference/statements/select/order-by.md Co-authored-by: Alexey Boykov <33257111+mathalex@users.noreply.github.com> --- docs/en/sql-reference/statements/select/order-by.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index 66dac8620fe..4c1c7e20b73 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -280,7 +280,7 @@ To fill multiple columns, add `WITH FILL` modifier with optional parameters afte ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr] ``` -`WITH FILL` can be applied for fields with Numeric (all kind of float, decimal, int) or Date/DateTime types. When applied for `String` fields, missed values are filled with empty strings. +`WITH FILL` can be applied for fields with Numeric (all kinds of float, decimal, int) or Date/DateTime types. When applied for `String` fields, missed values are filled with empty strings. When `FROM const_expr` not defined sequence of filling use minimal `expr` field value from `ORDER BY`. When `TO const_expr` not defined sequence of filling use maximum `expr` field value from `ORDER BY`. When `STEP const_numeric_expr` defined then `const_numeric_expr` interprets `as is` for numeric types as `days` for Date type and as `seconds` for DateTime type. From 1a4d9fec9bb529fa1581c2e6a7167817f21d4aff Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Mon, 2 Aug 2021 20:13:48 +0300 Subject: [PATCH 076/198] Update docs/en/sql-reference/statements/select/order-by.md Co-authored-by: Alexey Boykov <33257111+mathalex@users.noreply.github.com> --- docs/en/sql-reference/statements/select/order-by.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index 4c1c7e20b73..e6d4b8dec55 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -286,7 +286,7 @@ When `TO const_expr` not defined sequence of filling use maximum `expr` field va When `STEP const_numeric_expr` defined then `const_numeric_expr` interprets `as is` for numeric types as `days` for Date type and as `seconds` for DateTime type. When `STEP const_numeric_expr` omitted then sequence of filling use `1.0` for numeric type, `1 day` for Date type and `1 second` for DateTime type. -Example of query without `WITH FILL`: +Example of a query without `WITH FILL`: ``` sql SELECT n, source FROM ( From c5df30594df27d6f4de572cf04b27cf2e258e1aa Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Mon, 2 Aug 2021 20:13:58 +0300 Subject: [PATCH 077/198] Update docs/en/sql-reference/statements/select/order-by.md Co-authored-by: Alexey Boykov <33257111+mathalex@users.noreply.github.com> --- docs/en/sql-reference/statements/select/order-by.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index e6d4b8dec55..bc22356cb5d 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -334,7 +334,7 @@ Result: └─────┴──────────┘ ``` -For the case with multiple fields `ORDER BY field2 WITH FILL, field1 WITH FILL` order of filling will follow the order of fields in `ORDER BY` clause. +For the case with multiple fields `ORDER BY field2 WITH FILL, field1 WITH FILL` order of filling will follow the order of fields in the `ORDER BY` clause. Example: From e03539fdd4cccc55719151a6d4e68f33b240e11d Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Mon, 2 Aug 2021 20:14:09 +0300 Subject: [PATCH 078/198] Update docs/en/sql-reference/statements/select/order-by.md Co-authored-by: Alexey Boykov <33257111+mathalex@users.noreply.github.com> --- docs/en/sql-reference/statements/select/order-by.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index bc22356cb5d..23a108bcf81 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -364,7 +364,7 @@ Result: └────────────┴────────────┴──────────┘ ``` -Field `d1` does not fill in and use the default value cause we do not have repeated values for `d2` value, and sequence for `d1` can’t be properly calculated. +Field `d1` does not fill in and use the default value cause we do not have repeated values for `d2` value, and the sequence for `d1` can’t be properly calculated. The following query with a changed field in `ORDER BY`: From d10835f22360a6c835546bc74a5236b0dc4da331 Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Mon, 2 Aug 2021 20:14:22 +0300 Subject: [PATCH 079/198] Update docs/en/sql-reference/statements/select/order-by.md Co-authored-by: Alexey Boykov <33257111+mathalex@users.noreply.github.com> --- docs/en/sql-reference/statements/select/order-by.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index 23a108bcf81..156f68935b5 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -366,7 +366,7 @@ Result: Field `d1` does not fill in and use the default value cause we do not have repeated values for `d2` value, and the sequence for `d1` can’t be properly calculated. -The following query with a changed field in `ORDER BY`: +The following query with the changed field in `ORDER BY`: ``` sql SELECT From 3b4eb2d769905ee2194b90a284bef5d002acb382 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 3 Aug 2021 00:53:43 +0300 Subject: [PATCH 080/198] Fixed code review issues --- .../external-dictionaries/external-dicts-dict-layout.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index 81b3df6742f..ffa0fd6f29e 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -280,12 +280,11 @@ If keys are not found in dictionary, then update cache task is created and added For cache dictionaries, the expiration [lifetime](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md) of data in the cache can be set. If more time than `lifetime` has passed since loading the data in a cell, the cell’s value is not used and key becomes expired, and it is re-requested the next time it needs to be used this behaviour can be configured with setting `allow_read_expired_keys`. This is the least effective of all the ways to store dictionaries. The speed of the cache depends strongly on correct settings and the usage scenario. A cache type dictionary performs well only when the hit rates are high enough (recommended 99% and higher). You can view the average hit rate in the `system.dictionaries` table. -If setting `allow_read_expired_keys` is set to 1, by default 0. Then dictionary can support asynchronous updates. If client requests keys and all of them are in cache, but some of them are expired, then dictionary will return expired keys for client and request them asynchronously from source. +If setting `allow_read_expired_keys` is set to 1, by default 0. Then dictionary can support asynchronous updates. If a client requests keys and all of them are in cache, but some of them are expired, then dictionary will return expired keys for a client and request them asynchronously from the source. To improve cache performance, use a subquery with `LIMIT`, and call the function with the dictionary externally. -Supported [sources](../../../ -sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md): MySQL, ClickHouse, executable, HTTP. +Supported [sources](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md): MySQL, ClickHouse, executable, HTTP. Example of settings: From 40a3c8281a9c8b8a7e1d18fb2fc1a55dd4144502 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 31 Jul 2021 16:58:54 +0300 Subject: [PATCH 081/198] Fix synchronization in GRPCServer --- src/Server/GRPCServer.cpp | 70 +++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index ac59f82d419..b90b0c33f17 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -480,6 +481,40 @@ namespace }; + /// A boolean state protected by mutex able to wait until other thread sets it to a specific value. + class BoolState + { + public: + explicit BoolState(bool initial_value) : value(initial_value) {} + + bool get() const + { + std::lock_guard lock{mutex}; + return value; + } + + void set(bool new_value) + { + std::lock_guard lock{mutex}; + if (value == new_value) + return; + value = new_value; + changed.notify_all(); + } + + void wait(bool wanted_value) const + { + std::unique_lock lock{mutex}; + changed.wait(lock, [this, wanted_value]() { return value == wanted_value; }); + } + + private: + bool value; + mutable std::mutex mutex; + mutable std::condition_variable changed; + }; + + /// Handles a connection after a responder is started (i.e. after getting a new call). class Call { @@ -564,18 +599,15 @@ namespace UInt64 waited_for_client_writing = 0; /// The following fields are accessed both from call_thread and queue_thread. - std::atomic reading_query_info = false; + BoolState reading_query_info{false}; std::atomic failed_to_read_query_info = false; GRPCQueryInfo next_query_info_while_reading; std::atomic want_to_cancel = false; std::atomic check_query_info_contains_cancel_only = false; - std::atomic sending_result = false; + BoolState sending_result{false}; std::atomic failed_to_send_result = false; ThreadFromGlobalPool call_thread; - std::condition_variable read_finished; - std::condition_variable write_finished; - std::mutex dummy_mutex; /// Doesn't protect anything. }; Call::Call(CallType call_type_, std::unique_ptr responder_, IServer & iserver_, Poco::Logger * log_) @@ -610,6 +642,7 @@ namespace { try { + setThreadName("GRPCServerCall"); receiveQuery(); executeQuery(); processInput(); @@ -1230,8 +1263,7 @@ namespace { auto start_reading = [&] { - assert(!reading_query_info); - reading_query_info = true; + reading_query_info.set(true); responder->read(next_query_info_while_reading, [this](bool ok) { /// Called on queue_thread. @@ -1256,18 +1288,16 @@ namespace /// on queue_thread. failed_to_read_query_info = true; } - reading_query_info = false; - read_finished.notify_one(); + reading_query_info.set(false); }); }; auto finish_reading = [&] { - if (reading_query_info) + if (reading_query_info.get()) { Stopwatch client_writing_watch; - std::unique_lock lock{dummy_mutex}; - read_finished.wait(lock, [this] { return !reading_query_info; }); + reading_query_info.wait(false); waited_for_client_writing += client_writing_watch.elapsedNanoseconds(); } throwIfFailedToReadQueryInfo(); @@ -1430,11 +1460,10 @@ namespace /// Wait for previous write to finish. /// (gRPC doesn't allow to start sending another result while the previous is still being sending.) - if (sending_result) + if (sending_result.get()) { Stopwatch client_reading_watch; - std::unique_lock lock{dummy_mutex}; - write_finished.wait(lock, [this] { return !sending_result; }); + sending_result.wait(false); waited_for_client_reading += client_reading_watch.elapsedNanoseconds(); } throwIfFailedToSendResult(); @@ -1445,14 +1474,13 @@ namespace if (write_buffer) write_buffer->finalize(); - sending_result = true; + sending_result.set(true); auto callback = [this](bool ok) { /// Called on queue_thread. if (!ok) failed_to_send_result = true; - sending_result = false; - write_finished.notify_one(); + sending_result.set(false); }; Stopwatch client_reading_final_watch; @@ -1472,8 +1500,7 @@ namespace if (send_final_message) { /// Wait until the result is actually sent. - std::unique_lock lock{dummy_mutex}; - write_finished.wait(lock, [this] { return !sending_result; }); + sending_result.wait(false); waited_for_client_reading += client_reading_final_watch.elapsedNanoseconds(); throwIfFailedToSendResult(); LOG_TRACE(log, "Final result has been sent to the client"); @@ -1584,7 +1611,7 @@ private: { /// Called on call_thread. That's why we can't destroy the `call` right now /// (thread can't join to itself). Thus here we only move the `call` from - /// `current_call` to `finished_calls` and run() will actually destroy the `call`. + /// `current_calls` to `finished_calls` and run() will actually destroy the `call`. std::lock_guard lock{mutex}; auto it = current_calls.find(call); finished_calls.push_back(std::move(it->second)); @@ -1593,6 +1620,7 @@ private: void run() { + setThreadName("GRPCServerQueue"); while (true) { { From b541b5eca35b12b7419ffd27c6494cd4d473961d Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 2 Aug 2021 21:51:33 +0800 Subject: [PATCH 082/198] Normalize hostname in CI --- tests/clickhouse-test | 4 ++++ tests/queries/0_stateless/02001_hostname_test.reference | 2 ++ tests/queries/0_stateless/02001_hostname_test.sql | 2 ++ 3 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/02001_hostname_test.reference create mode 100644 tests/queries/0_stateless/02001_hostname_test.sql diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 362d1d225d9..d83b3f08c42 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -27,6 +27,7 @@ except ImportError: import random import string import multiprocessing +import socket from contextlib import closing USE_JINJA = True @@ -267,6 +268,9 @@ def run_single_test(args, ext, server_logs_level, client_options, case_file, std os.system("LC_ALL=C sed -i -e 's|/auto_{{shard}}||g' {file}".format(file=stdout_file)) os.system("LC_ALL=C sed -i -e 's|auto_{{replica}}||g' {file}".format(file=stdout_file)) + # Normalize hostname in stdout file. + os.system("LC_ALL=C sed -i -e 's/{hostname}/localhost/g' {file}".format(hostname=socket.gethostname(), file=stdout_file)) + stdout = open(stdout_file, 'rb').read() if os.path.exists(stdout_file) else b'' stdout = str(stdout, errors='replace', encoding='utf-8') stderr = open(stderr_file, 'rb').read() if os.path.exists(stderr_file) else b'' diff --git a/tests/queries/0_stateless/02001_hostname_test.reference b/tests/queries/0_stateless/02001_hostname_test.reference new file mode 100644 index 00000000000..da8a2d07eab --- /dev/null +++ b/tests/queries/0_stateless/02001_hostname_test.reference @@ -0,0 +1,2 @@ +localhost +localhost 2 diff --git a/tests/queries/0_stateless/02001_hostname_test.sql b/tests/queries/0_stateless/02001_hostname_test.sql new file mode 100644 index 00000000000..a8c7a8dab0c --- /dev/null +++ b/tests/queries/0_stateless/02001_hostname_test.sql @@ -0,0 +1,2 @@ +select hostname(); +select hostName() h, count() from cluster(test_cluster_two_shards, system.one) group by h; From d6844facaf63618894ef0429047976c6d1bc6ff1 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 3 Aug 2021 11:01:01 +0300 Subject: [PATCH 083/198] one more fix --- contrib/AMQP-CPP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/AMQP-CPP b/contrib/AMQP-CPP index dda64f5debc..8d83306aa0e 160000 --- a/contrib/AMQP-CPP +++ b/contrib/AMQP-CPP @@ -1 +1 @@ -Subproject commit dda64f5debceb15ad7e521af3e7d78e5d929b3d6 +Subproject commit 8d83306aa0ef33704ee951f789163ca90ab74402 From 7ad4a420035ac171f919b1107118d5cf59c502bc Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 3 Aug 2021 11:27:23 +0300 Subject: [PATCH 084/198] upadte CMakeLists --- contrib/arrow-cmake/CMakeLists.txt | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 2237be9913a..2c72055a3e7 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -194,9 +194,18 @@ set(ARROW_SRCS "${LIBRARY_DIR}/compute/cast.cc" "${LIBRARY_DIR}/compute/exec.cc" "${LIBRARY_DIR}/compute/function.cc" + "${LIBRARY_DIR}/compute/function_internal.cc" "${LIBRARY_DIR}/compute/kernel.cc" "${LIBRARY_DIR}/compute/registry.cc" + "${LIBRARY_DIR}/compute/exec/exec_plan.cc" + "${LIBRARY_DIR}/compute/exec/expression.cc" + "${LIBRARY_DIR}/compute/exec/key_compare.cc" + "${LIBRARY_DIR}/compute/exec/key_encode.cc" + "${LIBRARY_DIR}/compute/exec/key_hash.cc" + "${LIBRARY_DIR}/compute/exec/key_map.cc" + "${LIBRARY_DIR}/compute/exec/util.cc" + "${LIBRARY_DIR}/compute/kernels/aggregate_basic.cc" "${LIBRARY_DIR}/compute/kernels/aggregate_mode.cc" "${LIBRARY_DIR}/compute/kernels/aggregate_quantile.cc" @@ -207,6 +216,7 @@ set(ARROW_SRCS "${LIBRARY_DIR}/compute/kernels/scalar_arithmetic.cc" "${LIBRARY_DIR}/compute/kernels/scalar_boolean.cc" "${LIBRARY_DIR}/compute/kernels/scalar_cast_boolean.cc" + "${LIBRARY_DIR}/compute/kernels/scalar_cast_dictionary.cc" "${LIBRARY_DIR}/compute/kernels/scalar_cast_internal.cc" "${LIBRARY_DIR}/compute/kernels/scalar_cast_nested.cc" "${LIBRARY_DIR}/compute/kernels/scalar_cast_numeric.cc" @@ -214,15 +224,18 @@ set(ARROW_SRCS "${LIBRARY_DIR}/compute/kernels/scalar_cast_temporal.cc" "${LIBRARY_DIR}/compute/kernels/scalar_compare.cc" "${LIBRARY_DIR}/compute/kernels/scalar_fill_null.cc" + "${LIBRARY_DIR}/compute/kernels/scalar_if_else.cc" "${LIBRARY_DIR}/compute/kernels/scalar_nested.cc" "${LIBRARY_DIR}/compute/kernels/scalar_set_lookup.cc" "${LIBRARY_DIR}/compute/kernels/scalar_string.cc" + "${LIBRARY_DIR}/compute/kernels/scalar_temporal.cc" "${LIBRARY_DIR}/compute/kernels/scalar_validity.cc" + "${LIBRARY_DIR}/compute/kernels/util_internal.cc" "${LIBRARY_DIR}/compute/kernels/vector_hash.cc" "${LIBRARY_DIR}/compute/kernels/vector_nested.cc" + "${LIBRARY_DIR}/compute/kernels/vector_replace.cc" "${LIBRARY_DIR}/compute/kernels/vector_selection.cc" "${LIBRARY_DIR}/compute/kernels/vector_sort.cc" - "${LIBRARY_DIR}/compute/kernels/util_internal.cc" "${LIBRARY_DIR}/csv/chunker.cc" "${LIBRARY_DIR}/csv/column_builder.cc" @@ -231,6 +244,7 @@ set(ARROW_SRCS "${LIBRARY_DIR}/csv/options.cc" "${LIBRARY_DIR}/csv/parser.cc" "${LIBRARY_DIR}/csv/reader.cc" + "${LIBRARY_DIR}/csv/writer.cc" "${LIBRARY_DIR}/ipc/dictionary.cc" "${LIBRARY_DIR}/ipc/feather.cc" @@ -247,6 +261,7 @@ set(ARROW_SRCS "${LIBRARY_DIR}/io/interfaces.cc" "${LIBRARY_DIR}/io/memory.cc" "${LIBRARY_DIR}/io/slow.cc" + "${LIBRARY_DIR}/io/stdio.cc" "${LIBRARY_DIR}/io/transform.cc" "${LIBRARY_DIR}/tensor/coo_converter.cc" @@ -257,9 +272,9 @@ set(ARROW_SRCS "${LIBRARY_DIR}/util/bit_block_counter.cc" "${LIBRARY_DIR}/util/bit_run_reader.cc" "${LIBRARY_DIR}/util/bit_util.cc" - "${LIBRARY_DIR}/util/bitmap.cc" "${LIBRARY_DIR}/util/bitmap_builders.cc" "${LIBRARY_DIR}/util/bitmap_ops.cc" + "${LIBRARY_DIR}/util/bitmap.cc" "${LIBRARY_DIR}/util/bpacking.cc" "${LIBRARY_DIR}/util/cancel.cc" "${LIBRARY_DIR}/util/compression.cc" From 4f7e007d355bc76625af927f39e82e37082e4679 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 3 Aug 2021 14:23:46 +0800 Subject: [PATCH 085/198] Specialize date time comparision. --- src/Functions/FunctionsComparison.h | 37 +++++++++++++++++------ tests/performance/datetime_comparison.xml | 1 + 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 389b150e381..b9c7c211b74 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -1218,17 +1218,36 @@ public: { return res; } - else if ((isColumnedAsDecimal(left_type) || isColumnedAsDecimal(right_type)) - // Comparing Date and DateTime64 requires implicit conversion, - // otherwise Date is treated as number. - && !(date_and_datetime && (isDate(left_type) || isDate(right_type)))) + else if ((isColumnedAsDecimal(left_type) || isColumnedAsDecimal(right_type))) { - // compare - if (!allowDecimalComparison(left_type, right_type) && !date_and_datetime) - throw Exception("No operation " + getName() + " between " + left_type->getName() + " and " + right_type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + // Comparing Date and DateTime64 requires implicit conversion, + if (date_and_datetime && (isDate(left_type) || isDate(right_type))) + { + DataTypePtr common_type = getLeastSupertype({left_type, right_type}); + ColumnPtr c0_converted = castColumn(col_with_type_and_name_left, common_type); + ColumnPtr c1_converted = castColumn(col_with_type_and_name_right, common_type); + return executeDecimal({c0_converted, common_type, "left"}, {c1_converted, common_type, "right"}); + } + else + { + // compare + if (!allowDecimalComparison(left_type, right_type) && !date_and_datetime) + throw Exception( + "No operation " + getName() + " between " + left_type->getName() + " and " + right_type->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + return executeDecimal(col_with_type_and_name_left, col_with_type_and_name_right); + } - return executeDecimal(col_with_type_and_name_left, col_with_type_and_name_right); + } + else if (date_and_datetime) + { + DataTypePtr common_type = getLeastSupertype({left_type, right_type}); + ColumnPtr c0_converted = castColumn(col_with_type_and_name_left, common_type); + ColumnPtr c1_converted = castColumn(col_with_type_and_name_right, common_type); + if (!((res = executeNumLeftType(c0_converted.get(), c1_converted.get())) + || (res = executeNumLeftType(c0_converted.get(), c1_converted.get())))) + throw Exception("Date related common types can only be UInt32 or UInt64", ErrorCodes::LOGICAL_ERROR); + return res; } else if (left_type->equals(*right_type)) { diff --git a/tests/performance/datetime_comparison.xml b/tests/performance/datetime_comparison.xml index 2d47ded0b1a..8d7b0c8c4de 100644 --- a/tests/performance/datetime_comparison.xml +++ b/tests/performance/datetime_comparison.xml @@ -2,4 +2,5 @@ SELECT count() FROM numbers(1000000000) WHERE materialize(now()) > toString(toDateTime('2020-09-30 00:00:00')) SELECT count() FROM numbers(1000000000) WHERE materialize(now()) > toUInt32(toDateTime('2020-09-30 00:00:00')) SELECT count() FROM numbers(1000000000) WHERE materialize(now()) > toDateTime('2020-09-30 00:00:00') + SELECT count() FROM numbers(1000000000) WHERE materialize(now()) > toDate('2020-09-30 00:00:00') From cc0c3a90336f7527209e2be8d00089db8b9c2697 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 3 Aug 2021 12:34:08 +0300 Subject: [PATCH 086/198] Make IJoin::alwaysReturnsEmptySet pure virtual --- src/Interpreters/IJoin.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/IJoin.h b/src/Interpreters/IJoin.h index c2cf007d823..8fa85de4951 100644 --- a/src/Interpreters/IJoin.h +++ b/src/Interpreters/IJoin.h @@ -37,7 +37,7 @@ public: virtual size_t getTotalRowCount() const = 0; virtual size_t getTotalByteCount() const = 0; - virtual bool alwaysReturnsEmptySet() const { return false; } + virtual bool alwaysReturnsEmptySet() const = 0; /// StorageJoin/Dictionary is already filled. No need to call addJoinedBlock. /// Different query plan is used for such joins. From f4473c5de49b2f3b2f9b1c4741b2f672dbb4beb1 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 3 Aug 2021 14:18:47 +0300 Subject: [PATCH 087/198] GlobalSubqueriesVisitor external storage check fix --- src/Interpreters/GlobalSubqueriesVisitor.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h index 71ff7fd2e0e..fbf9b22a734 100644 --- a/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/src/Interpreters/GlobalSubqueriesVisitor.h @@ -63,7 +63,7 @@ public: return; bool is_table = false; - ASTPtr subquery_or_table_name = ast; /// ASTTableIdentifier | ASTSubquery | ASTTableExpression + ASTPtr subquery_or_table_name; /// ASTTableIdentifier | ASTSubquery | ASTTableExpression if (const auto * ast_table_expr = ast->as()) { @@ -76,7 +76,10 @@ public: } } else if (ast->as()) + { + subquery_or_table_name = ast; is_table = true; + } if (!subquery_or_table_name) throw Exception("Global subquery requires subquery or table name", ErrorCodes::WRONG_GLOBAL_SUBQUERY); From 0dc7da92b4401524c7ad69495b32bcf17c160414 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 3 Aug 2021 14:24:21 +0300 Subject: [PATCH 088/198] Added test --- .../02002_global_subqueries_subquery_or_table_name.reference | 0 .../02002_global_subqueries_subquery_or_table_name.sql | 5 +++++ 2 files changed, 5 insertions(+) create mode 100644 tests/queries/0_stateless/02002_global_subqueries_subquery_or_table_name.reference create mode 100644 tests/queries/0_stateless/02002_global_subqueries_subquery_or_table_name.sql diff --git a/tests/queries/0_stateless/02002_global_subqueries_subquery_or_table_name.reference b/tests/queries/0_stateless/02002_global_subqueries_subquery_or_table_name.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02002_global_subqueries_subquery_or_table_name.sql b/tests/queries/0_stateless/02002_global_subqueries_subquery_or_table_name.sql new file mode 100644 index 00000000000..1b24617569c --- /dev/null +++ b/tests/queries/0_stateless/02002_global_subqueries_subquery_or_table_name.sql @@ -0,0 +1,5 @@ +SELECT + cityHash64(number GLOBAL IN (NULL, -2147483648, -9223372036854775808), nan, 1024, NULL, NULL, 1.000100016593933, NULL), + (NULL, cityHash64(inf, -2147483648, NULL, NULL, 10.000100135803223), cityHash64(1.1754943508222875e-38, NULL, NULL, NULL), 2147483647) +FROM cluster(test_cluster_two_shards_localhost, numbers((NULL, cityHash64(0., 65536, NULL, NULL, 10000000000., NULL), 0) GLOBAL IN (some_identifier), 65536)) +WHERE number GLOBAL IN [1025] --{serverError 284} From 06903485d793d46b146626f026d70d1e059289a9 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 3 Aug 2021 14:38:29 +0300 Subject: [PATCH 089/198] Fix build on freebsd --- contrib/AMQP-CPP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/AMQP-CPP b/contrib/AMQP-CPP index 8d83306aa0e..e367d72cd44 160000 --- a/contrib/AMQP-CPP +++ b/contrib/AMQP-CPP @@ -1 +1 @@ -Subproject commit 8d83306aa0ef33704ee951f789163ca90ab74402 +Subproject commit e367d72cd44578c1f03cef96fba21d12cac56b14 From f9d1598847a7a3567a3ec916b2bac7da0bb54e49 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 3 Aug 2021 14:42:14 +0300 Subject: [PATCH 090/198] Update 10_question.md --- .github/ISSUE_TEMPLATE/10_question.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/10_question.md b/.github/ISSUE_TEMPLATE/10_question.md index 6e23fbdc605..a112b9599d5 100644 --- a/.github/ISSUE_TEMPLATE/10_question.md +++ b/.github/ISSUE_TEMPLATE/10_question.md @@ -7,6 +7,6 @@ assignees: '' --- -Make sure to check documentation https://clickhouse.yandex/docs/en/ first. If the question is concise and probably has a short answer, asking it in Telegram chat https://telegram.me/clickhouse_en is probably the fastest way to find the answer. For more complicated questions, consider asking them on StackOverflow with "clickhouse" tag https://stackoverflow.com/questions/tagged/clickhouse +> Make sure to check documentation https://clickhouse.yandex/docs/en/ first. If the question is concise and probably has a short answer, asking it in Telegram chat https://telegram.me/clickhouse_en is probably the fastest way to find the answer. For more complicated questions, consider asking them on StackOverflow with "clickhouse" tag https://stackoverflow.com/questions/tagged/clickhouse -If you still prefer GitHub issues, remove all this text and ask your question here. +> If you still prefer GitHub issues, remove all this text and ask your question here. From 73dd3bd6cc966bc56ca2f13b034fade9ea8616d5 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 3 Aug 2021 14:43:00 +0300 Subject: [PATCH 091/198] Update 40_bug-report.md --- .github/ISSUE_TEMPLATE/40_bug-report.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/40_bug-report.md b/.github/ISSUE_TEMPLATE/40_bug-report.md index 5c8611d47e6..d62ec578f8d 100644 --- a/.github/ISSUE_TEMPLATE/40_bug-report.md +++ b/.github/ISSUE_TEMPLATE/40_bug-report.md @@ -7,11 +7,11 @@ assignees: '' --- -You have to provide the following information whenever possible. +> You have to provide the following information whenever possible. **Describe the bug** -A clear and concise description of what works not as it is supposed to. +> A clear and concise description of what works not as it is supposed to. **Does it reproduce on recent release?** @@ -19,7 +19,7 @@ A clear and concise description of what works not as it is supposed to. **Enable crash reporting** -If possible, change "enabled" to true in "send_crash_reports" section in `config.xml`: +> If possible, change "enabled" to true in "send_crash_reports" section in `config.xml`: ``` @@ -39,12 +39,12 @@ If possible, change "enabled" to true in "send_crash_reports" section in `config **Expected behavior** -A clear and concise description of what you expected to happen. +> A clear and concise description of what you expected to happen. **Error message and/or stacktrace** -If applicable, add screenshots to help explain your problem. +> If applicable, add screenshots to help explain your problem. **Additional context** -Add any other context about the problem here. +> Add any other context about the problem here. From cd566f0a44f16eaa7fe21d04f684ef639a918f1c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 3 Aug 2021 14:43:26 +0300 Subject: [PATCH 092/198] Update 20_feature-request.md --- .github/ISSUE_TEMPLATE/20_feature-request.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/20_feature-request.md b/.github/ISSUE_TEMPLATE/20_feature-request.md index 99a762a2019..f59dbc2c40f 100644 --- a/.github/ISSUE_TEMPLATE/20_feature-request.md +++ b/.github/ISSUE_TEMPLATE/20_feature-request.md @@ -7,16 +7,20 @@ assignees: '' --- -(you don't have to strictly follow this form) +> (you don't have to strictly follow this form) **Use case** -A clear and concise description of what is the intended usage scenario is. + +> A clear and concise description of what is the intended usage scenario is. **Describe the solution you'd like** -A clear and concise description of what you want to happen. + +> A clear and concise description of what you want to happen. **Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. + +> A clear and concise description of any alternative solutions or features you've considered. **Additional context** -Add any other context or screenshots about the feature request here. + +> Add any other context or screenshots about the feature request here. From 3184902c6804ecc6de61238f58a3e1d8d2a69b67 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 3 Aug 2021 14:43:56 +0300 Subject: [PATCH 093/198] Update 50_build-issue.md --- .github/ISSUE_TEMPLATE/50_build-issue.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/50_build-issue.md b/.github/ISSUE_TEMPLATE/50_build-issue.md index 73c97cc3cfb..a358575cd7c 100644 --- a/.github/ISSUE_TEMPLATE/50_build-issue.md +++ b/.github/ISSUE_TEMPLATE/50_build-issue.md @@ -7,10 +7,11 @@ assignees: '' --- -Make sure that `git diff` result is empty and you've just pulled fresh master. Try cleaning up cmake cache. Just in case, official build instructions are published here: https://clickhouse.yandex/docs/en/development/build/ +> Make sure that `git diff` result is empty and you've just pulled fresh master. Try cleaning up cmake cache. Just in case, official build instructions are published here: https://clickhouse.yandex/docs/en/development/build/ **Operating system** -OS kind or distribution, specific version/release, non-standard kernel if any. If you are trying to build inside virtual machine, please mention it too. + +> OS kind or distribution, specific version/release, non-standard kernel if any. If you are trying to build inside virtual machine, please mention it too. **Cmake version** From a9513f35878640a06f0b67e30f4272e340bc4bd0 Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Tue, 3 Aug 2021 11:54:37 +0000 Subject: [PATCH 094/198] add compression for INTO OUTFILE --- programs/client/Client.cpp | 13 +++++++----- src/Interpreters/executeQuery.cpp | 35 ++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index c5e43449557..5b204751ba0 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -197,7 +197,7 @@ private: std::unique_ptr pager_cmd; /// The user can specify to redirect query output to a file. - std::optional out_file_buf; + std::unique_ptr out_file_buf; BlockOutputStreamPtr block_out_stream; /// The user could specify special file for server logs (stderr by default) @@ -2238,8 +2238,11 @@ private: const auto & out_file_node = query_with_output->out_file->as(); const auto & out_file = out_file_node.value.safeGet(); - out_file_buf.emplace(out_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_EXCL | O_CREAT); - out_buf = &*out_file_buf; + out_file_buf = wrapWriteBufferWithCompressionMethod( + std::make_unique(out_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_EXCL | O_CREAT), + chooseCompressionMethod(out_file, ""), + /* compression level = */ 3 + ); // We are writing to file, so default format is the same as in non-interactive mode. if (is_interactive && is_default_format) @@ -2259,9 +2262,9 @@ private: /// It is not clear how to write progress with parallel formatting. It may increase code complexity significantly. if (!need_render_progress) - block_out_stream = context->getOutputStreamParallelIfPossible(current_format, *out_buf, block); + block_out_stream = context->getOutputStreamParallelIfPossible(current_format, out_file_buf ? *out_file_buf : *out_buf, block); else - block_out_stream = context->getOutputStream(current_format, *out_buf, block); + block_out_stream = context->getOutputStream(current_format, out_file_buf ? *out_file_buf : *out_buf, block); block_out_stream->writePrefix(); } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 75b89554820..23031fe36d4 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -1010,22 +1010,31 @@ void executeQuery( const auto * ast_query_with_output = dynamic_cast(ast.get()); WriteBuffer * out_buf = &ostr; - std::optional out_file_buf; + std::unique_ptr compressed_buffer; if (ast_query_with_output && ast_query_with_output->out_file) { if (!allow_into_outfile) throw Exception("INTO OUTFILE is not allowed", ErrorCodes::INTO_OUTFILE_NOT_ALLOWED); const auto & out_file = ast_query_with_output->out_file->as().value.safeGet(); - out_file_buf.emplace(out_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_EXCL | O_CREAT); - out_buf = &*out_file_buf; + compressed_buffer = wrapWriteBufferWithCompressionMethod( + std::make_unique(out_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_EXCL | O_CREAT), + chooseCompressionMethod(out_file, ""), + /* compression level = */ 3 + ); } String format_name = ast_query_with_output && (ast_query_with_output->format != nullptr) ? getIdentifierName(ast_query_with_output->format) : context->getDefaultFormat(); - auto out = FormatFactory::instance().getOutputStreamParallelIfPossible(format_name, *out_buf, streams.in->getHeader(), context, {}, output_format_settings); + auto out = FormatFactory::instance().getOutputStreamParallelIfPossible( + format_name, + compressed_buffer ? *compressed_buffer : *out_buf, + streams.in->getHeader(), + context, + {}, + output_format_settings); /// Save previous progress callback if any. TODO Do it more conveniently. auto previous_progress_callback = context->getProgressCallback(); @@ -1049,15 +1058,18 @@ void executeQuery( const ASTQueryWithOutput * ast_query_with_output = dynamic_cast(ast.get()); WriteBuffer * out_buf = &ostr; - std::optional out_file_buf; + std::unique_ptr compressed_buffer; if (ast_query_with_output && ast_query_with_output->out_file) { if (!allow_into_outfile) throw Exception("INTO OUTFILE is not allowed", ErrorCodes::INTO_OUTFILE_NOT_ALLOWED); const auto & out_file = typeid_cast(*ast_query_with_output->out_file).value.safeGet(); - out_file_buf.emplace(out_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_EXCL | O_CREAT); - out_buf = &*out_file_buf; + compressed_buffer = wrapWriteBufferWithCompressionMethod( + std::make_unique(out_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_EXCL | O_CREAT), + chooseCompressionMethod(out_file, ""), + /* compression level = */ 3 + ); } String format_name = ast_query_with_output && (ast_query_with_output->format != nullptr) @@ -1071,7 +1083,14 @@ void executeQuery( return std::make_shared(header); }); - auto out = FormatFactory::instance().getOutputFormatParallelIfPossible(format_name, *out_buf, pipeline.getHeader(), context, {}, output_format_settings); + auto out = FormatFactory::instance().getOutputFormatParallelIfPossible( + format_name, + compressed_buffer ? *compressed_buffer : *out_buf, + pipeline.getHeader(), + context, + {}, + output_format_settings); + out->setAutoFlush(); /// Save previous progress callback if any. TODO Do it more conveniently. From d8c112d0c0c9bbb52db7bbb8b435211b6c754ca2 Mon Sep 17 00:00:00 2001 From: adevyatova Date: Tue, 3 Aug 2021 12:03:10 +0000 Subject: [PATCH 095/198] Small current fixes --- .../database-engines/materialize-mysql.md | 4 +- .../en/engines/database-engines/replicated.md | 5 + .../functions/other-functions.md | 155 ++++++++++++++++++ .../database-engines/materialize-mysql.md | 7 +- .../ru/engines/database-engines/replicated.md | 4 + 5 files changed, 167 insertions(+), 8 deletions(-) diff --git a/docs/en/engines/database-engines/materialize-mysql.md b/docs/en/engines/database-engines/materialize-mysql.md index 198808fa952..b53e09bba89 100644 --- a/docs/en/engines/database-engines/materialize-mysql.md +++ b/docs/en/engines/database-engines/materialize-mysql.md @@ -3,14 +3,12 @@ toc_priority: 29 toc_title: MaterializeMySQL --- -# MaterializeMySQL {#materialize-mysql} +# [experimental] MaterializeMySQL {#materialize-mysql} Creates ClickHouse database with all the tables existing in MySQL, and all the data in those tables. ClickHouse server works as MySQL replica. It reads binlog and performs DDL and DML queries. -This feature is experimental. - ## Creating a Database {#creating-a-database} ``` sql diff --git a/docs/en/engines/database-engines/replicated.md b/docs/en/engines/database-engines/replicated.md index 575aa9d3ee5..18c39d1754d 100644 --- a/docs/en/engines/database-engines/replicated.md +++ b/docs/en/engines/database-engines/replicated.md @@ -1,3 +1,8 @@ +--- +toc_priority: 40 +toc_title: Replicated +--- + # [experimental] Replicated {#replicated} The engine is based on the [Atomic](../../engines/database-engines/atomic.md) engine. It supports replication of metadata via DDL log being written to ZooKeeper and executed on all of the replicas for a given database. diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 30e2e427158..63892bc712c 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -2138,3 +2138,158 @@ Result: - [tcp_port](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port) +## currentProfiles {#current-profiles} + +Short description. + +**Syntax** + +``` sql + +``` + +Alias: ``. (Optional) + +More text (Optional). + +**Arguments** (Optional) + +- `x` — Description. Optional (only for optional arguments). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type). +- `y` — Description. Optional (only for optional arguments). Possible values: .Default value: . [Type name](relative/path/to/type/dscr.md#type). + +**Parameters** (Optional, only for parametric aggregate functions) + +- `z` — Description. Optional (only for optional parameters). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type). + +**Returned value(s)** + +- Returned values list. + +Type: [Type name](relative/path/to/type/dscr.md#type). + +**Example** + +The example must show usage and/or a use cases. The following text contains recommended parts of an example. + +Input table (Optional): + +``` text +``` + +Query: + +``` sql +``` + +Result: + +``` text +``` + +**See Also** (Optional) + +- [link](#) + +## enabledProfiles {#enabled-profiles} + +Short description. + +**Syntax** + +``` sql + +``` + +Alias: ``. (Optional) + +More text (Optional). + +**Arguments** (Optional) + +- `x` — Description. Optional (only for optional arguments). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type). +- `y` — Description. Optional (only for optional arguments). Possible values: .Default value: . [Type name](relative/path/to/type/dscr.md#type). + +**Parameters** (Optional, only for parametric aggregate functions) + +- `z` — Description. Optional (only for optional parameters). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type). + +**Returned value(s)** + +- Returned values list. + +Type: [Type name](relative/path/to/type/dscr.md#type). + +**Example** + +The example must show usage and/or a use cases. The following text contains recommended parts of an example. + +Input table (Optional): + +``` text +``` + +Query: + +``` sql +``` + +Result: + +``` text +``` + +**See Also** (Optional) + +- [link](#) + +## defaultProfiles {#default-profiles} + +Short description. + +**Syntax** + +``` sql + +``` + +Alias: ``. (Optional) + +More text (Optional). + +**Arguments** (Optional) + +- `x` — Description. Optional (only for optional arguments). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type). +- `y` — Description. Optional (only for optional arguments). Possible values: .Default value: . [Type name](relative/path/to/type/dscr.md#type). + +**Parameters** (Optional, only for parametric aggregate functions) + +- `z` — Description. Optional (only for optional parameters). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type). + +**Returned value(s)** + +- Returned values list. + +Type: [Type name](relative/path/to/type/dscr.md#type). + +**Example** + +The example must show usage and/or a use cases. The following text contains recommended parts of an example. + +Input table (Optional): + +``` text +``` + +Query: + +``` sql +``` + +Result: + +``` text +``` + +**See Also** (Optional) + +- [link](#) \ No newline at end of file diff --git a/docs/ru/engines/database-engines/materialize-mysql.md b/docs/ru/engines/database-engines/materialize-mysql.md index b62670bc3c9..d2bef29fc16 100644 --- a/docs/ru/engines/database-engines/materialize-mysql.md +++ b/docs/ru/engines/database-engines/materialize-mysql.md @@ -1,16 +1,13 @@ - --- toc_priority: 29 toc_title: MaterializeMySQL --- -# MaterializeMySQL {#materialize-mysql} +# [экспериментальный] MaterializeMySQL {#materialize-mysql} Создает базу данных ClickHouse со всеми таблицами, существующими в MySQL, и всеми данными в этих таблицах. -Сервер ClickHouse работает как реплика MySQL. Он читает файл binlog и выполняет DDL and DML-запросы. - -`MaterializeMySQL` — экспериментальный движок баз данных. +Сервер ClickHouse работает как реплика MySQL. Он читает файл binlog и выполняет DDL и DML-запросы. ## Создание базы данных {#creating-a-database} diff --git a/docs/ru/engines/database-engines/replicated.md b/docs/ru/engines/database-engines/replicated.md index 62be07f617c..063c19bcebd 100644 --- a/docs/ru/engines/database-engines/replicated.md +++ b/docs/ru/engines/database-engines/replicated.md @@ -1,3 +1,7 @@ +--- +toc_priority: 40 +toc_title: Replicated +--- # [экспериментальный] Replicated {#replicated} From cb9627255ae7ef4a786ba65394c94dbca4009233 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 3 Aug 2021 15:16:55 +0300 Subject: [PATCH 096/198] Fixed tests --- src/Interpreters/GlobalSubqueriesVisitor.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h index fbf9b22a734..6a87527dc9c 100644 --- a/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/src/Interpreters/GlobalSubqueriesVisitor.h @@ -80,6 +80,10 @@ public: subquery_or_table_name = ast; is_table = true; } + else if (ast->as()) + { + subquery_or_table_name = ast; + } if (!subquery_or_table_name) throw Exception("Global subquery requires subquery or table name", ErrorCodes::WRONG_GLOBAL_SUBQUERY); From 2aea1c8d4a5be320365472052d8a48bf69fd9fe9 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 3 Aug 2021 12:24:16 +0000 Subject: [PATCH 097/198] done --- src/Compression/CompressionCodecDelta.cpp | 4 ++ .../CompressionCodecDoubleDelta.cpp | 4 ++ src/Compression/CompressionCodecGorilla.cpp | 4 ++ src/Compression/CompressionCodecLZ4.cpp | 6 ++- src/Compression/LZ4_decompress_faster.cpp | 42 +++++++++++++------ src/Compression/LZ4_decompress_faster.h | 6 +-- 6 files changed, 50 insertions(+), 16 deletions(-) diff --git a/src/Compression/CompressionCodecDelta.cpp b/src/Compression/CompressionCodecDelta.cpp index 447abe9e840..e281609ff43 100644 --- a/src/Compression/CompressionCodecDelta.cpp +++ b/src/Compression/CompressionCodecDelta.cpp @@ -132,6 +132,10 @@ void CompressionCodecDelta::doDecompressData(const char * source, UInt32 source_ throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); UInt8 bytes_size = source[0]; + + if (bytes_size == 0) + throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); + UInt8 bytes_to_skip = uncompressed_size % bytes_size; if (UInt32(2 + bytes_to_skip) > source_size) diff --git a/src/Compression/CompressionCodecDoubleDelta.cpp b/src/Compression/CompressionCodecDoubleDelta.cpp index 79ced55594a..c416582eb6b 100644 --- a/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/src/Compression/CompressionCodecDoubleDelta.cpp @@ -502,6 +502,10 @@ void CompressionCodecDoubleDelta::doDecompressData(const char * source, UInt32 s throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); UInt8 bytes_size = source[0]; + + if (bytes_size == 0) + throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); + UInt8 bytes_to_skip = uncompressed_size % bytes_size; if (UInt32(2 + bytes_to_skip) > source_size) diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index 7fcb2183503..1276ac911f1 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -410,6 +410,10 @@ void CompressionCodecGorilla::doDecompressData(const char * source, UInt32 sourc throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); UInt8 bytes_size = source[0]; + + if (bytes_size == 0) + throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); + UInt8 bytes_to_skip = uncompressed_size % bytes_size; if (UInt32(2 + bytes_to_skip) > source_size) diff --git a/src/Compression/CompressionCodecLZ4.cpp b/src/Compression/CompressionCodecLZ4.cpp index 8cb81e460b1..379eea7e45c 100644 --- a/src/Compression/CompressionCodecLZ4.cpp +++ b/src/Compression/CompressionCodecLZ4.cpp @@ -62,6 +62,7 @@ private: namespace ErrorCodes { extern const int CANNOT_COMPRESS; + extern const int CANNOT_DECOMPRESS; extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; extern const int ILLEGAL_CODEC_PARAMETER; } @@ -93,7 +94,10 @@ UInt32 CompressionCodecLZ4::doCompressData(const char * source, UInt32 source_si void CompressionCodecLZ4::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const { - LZ4::decompress(source, dest, source_size, uncompressed_size, lz4_stat); + bool success = LZ4::decompress(source, dest, source_size, uncompressed_size, lz4_stat); + + if (!success) + throw Exception("Cannot decomress", ErrorCodes::CANNOT_DECOMPRESS); } void registerCodecLZ4(CompressionCodecFactory & factory) diff --git a/src/Compression/LZ4_decompress_faster.cpp b/src/Compression/LZ4_decompress_faster.cpp index dc293941310..6972457f11b 100644 --- a/src/Compression/LZ4_decompress_faster.cpp +++ b/src/Compression/LZ4_decompress_faster.cpp @@ -412,13 +412,16 @@ template <> void inline copyOverlap<32, false>(UInt8 * op, const UInt8 *& match, /// See also https://stackoverflow.com/a/30669632 template -void NO_INLINE decompressImpl( +bool NO_INLINE decompressImpl( const char * const source, char * const dest, + size_t source_size, size_t dest_size) { const UInt8 * ip = reinterpret_cast(source); UInt8 * op = reinterpret_cast(dest); + const UInt8 * const input_end = ip + source_size; + UInt8 * const output_begin = op; UInt8 * const output_end = op + dest_size; /// Unrolling with clang is doing >10% performance degrade. @@ -461,13 +464,19 @@ void NO_INLINE decompressImpl( /// output: xyzHello, w /// ^-op (we will overwrite excessive bytes on next iteration) - wildCopy(op, ip, copy_end); /// Here we can write up to copy_amount - 1 bytes after buffer. + { + auto * target = std::min(copy_end, output_end); + wildCopy(op, ip, target); /// Here we can write up to copy_amount - 1 bytes after buffer. + + if (target == output_end) + return true; + } ip += length; op = copy_end; - if (copy_end >= output_end) - return; + if (unlikely(ip > input_end)) + return false; /// Get match offset. @@ -475,6 +484,9 @@ void NO_INLINE decompressImpl( ip += 2; const UInt8 * match = op - offset; + if (unlikely(match < output_begin)) + return false; + /// Get match length. length = token & 0x0F; @@ -515,7 +527,10 @@ void NO_INLINE decompressImpl( copy(op, match); /// copy_amount + copy_amount - 1 - 4 * 2 bytes after buffer. if (length > copy_amount * 2) - wildCopy(op + copy_amount, match + copy_amount, copy_end); + { + auto * target = std::min(copy_end, output_end); + wildCopy(op + copy_amount, match + copy_amount, target); + } op = copy_end; } @@ -524,7 +539,7 @@ void NO_INLINE decompressImpl( } -void decompress( +bool decompress( const char * const source, char * const dest, size_t source_size, @@ -532,7 +547,7 @@ void decompress( PerformanceStatistics & statistics [[maybe_unused]]) { if (source_size == 0 || dest_size == 0) - return; + return true; /// Don't run timer if the block is too small. if (dest_size >= 32768) @@ -542,24 +557,27 @@ void decompress( /// Run the selected method and measure time. Stopwatch watch; + bool success = true; if (best_variant == 0) - decompressImpl<16, true>(source, dest, dest_size); + success = decompressImpl<16, true>(source, dest, source_size, dest_size); if (best_variant == 1) - decompressImpl<16, false>(source, dest, dest_size); + success = decompressImpl<16, false>(source, dest, source_size, dest_size); if (best_variant == 2) - decompressImpl<8, true>(source, dest, dest_size); + success = decompressImpl<8, true>(source, dest, source_size, dest_size); if (best_variant == 3) - decompressImpl<32, false>(source, dest, dest_size); + success = decompressImpl<32, false>(source, dest, source_size, dest_size); watch.stop(); /// Update performance statistics. statistics.data[best_variant].update(watch.elapsedSeconds(), dest_size); + + return success; } else { - decompressImpl<8, false>(source, dest, dest_size); + return decompressImpl<8, false>(source, dest, source_size, dest_size); } } diff --git a/src/Compression/LZ4_decompress_faster.h b/src/Compression/LZ4_decompress_faster.h index 30a0d7acb22..c1b54cf20c9 100644 --- a/src/Compression/LZ4_decompress_faster.h +++ b/src/Compression/LZ4_decompress_faster.h @@ -122,14 +122,14 @@ struct PerformanceStatistics return choose_method; } - PerformanceStatistics() {} - PerformanceStatistics(ssize_t choose_method_) : choose_method(choose_method_) {} + PerformanceStatistics() = default; + explicit PerformanceStatistics(ssize_t choose_method_) : choose_method(choose_method_) {} }; /** This method dispatch to one of different implementations depending on performance statistics. */ -void decompress( +bool decompress( const char * const source, char * const dest, size_t source_size, From 6d83eacec42c7c403c99804a713a9d38caa4a45d Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 3 Aug 2021 12:27:03 +0000 Subject: [PATCH 098/198] typo --- src/Compression/CompressionCodecLZ4.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compression/CompressionCodecLZ4.cpp b/src/Compression/CompressionCodecLZ4.cpp index 379eea7e45c..396f6fad2c3 100644 --- a/src/Compression/CompressionCodecLZ4.cpp +++ b/src/Compression/CompressionCodecLZ4.cpp @@ -97,7 +97,7 @@ void CompressionCodecLZ4::doDecompressData(const char * source, UInt32 source_si bool success = LZ4::decompress(source, dest, source_size, uncompressed_size, lz4_stat); if (!success) - throw Exception("Cannot decomress", ErrorCodes::CANNOT_DECOMPRESS); + throw Exception("Cannot decompress", ErrorCodes::CANNOT_DECOMPRESS); } void registerCodecLZ4(CompressionCodecFactory & factory) From f4275b952e79e1d2240e68f397f3f688169ca3cd Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Tue, 3 Aug 2021 12:31:38 +0000 Subject: [PATCH 099/198] add simple test --- .../0_stateless/02001_compress_output_file.reference | 2 ++ .../queries/0_stateless/02001_compress_output_file.sh | 11 +++++++++++ 2 files changed, 13 insertions(+) create mode 100644 tests/queries/0_stateless/02001_compress_output_file.reference create mode 100755 tests/queries/0_stateless/02001_compress_output_file.sh diff --git a/tests/queries/0_stateless/02001_compress_output_file.reference b/tests/queries/0_stateless/02001_compress_output_file.reference new file mode 100644 index 00000000000..6f51dfc24e1 --- /dev/null +++ b/tests/queries/0_stateless/02001_compress_output_file.reference @@ -0,0 +1,2 @@ +Hello, World! From client. +Hello, World! From local. diff --git a/tests/queries/0_stateless/02001_compress_output_file.sh b/tests/queries/0_stateless/02001_compress_output_file.sh new file mode 100755 index 00000000000..4034e141d6c --- /dev/null +++ b/tests/queries/0_stateless/02001_compress_output_file.sh @@ -0,0 +1,11 @@ +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query "SELECT * FROM (SELECT 'Hello, World! From client.') INTO OUTFILE '${CLICKHOUSE_TMP}/test_compression_of_output_file_from_client.gz'" +gunzip ${CLICKHOUSE_TMP}/test_compression_of_output_file_from_client.gz +cat ${CLICKHOUSE_TMP}/test_compression_of_output_file_from_client + +${CLICKHOUSE_LOCAL} --query "SELECT * FROM (SELECT 'Hello, World! From local.') INTO OUTFILE '${CLICKHOUSE_TMP}/test_compression_of_output_file_from_local.gz'" +gunzip ${CLICKHOUSE_TMP}/test_compression_of_output_file_from_local.gz +cat ${CLICKHOUSE_TMP}/test_compression_of_output_file_from_local \ No newline at end of file From ad65e3c17bcd9c5e7f2da1e63bbd7089fa1fdcfc Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 3 Aug 2021 15:32:17 +0300 Subject: [PATCH 100/198] slightly more optimize reading in order of key --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 74482b48f03..f8c12449c7e 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -409,7 +409,6 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( { RangesInDataPart part = parts_with_ranges.back(); parts_with_ranges.pop_back(); - size_t & marks_in_part = info.sum_marks_in_parts.back(); /// We will not take too few rows from a part. @@ -424,8 +423,13 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( MarkRanges ranges_to_get_from_part; + /// We take full part if it contains enough marks or + /// if we know limit and part contains less than 'limit' rows. + bool take_full_part = marks_in_part <= need_marks + || (input_order_info->limit && input_order_info->limit < part.getRowsCount()); + /// We take the whole part if it is small enough. - if (marks_in_part <= need_marks) + if (take_full_part) { ranges_to_get_from_part = part.ranges; From 5c6974736d327903bc0241351382bb57c6b56511 Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Tue, 3 Aug 2021 12:52:46 +0000 Subject: [PATCH 101/198] add newline --- tests/queries/0_stateless/02001_compress_output_file.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02001_compress_output_file.sh b/tests/queries/0_stateless/02001_compress_output_file.sh index 4034e141d6c..d884cab38d2 100755 --- a/tests/queries/0_stateless/02001_compress_output_file.sh +++ b/tests/queries/0_stateless/02001_compress_output_file.sh @@ -8,4 +8,4 @@ cat ${CLICKHOUSE_TMP}/test_compression_of_output_file_from_client ${CLICKHOUSE_LOCAL} --query "SELECT * FROM (SELECT 'Hello, World! From local.') INTO OUTFILE '${CLICKHOUSE_TMP}/test_compression_of_output_file_from_local.gz'" gunzip ${CLICKHOUSE_TMP}/test_compression_of_output_file_from_local.gz -cat ${CLICKHOUSE_TMP}/test_compression_of_output_file_from_local \ No newline at end of file +cat ${CLICKHOUSE_TMP}/test_compression_of_output_file_from_local From ff2572c243aa561c376be5e275ae3f3918390a58 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 3 Aug 2021 16:28:03 +0300 Subject: [PATCH 102/198] fix --- src/Databases/DatabaseReplicated.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index ac6c525af68..26dd8763c40 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -525,7 +525,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep query_context->getClientInfo().is_replicated_database_internal = true; query_context->setCurrentDatabase(database_name); query_context->setCurrentQueryId(""); - auto txn = std::make_shared(current_zookeeper, zookeeper_path, false); + auto txn = std::make_shared(current_zookeeper, zookeeper_path, false, ""); query_context->initZooKeeperMetadataTransaction(txn); return query_context; }; From f4a59a6abf3d73d003d961e5cc35981d14e929c2 Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Tue, 3 Aug 2021 13:41:58 +0000 Subject: [PATCH 103/198] correct tests --- src/Interpreters/executeQuery.cpp | 2 +- tests/queries/0_stateless/02001_compress_output_file.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 23031fe36d4..c59cbc772e7 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -1090,7 +1090,7 @@ void executeQuery( context, {}, output_format_settings); - + out->setAutoFlush(); /// Save previous progress callback if any. TODO Do it more conveniently. diff --git a/tests/queries/0_stateless/02001_compress_output_file.sh b/tests/queries/0_stateless/02001_compress_output_file.sh index d884cab38d2..336c8319ee4 100755 --- a/tests/queries/0_stateless/02001_compress_output_file.sh +++ b/tests/queries/0_stateless/02001_compress_output_file.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh From 019a2215509633cd3830c51edbf1384deb726235 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 3 Aug 2021 17:29:59 +0300 Subject: [PATCH 104/198] fix unbundled build --- contrib/AMQP-CPP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/AMQP-CPP b/contrib/AMQP-CPP index e367d72cd44..1a6c51f4ac5 160000 --- a/contrib/AMQP-CPP +++ b/contrib/AMQP-CPP @@ -1 +1 @@ -Subproject commit e367d72cd44578c1f03cef96fba21d12cac56b14 +Subproject commit 1a6c51f4ac51ac56610fa95081bd2f349911375a From 1df8a5f57a0f48a1098b704a114f7a4acc3bd550 Mon Sep 17 00:00:00 2001 From: Malte Date: Tue, 3 Aug 2021 14:40:28 -0400 Subject: [PATCH 105/198] Update gui.md We had some issues figuring out how to use the session settings and hope this helps other folks as well --- docs/en/interfaces/third-party/gui.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md index fffe0c87a53..2d7f3a24011 100644 --- a/docs/en/interfaces/third-party/gui.md +++ b/docs/en/interfaces/third-party/gui.md @@ -84,6 +84,8 @@ Features: - Table data preview. - Full-text search. +By default, DBeaver does not connect using a session (the CLI for example does). If you require session support (for example to set settings for your session), edit the driver connection properties and set session_id to a random string (it uses the http connection under the hood). Then you can use any setting from the query window + ### clickhouse-cli {#clickhouse-cli} [clickhouse-cli](https://github.com/hatarist/clickhouse-cli) is an alternative command-line client for ClickHouse, written in Python 3. From edeeb801c09dc6fbc348a60bfe5af591516b9797 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 3 Aug 2021 21:46:25 +0300 Subject: [PATCH 106/198] Updated readIntTextUnsafe --- src/IO/ReadHelpers.h | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index d8e31c18617..e9055a43660 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -403,7 +403,6 @@ bool tryReadIntText(T & x, ReadBuffer & buf) // -V1071 * Differs in following: * - for numbers starting with zero, parsed only zero; * - symbol '+' before number is not supported; - * - symbols :;<=>? are parsed as some numbers. */ template void readIntTextUnsafe(T & x, ReadBuffer & buf) @@ -437,15 +436,12 @@ void readIntTextUnsafe(T & x, ReadBuffer & buf) while (!buf.eof()) { - /// This check is suddenly faster than - /// unsigned char c = *buf.position() - '0'; - /// if (c < 10) - /// for unknown reason on Xeon E5645. + auto value = *buf.position() - '0'; - if ((*buf.position() & 0xF0) == 0x30) /// It makes sense to have this condition inside loop. + if (value < 10) { res *= 10; - res += *buf.position() & 0x0F; + res += value; ++buf.position(); } else From dfa597f11db712432f87122eaa1f0a97c6be210d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 3 Aug 2021 22:31:13 +0300 Subject: [PATCH 107/198] Update ReadHelpers.h --- src/IO/ReadHelpers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index e9055a43660..f985070788c 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -436,7 +436,7 @@ void readIntTextUnsafe(T & x, ReadBuffer & buf) while (!buf.eof()) { - auto value = *buf.position() - '0'; + unsigned char value = *buf.position() - '0'; if (value < 10) { From cc2c1b80df83a6bf187c7a934fd78c021aecfb97 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 3 Aug 2021 19:47:59 +0000 Subject: [PATCH 108/198] Fix tests --- src/Bridge/LibraryBridgeHelper.cpp | 7 ++++--- src/Bridge/LibraryBridgeHelper.h | 1 + tests/integration/test_library_bridge/test.py | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Bridge/LibraryBridgeHelper.cpp b/src/Bridge/LibraryBridgeHelper.cpp index 9341ddda922..95a96074be0 100644 --- a/src/Bridge/LibraryBridgeHelper.cpp +++ b/src/Bridge/LibraryBridgeHelper.cpp @@ -39,6 +39,7 @@ LibraryBridgeHelper::LibraryBridgeHelper( , http_timeout(context_->getGlobalContext()->getSettingsRef().http_receive_timeout.value) , library_data(library_data_) , dictionary_id(dictionary_id_) + , http_timeouts(ConnectionTimeouts::getHTTPTimeouts(context_)) { bridge_port = config.getUInt("library_bridge.port", DEFAULT_PORT); bridge_host = config.getString("library_bridge.host", DEFAULT_HOST); @@ -75,7 +76,7 @@ bool LibraryBridgeHelper::bridgeHandShake() String result; try { - ReadWriteBufferFromHTTP buf(createRequestURI(PING), Poco::Net::HTTPRequest::HTTP_GET, {}, ConnectionTimeouts::getHTTPTimeouts(getContext())); + ReadWriteBufferFromHTTP buf(createRequestURI(PING), Poco::Net::HTTPRequest::HTTP_GET, {}, http_timeouts); readString(result, buf); } catch (...) @@ -239,7 +240,7 @@ bool LibraryBridgeHelper::executeRequest(const Poco::URI & uri, ReadWriteBufferF uri, Poco::Net::HTTPRequest::HTTP_POST, std::move(out_stream_callback), - ConnectionTimeouts::getHTTPTimeouts(getContext())); + http_timeouts); bool res; readBoolText(res, buf); @@ -253,7 +254,7 @@ BlockInputStreamPtr LibraryBridgeHelper::loadBase(const Poco::URI & uri, ReadWri uri, Poco::Net::HTTPRequest::HTTP_POST, std::move(out_stream_callback), - ConnectionTimeouts::getHTTPTimeouts(getContext()), + http_timeouts, 0, Poco::Net::HTTPBasicCredentials{}, DBMS_DEFAULT_BUFFER_SIZE, diff --git a/src/Bridge/LibraryBridgeHelper.h b/src/Bridge/LibraryBridgeHelper.h index 6615162efe7..fec644240da 100644 --- a/src/Bridge/LibraryBridgeHelper.h +++ b/src/Bridge/LibraryBridgeHelper.h @@ -98,6 +98,7 @@ private: std::string bridge_host; size_t bridge_port; bool library_initialized = false; + ConnectionTimeouts http_timeouts; }; } diff --git a/tests/integration/test_library_bridge/test.py b/tests/integration/test_library_bridge/test.py index 4bef197f8bd..607afb6db5f 100644 --- a/tests/integration/test_library_bridge/test.py +++ b/tests/integration/test_library_bridge/test.py @@ -2,6 +2,7 @@ import os import os.path as p import pytest import time +import logging from helpers.cluster import ClickHouseCluster, run_and_check From 6a5925b6ce1d0f8edf4e544a0560aa346171b489 Mon Sep 17 00:00:00 2001 From: zhangxiao871 Date: Wed, 4 Aug 2021 11:07:39 +0800 Subject: [PATCH 109/198] Fix CURR_DATABASE empty for 01034_move_partition_from_table_zookeeper.sh --- .../0_stateless/01034_move_partition_from_table_zookeeper.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh index ae3dd7851c8..af4969d77c8 100755 --- a/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh @@ -75,8 +75,8 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE dst;" $CLICKHOUSE_CLIENT --query="SELECT 'MOVE incompatible schema different order by';" -$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CURR_DATABASE/src3', '1') PARTITION BY p ORDER BY (p, k, d);" -$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CURR_DATABASE/dst3', '1') PARTITION BY p ORDER BY (d, k, p);" +$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/src3', '1') PARTITION BY p ORDER BY (p, k, d);" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/dst3', '1') PARTITION BY p ORDER BY (d, k, p);" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (0, '0', 1);" From c2410920d3386c216c2e3034155694bc03dfa3e6 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Wed, 4 Aug 2021 09:14:20 +0300 Subject: [PATCH 110/198] Safer `ReadBufferFromS3` for merges and backports. --- src/IO/ReadBufferFromS3.h | 2 +- src/Storages/StorageS3.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h index 46fb79a4a14..e24d571b557 100644 --- a/src/IO/ReadBufferFromS3.h +++ b/src/IO/ReadBufferFromS3.h @@ -43,7 +43,7 @@ public: const String & bucket_, const String & key_, UInt64 max_single_read_retries_, - size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE); + size_t buffer_size_); bool nextImpl() override; diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 1008bae346d..fc3ce3a10ed 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -208,7 +208,8 @@ bool StorageS3Source::initialize() file_path = fs::path(bucket) / current_key; read_buf = wrapReadBufferWithCompressionMethod( - std::make_unique(client, bucket, current_key, max_single_read_retries), chooseCompressionMethod(current_key, compression_hint)); + std::make_unique(client, bucket, current_key, max_single_read_retries, DBMS_DEFAULT_BUFFER_SIZE), + chooseCompressionMethod(current_key, compression_hint)); auto input_format = FormatFactory::instance().getInput(format, *read_buf, sample_block, getContext(), max_block_size); pipeline = std::make_unique(); pipeline->init(Pipe(input_format)); From cc32a61488f35933e1870651200df1c9b50f38aa Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Wed, 4 Aug 2021 10:40:02 +0300 Subject: [PATCH 111/198] Fix. --- src/Disks/S3/DiskS3.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 1f1c73c32c3..6dd29165566 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -363,7 +363,8 @@ int DiskS3::readSchemaVersion(const String & source_bucket, const String & sourc settings->client, source_bucket, source_path + SCHEMA_VERSION_OBJECT, - settings->s3_max_single_read_retries); + settings->s3_max_single_read_retries, + DBMS_DEFAULT_BUFFER_SIZE); readIntText(version, buffer); From e740d8e643a418875cb6d7dd9fe9a9c08fdef991 Mon Sep 17 00:00:00 2001 From: kolsys Date: Wed, 4 Aug 2021 12:48:56 +0300 Subject: [PATCH 112/198] Update play.html Allow to pass query settings via server URI --- programs/server/play.html | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/programs/server/play.html b/programs/server/play.html index 7b13807f2d9..3618c420330 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -361,23 +361,22 @@ function postImpl(posted_request_num, query) { - /// TODO: Check if URL already contains query string (append parameters). + const user = document.getElementById('user').value; + const password = document.getElementById('password').value; - let user = document.getElementById('user').value; - let password = document.getElementById('password').value; + const server_address = document.getElementById('url').value; - let server_address = document.getElementById('url').value; - - let url = server_address + + const url = server_address + + (~server_address.indexOf('?') ? '&' : '?') + /// Ask server to allow cross-domain requests. - '?add_http_cors_header=1' + + 'add_http_cors_header=1' + '&user=' + encodeURIComponent(user) + '&password=' + encodeURIComponent(password) + '&default_format=JSONCompact' + /// Safety settings to prevent results that browser cannot display. '&max_result_rows=1000&max_result_bytes=10000000&result_overflow_mode=break'; - let xhr = new XMLHttpRequest; + const xhr = new XMLHttpRequest; xhr.open('POST', url, true); @@ -391,12 +390,12 @@ /// The query is saved in browser history (in state JSON object) /// as well as in URL fragment identifier. if (query != previous_query) { - let state = { + const state = { query: query, status: this.status, response: this.response.length > 100000 ? null : this.response /// Lower than the browser's limit. }; - let title = "ClickHouse Query: " + query; + const title = "ClickHouse Query: " + query; let history_url = window.location.pathname + '?user=' + encodeURIComponent(user); if (server_address != location.origin) { From 77ade599d368dbc0d9c22790167d4fe2c9e3c508 Mon Sep 17 00:00:00 2001 From: Dmitriy Lushnikov <40070832+lushndm@users.noreply.github.com> Date: Wed, 4 Aug 2021 12:52:30 +0300 Subject: [PATCH 113/198] Update index.md --- docs/ru/engines/table-engines/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/index.md b/docs/ru/engines/table-engines/index.md index b17b2124250..1636efc5112 100644 --- a/docs/ru/engines/table-engines/index.md +++ b/docs/ru/engines/table-engines/index.md @@ -87,7 +87,7 @@ toc_title: "Введение" Виртуальный столбец — это неотъемлемый атрибут движка таблиц, определенный в исходном коде движка. -Виртуальные столбцы не надо указывать в запросе `CREATE TABLE` и их не отображаются в результатах запросов `SHOW CREATE TABLE` и `DESCRIBE TABLE`. Также виртуальные столбцы доступны только для чтения, поэтому вы не можете вставлять в них данные. +Виртуальные столбцы не надо указывать в запросе `CREATE TABLE` и они не отображаются в результатах запросов `SHOW CREATE TABLE` и `DESCRIBE TABLE`. Также виртуальные столбцы доступны только для чтения, поэтому вы не можете вставлять в них данные. Чтобы получить данные из виртуального столбца, необходимо указать его название в запросе `SELECT`. `SELECT *` не отображает данные из виртуальных столбцов. From 9b9982ee3dc6f3649190aad6a19308f44e6e3135 Mon Sep 17 00:00:00 2001 From: Artur Filatenkov <613623@mail.ru> Date: Wed, 4 Aug 2021 12:53:41 +0300 Subject: [PATCH 114/198] correct test with deleting files after usage --- .../queries/0_stateless/02001_compress_output_file.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/queries/0_stateless/02001_compress_output_file.sh b/tests/queries/0_stateless/02001_compress_output_file.sh index 336c8319ee4..11df227cc14 100755 --- a/tests/queries/0_stateless/02001_compress_output_file.sh +++ b/tests/queries/0_stateless/02001_compress_output_file.sh @@ -4,10 +4,20 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +set -e + +[ -e "${CLICKHOUSE_TMP}"/test_compression_of_output_file_from_client.gz ] && rm "${CLICKHOUSE_TMP}"/test_compression_of_output_file_from_client.gz + ${CLICKHOUSE_CLIENT} --query "SELECT * FROM (SELECT 'Hello, World! From client.') INTO OUTFILE '${CLICKHOUSE_TMP}/test_compression_of_output_file_from_client.gz'" gunzip ${CLICKHOUSE_TMP}/test_compression_of_output_file_from_client.gz cat ${CLICKHOUSE_TMP}/test_compression_of_output_file_from_client +rm -f "${CLICKHOUSE_TMP}/test_compression_of_output_file_from_client" + +[ -e "${CLICKHOUSE_TMP}"/test_compression_of_output_file_from_local.gz ] && rm "${CLICKHOUSE_TMP}"/test_compression_of_output_file_from_local.gz + ${CLICKHOUSE_LOCAL} --query "SELECT * FROM (SELECT 'Hello, World! From local.') INTO OUTFILE '${CLICKHOUSE_TMP}/test_compression_of_output_file_from_local.gz'" gunzip ${CLICKHOUSE_TMP}/test_compression_of_output_file_from_local.gz cat ${CLICKHOUSE_TMP}/test_compression_of_output_file_from_local + +rm -f "${CLICKHOUSE_TMP}/test_compression_of_output_file_from_local" From 659b874934e896d2eaed2aa3f309c552583ed980 Mon Sep 17 00:00:00 2001 From: Dmitriy Lushnikov <40070832+lushndm@users.noreply.github.com> Date: Wed, 4 Aug 2021 13:02:11 +0300 Subject: [PATCH 115/198] Update gui.md --- docs/ru/interfaces/third-party/gui.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/interfaces/third-party/gui.md b/docs/ru/interfaces/third-party/gui.md index dc96c32e996..9cb28a2c9a2 100644 --- a/docs/ru/interfaces/third-party/gui.md +++ b/docs/ru/interfaces/third-party/gui.md @@ -111,7 +111,7 @@ toc_title: "Визуальные интерфейсы от сторонних р ### DataGrip {#datagrip} -[DataGrip](https://www.jetbrains.com/datagrip/) — это IDE для баз данных о JetBrains с выделенной поддержкой ClickHouse. Он также встроен в другие инструменты на основе IntelliJ: PyCharm, IntelliJ IDEA, GoLand, PhpStorm и другие. +[DataGrip](https://www.jetbrains.com/datagrip/) — это IDE для баз данных от JetBrains с выделенной поддержкой ClickHouse. Он также встроен в другие инструменты на основе IntelliJ: PyCharm, IntelliJ IDEA, GoLand, PhpStorm и другие. Основные возможности: From bb8c6813a1c5e1225fda84a688575adf86a42297 Mon Sep 17 00:00:00 2001 From: adevyatova Date: Wed, 4 Aug 2021 10:15:50 +0000 Subject: [PATCH 116/198] Update --- .../functions/other-functions.md | 114 +----------------- 1 file changed, 3 insertions(+), 111 deletions(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 63892bc712c..1826318217b 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -2148,48 +2148,12 @@ Short description. ``` -Alias: ``. (Optional) - -More text (Optional). - -**Arguments** (Optional) - -- `x` — Description. Optional (only for optional arguments). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type). -- `y` — Description. Optional (only for optional arguments). Possible values: .Default value: . [Type name](relative/path/to/type/dscr.md#type). - -**Parameters** (Optional, only for parametric aggregate functions) - -- `z` — Description. Optional (only for optional parameters). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type). - -**Returned value(s)** +**Returned value** - Returned values list. Type: [Type name](relative/path/to/type/dscr.md#type). -**Example** - -The example must show usage and/or a use cases. The following text contains recommended parts of an example. - -Input table (Optional): - -``` text -``` - -Query: - -``` sql -``` - -Result: - -``` text -``` - -**See Also** (Optional) - -- [link](#) - ## enabledProfiles {#enabled-profiles} Short description. @@ -2200,48 +2164,12 @@ Short description. ``` -Alias: ``. (Optional) - -More text (Optional). - -**Arguments** (Optional) - -- `x` — Description. Optional (only for optional arguments). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type). -- `y` — Description. Optional (only for optional arguments). Possible values: .Default value: . [Type name](relative/path/to/type/dscr.md#type). - -**Parameters** (Optional, only for parametric aggregate functions) - -- `z` — Description. Optional (only for optional parameters). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type). - -**Returned value(s)** +**Returned value** - Returned values list. Type: [Type name](relative/path/to/type/dscr.md#type). -**Example** - -The example must show usage and/or a use cases. The following text contains recommended parts of an example. - -Input table (Optional): - -``` text -``` - -Query: - -``` sql -``` - -Result: - -``` text -``` - -**See Also** (Optional) - -- [link](#) - ## defaultProfiles {#default-profiles} Short description. @@ -2252,44 +2180,8 @@ Short description. ``` -Alias: ``. (Optional) - -More text (Optional). - -**Arguments** (Optional) - -- `x` — Description. Optional (only for optional arguments). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type). -- `y` — Description. Optional (only for optional arguments). Possible values: .Default value: . [Type name](relative/path/to/type/dscr.md#type). - -**Parameters** (Optional, only for parametric aggregate functions) - -- `z` — Description. Optional (only for optional parameters). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type). - -**Returned value(s)** +**Returned value** - Returned values list. Type: [Type name](relative/path/to/type/dscr.md#type). - -**Example** - -The example must show usage and/or a use cases. The following text contains recommended parts of an example. - -Input table (Optional): - -``` text -``` - -Query: - -``` sql -``` - -Result: - -``` text -``` - -**See Also** (Optional) - -- [link](#) \ No newline at end of file From 5c39ae981c9998209e882196a6c732bb78b09951 Mon Sep 17 00:00:00 2001 From: adevyatova Date: Wed, 4 Aug 2021 11:26:24 +0000 Subject: [PATCH 117/198] Add desc for currentProfiles, enabledProfiles, defaultProfiles func --- .../en/engines/database-engines/replicated.md | 5 ---- .../functions/other-functions.md | 26 ++++++++++--------- .../ru/engines/database-engines/replicated.md | 5 ---- 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/docs/en/engines/database-engines/replicated.md b/docs/en/engines/database-engines/replicated.md index 1ac4d34b0de..ed8406e6a86 100644 --- a/docs/en/engines/database-engines/replicated.md +++ b/docs/en/engines/database-engines/replicated.md @@ -1,8 +1,3 @@ ---- -toc_priority: 40 -toc_title: Replicated ---- - # [experimental] Replicated {#replicated} The engine is based on the [Atomic](../../engines/database-engines/atomic.md) engine. It supports replication of metadata via DDL log being written to ZooKeeper and executed on all of the replicas for a given database. diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 1826318217b..fca5afaced3 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -2140,48 +2140,50 @@ Result: ## currentProfiles {#current-profiles} -Short description. +Returns list of the current [setting profiles](../../operations/access-rights/#settings-profiles-management) for current user. The command [SET PROFILE](../../sql-reference/statements/set.md#set-statement#query-set) could be used to change the current setting profile. **Syntax** ``` sql - +currentProfiles() ``` **Returned value** -- Returned values list. +- List of the current setting profiles. -Type: [Type name](relative/path/to/type/dscr.md#type). +Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). ## enabledProfiles {#enabled-profiles} -Short description. + Returns setting profiles are assigned to the user both explicitly and implicitly. Implicitly assigned profiles include parent profiles of other assigned profiles, profiles assigned via granted roles, profiles assigned via their own settings, and the main default profile (see the section `default_profile` in the main configuration file). + + If some profile occurs multiple times (with some other settings in between), the latest occurrence overrides all the previous ones. **Syntax** ``` sql - +enabledProfiles() ``` **Returned value** -- Returned values list. +- List of the enabled setting profiles. -Type: [Type name](relative/path/to/type/dscr.md#type). +Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). ## defaultProfiles {#default-profiles} -Short description. +Returns the default setting profiles, i.e. the settings which should be applied on the user's login. **Syntax** ``` sql - +defaultProfiles() ``` **Returned value** -- Returned values list. +- List of the default setting profiles. -Type: [Type name](relative/path/to/type/dscr.md#type). +Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). diff --git a/docs/ru/engines/database-engines/replicated.md b/docs/ru/engines/database-engines/replicated.md index c70d1aa25c0..c0e3c6af89c 100644 --- a/docs/ru/engines/database-engines/replicated.md +++ b/docs/ru/engines/database-engines/replicated.md @@ -1,8 +1,3 @@ ---- -toc_priority: 40 -toc_title: Replicated ---- - # [экспериментальный] Replicated {#replicated} Движок основан на движке [Atomic](../../engines/database-engines/atomic.md). Он поддерживает репликацию метаданных через журнал DDL, записываемый в ZooKeeper и выполняемый на всех репликах для данной базы данных. From 5bd665bfba463d97238c55d7e4e710187163b3e5 Mon Sep 17 00:00:00 2001 From: adevyatova Date: Wed, 4 Aug 2021 11:47:27 +0000 Subject: [PATCH 118/198] Add ru docs --- .../functions/other-functions.md | 4 +- .../ru/engines/database-engines/replicated.md | 1 + .../functions/other-functions.md | 49 +++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index fca5afaced3..7866dfc9c84 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -2140,7 +2140,7 @@ Result: ## currentProfiles {#current-profiles} -Returns list of the current [setting profiles](../../operations/access-rights/#settings-profiles-management) for current user. The command [SET PROFILE](../../sql-reference/statements/set.md#set-statement#query-set) could be used to change the current setting profile. +Returns list of the current [setting profiles](../../operations/access-rights.md#settings-profiles-management) for current user. The command [SET PROFILE](../../sql-reference/statements/set.md#set-statement#query-set) could be used to change the current setting profile. **Syntax** @@ -2156,7 +2156,7 @@ Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-refere ## enabledProfiles {#enabled-profiles} - Returns setting profiles are assigned to the user both explicitly and implicitly. Implicitly assigned profiles include parent profiles of other assigned profiles, profiles assigned via granted roles, profiles assigned via their own settings, and the main default profile (see the section `default_profile` in the main configuration file). + Returns setting profiles are assigned to the user both explicitly and implicitly. Implicitly assigned profiles to include parent profiles of other assigned profiles, profiles assigned via granted roles, profiles assigned via their own settings, and the main default profile (see the section `default_profile` in the main configuration file). If some profile occurs multiple times (with some other settings in between), the latest occurrence overrides all the previous ones. diff --git a/docs/ru/engines/database-engines/replicated.md b/docs/ru/engines/database-engines/replicated.md index c0e3c6af89c..f1d5755647a 100644 --- a/docs/ru/engines/database-engines/replicated.md +++ b/docs/ru/engines/database-engines/replicated.md @@ -1,3 +1,4 @@ + # [экспериментальный] Replicated {#replicated} Движок основан на движке [Atomic](../../engines/database-engines/atomic.md). Он поддерживает репликацию метаданных через журнал DDL, записываемый в ZooKeeper и выполняемый на всех репликах для данной базы данных. diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index a07bd19faa1..f6f7578a187 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -2088,3 +2088,52 @@ SELECT tcpPort(); - [tcp_port](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port) +## currentProfiles {#current-profiles} + +Возвращает список [профилей настроек](../../operations/access-rights.md#settings-profiles-management) для текущего пользователя. Для изменения текущего профиля настроек может быть использована команда [SET PROFILE](../../sql-reference/statements/set.md#set-statement#query-set). + +**Синтаксис** + +``` sql +currentProfiles() +``` + +**Возвращаемое значение** + +- Список профилей настроек для текущего пользователя. + +Тип: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). + +## enabledProfiles {#enabled-profiles} + +Возвращает профили настроек, назначенные пользователю как явно, так и неявно. Неявно назначенные профили включают родительские профили других назначенных профилей, профили, назначенные с помощью предоставленных ролей, профили, назначенные с помощью собственных настроек, и основной профиль по умолчанию (см. секцию `default_profile` в основном конфигурационном файле). + + Если какой-либо профиль встречается несколько раз (с некоторыми другими настройками между ними), последнее событие перезаписывает все предыдущие. + +**Синтаксис** + +``` sql +enabledProfiles() +``` + +**Возвращаемое значение** + +- Список доступных профилей для текущего пользователя. + +Тип: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). + +## defaultProfiles {#default-profiles} + +Возвращает профили настроек по умолчанию, т. е. профили, которые должны быть применены при входе пользователя в систему. + +**Синтаксис** + +``` sql +defaultProfiles() +``` + +**Возвращаемое значение** + +- Список профилей по умолчанию. + +ТИп: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). \ No newline at end of file From 61ddc53e261c27e5d738d2a0be3fd9b04c72953b Mon Sep 17 00:00:00 2001 From: adevyatova Date: Wed, 4 Aug 2021 11:51:22 +0000 Subject: [PATCH 119/198] Fixed links --- docs/en/sql-reference/functions/other-functions.md | 2 +- docs/ru/sql-reference/functions/other-functions.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 7866dfc9c84..dcd19cbf9b4 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -2140,7 +2140,7 @@ Result: ## currentProfiles {#current-profiles} -Returns list of the current [setting profiles](../../operations/access-rights.md#settings-profiles-management) for current user. The command [SET PROFILE](../../sql-reference/statements/set.md#set-statement#query-set) could be used to change the current setting profile. +Returns list of the current [setting profiles](../../operations/access-rights.md#settings-profiles-management) for current user. The command [SET PROFILE](../../sql-reference/statements/set.md#query-set) could be used to change the current setting profile. **Syntax** diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index f6f7578a187..4df98f71458 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -2090,7 +2090,7 @@ SELECT tcpPort(); ## currentProfiles {#current-profiles} -Возвращает список [профилей настроек](../../operations/access-rights.md#settings-profiles-management) для текущего пользователя. Для изменения текущего профиля настроек может быть использована команда [SET PROFILE](../../sql-reference/statements/set.md#set-statement#query-set). +Возвращает список [профилей настроек](../../operations/access-rights.md#settings-profiles-management) для текущего пользователя. Для изменения текущего профиля настроек может быть использована команда [SET PROFILE](../../sql-reference/statements/set.md#query-set). **Синтаксис** From d8ce46ea747a62c65c3c589d88c1d2f576fb3be5 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Wed, 4 Aug 2021 17:20:35 +0500 Subject: [PATCH 120/198] Show more details in testflows checks --- .../tests/array_tuple_map.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py b/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py index 04150d945f6..698e81ca1ac 100644 --- a/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py +++ b/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py @@ -1,5 +1,6 @@ import uuid +from testflows.asserts import error from extended_precision_data_types.requirements import * from extended_precision_data_types.common import * @@ -410,10 +411,11 @@ def map_func(self, data_type, node=None): sql = (f"INSERT INTO {table_name} SELECT mapPopulateSeries([1,2,3],", f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}, {to_data_type(data_type,3)}], 5)") - exitcode, message = 0, None + r = node.query(sql, exitcode=None, message=None) if data_type.startswith("Decimal"): - exitcode, message = 44, "Exception:" - node.query(sql, exitcode=exitcode, message=message) + assert r.exitcode == 44, error(r.output) + else: + assert r.exitcode == 0, error(f"sql: {sql}\noutput: {r.output}") execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") From 9b9c6e02d162969296371289391a8e0394e6276c Mon Sep 17 00:00:00 2001 From: adevyatova Date: Wed, 4 Aug 2021 12:34:50 +0000 Subject: [PATCH 121/198] Applied suggestion from code review --- docs/en/sql-reference/functions/other-functions.md | 6 +++--- docs/ru/sql-reference/functions/other-functions.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index dcd19cbf9b4..74eeb76577b 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -2140,7 +2140,7 @@ Result: ## currentProfiles {#current-profiles} -Returns list of the current [setting profiles](../../operations/access-rights.md#settings-profiles-management) for current user. The command [SET PROFILE](../../sql-reference/statements/set.md#query-set) could be used to change the current setting profile. +Returns list of the current [setting profiles](../../operations/access-rights.md#settings-profiles-management) for current user. The command [SET PROFILE](../../sql-reference/statements/set.md#query-set) could be used to change the current setting profile. If the command `SET PROFILE` hasn't been called the function returns the profiles specified at the current user's definition (see [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement), `SETTINGS` section). **Syntax** @@ -2156,7 +2156,7 @@ Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-refere ## enabledProfiles {#enabled-profiles} - Returns setting profiles are assigned to the user both explicitly and implicitly. Implicitly assigned profiles to include parent profiles of other assigned profiles, profiles assigned via granted roles, profiles assigned via their own settings, and the main default profile (see the section `default_profile` in the main configuration file). + Returns setting profiles are assigned to the user both explicitly and implicitly. Explicitly assigned profiles are the same profiles as returned by the function [currentProfiles](#current-profiles). Implicitly assigned profiles includes parent profiles of other assigned profiles, profiles assigned via granted roles, profiles assigned via their own settings, and the main default profile (see the section `default_profile` in the main configuration file). If some profile occurs multiple times (with some other settings in between), the latest occurrence overrides all the previous ones. @@ -2174,7 +2174,7 @@ Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-refere ## defaultProfiles {#default-profiles} -Returns the default setting profiles, i.e. the settings which should be applied on the user's login. +Returns all the profiles specified at the current user's definition (see [CREATE USER]()). **Syntax** diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 4df98f71458..4e3e5a638dc 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -2090,7 +2090,7 @@ SELECT tcpPort(); ## currentProfiles {#current-profiles} -Возвращает список [профилей настроек](../../operations/access-rights.md#settings-profiles-management) для текущего пользователя. Для изменения текущего профиля настроек может быть использована команда [SET PROFILE](../../sql-reference/statements/set.md#query-set). +Возвращает список [профилей настроек](../../operations/access-rights.md#settings-profiles-management) для текущего пользователя. Для изменения текущего профиля настроек может быть использована команда [SET PROFILE](../../sql-reference/statements/set.md#query-set). Если команда `SET PROFILE` не использовалась, функция возвращает профили, указанные при объявлении текущего пользователя (см. [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement)). **Синтаксис** @@ -2106,7 +2106,7 @@ currentProfiles() ## enabledProfiles {#enabled-profiles} -Возвращает профили настроек, назначенные пользователю как явно, так и неявно. Неявно назначенные профили включают родительские профили других назначенных профилей, профили, назначенные с помощью предоставленных ролей, профили, назначенные с помощью собственных настроек, и основной профиль по умолчанию (см. секцию `default_profile` в основном конфигурационном файле). +Возвращает профили настроек, назначенные пользователю как явно, так и неявно. Явно назначенные профили — это те же профили, которые возвращает функция [currentProfiles](#current-profiles). Неявно назначенные профили включают родительские профили других назначенных профилей, профили, назначенные с помощью предоставленных ролей, профили, назначенные с помощью собственных настроек, и основной профиль по умолчанию (см. секцию `default_profile` в основном конфигурационном файле). Если какой-либо профиль встречается несколько раз (с некоторыми другими настройками между ними), последнее событие перезаписывает все предыдущие. @@ -2124,7 +2124,7 @@ enabledProfiles() ## defaultProfiles {#default-profiles} -Возвращает профили настроек по умолчанию, т. е. профили, которые должны быть применены при входе пользователя в систему. +Возвращает все профили, указанные при объявлении текущего пользователя (см. [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement), секция `SETTINGS`) **Синтаксис** From 3f48c85722787a3dc6495c39e7ed06b99a35d59c Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 4 Aug 2021 16:19:42 +0300 Subject: [PATCH 122/198] StorageSystemReplicas added column replica_is_active --- docs/en/operations/system-tables/replicas.md | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 7 +++- src/Storages/StorageReplicatedMergeTree.h | 1 + src/Storages/System/StorageSystemReplicas.cpp | 17 +++++++- .../test_replica_is_active/__init__.py | 0 .../test_replica_is_active/test.py | 41 +++++++++++++++++++ 6 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 tests/integration/test_replica_is_active/__init__.py create mode 100644 tests/integration/test_replica_is_active/test.py diff --git a/docs/en/operations/system-tables/replicas.md b/docs/en/operations/system-tables/replicas.md index 5a6ec54723b..e2cc607f6d8 100644 --- a/docs/en/operations/system-tables/replicas.md +++ b/docs/en/operations/system-tables/replicas.md @@ -82,6 +82,7 @@ The next 4 columns have a non-zero value only where there is an active session w - `absolute_delay` (`UInt64`) - How big lag in seconds the current replica has. - `total_replicas` (`UInt8`) - The total number of known replicas of this table. - `active_replicas` (`UInt8`) - The number of replicas of this table that have a session in ZooKeeper (i.e., the number of functioning replicas). +- `replica_is_active` ([Map(String, UInt8)](../../sql-reference/data-types/map.md)) — Map between replica name and is replica active. If you request all the columns, the table may work a bit slowly, since several reads from ZooKeeper are made for each row. If you do not request the last 4 columns (log_max_index, log_pointer, total_replicas, active_replicas), the table works quickly. diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8b6267943bd..08161f8ef46 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5575,8 +5575,11 @@ void StorageReplicatedMergeTree::getStatus(Status & res, bool with_zk_fields) res.total_replicas = all_replicas.size(); for (const String & replica : all_replicas) - if (zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) - ++res.active_replicas; + { + bool is_replica_active = zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active"); + res.active_replicas += static_cast(is_replica_active); + res.replica_is_active.emplace(replica, is_replica_active); + } } catch (const Coordination::Exception &) { diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 0f9d71bd5a5..3d2727d7bb9 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -176,6 +176,7 @@ public: UInt8 active_replicas; /// If the error has happened fetching the info from ZooKeeper, this field will be set. String zookeeper_exception; + std::unordered_map replica_is_active; }; /// Get the status of the table. If with_zk_fields = false - do not fill in the fields that require queries to ZK. diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index fc33c6b421b..5c22d3c2fae 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -51,6 +52,7 @@ StorageSystemReplicas::StorageSystemReplicas(const StorageID & table_id_) { "total_replicas", std::make_shared() }, { "active_replicas", std::make_shared() }, { "zookeeper_exception", std::make_shared() }, + { "replica_is_active", std::make_shared(std::make_shared(), std::make_shared()) } })); setInMemoryMetadata(storage_metadata); } @@ -101,7 +103,8 @@ Pipe StorageSystemReplicas::read( || column_name == "log_pointer" || column_name == "total_replicas" || column_name == "active_replicas" - || column_name == "zookeeper_exception") + || column_name == "zookeeper_exception" + || column_name == "replica_is_active") { with_zk_fields = true; break; @@ -184,6 +187,18 @@ Pipe StorageSystemReplicas::read( res_columns[col_num++]->insert(status.total_replicas); res_columns[col_num++]->insert(status.active_replicas); res_columns[col_num++]->insert(status.zookeeper_exception); + + Map replica_is_active_values; + for (const auto & [name, is_active] : status.replica_is_active) + { + Tuple is_replica_active_value; + is_replica_active_value.emplace_back(name); + is_replica_active_value.emplace_back(is_active); + + replica_is_active_values.emplace_back(std::move(is_replica_active_value)); + } + + res_columns[col_num++]->insert(std::move(replica_is_active_values)); } Block header = metadata_snapshot->getSampleBlock(); diff --git a/tests/integration/test_replica_is_active/__init__.py b/tests/integration/test_replica_is_active/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_replica_is_active/test.py b/tests/integration/test_replica_is_active/test.py new file mode 100644 index 00000000000..14046ea7f7d --- /dev/null +++ b/tests/integration/test_replica_is_active/test.py @@ -0,0 +1,41 @@ +import pytest +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', with_zookeeper=True) +node2 = cluster.add_instance('node2', with_zookeeper=True) +node3 = cluster.add_instance('node3', with_zookeeper=True) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + + for i, node in enumerate((node1, node2, node3)): + node_name = 'node' + str(i + 1) + node.query( + ''' + CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_table', '{}') + PARTITION BY date ORDER BY id + '''.format(node_name) + ) + + yield cluster + + finally: + cluster.shutdown() + + +def test_replica_is_active(start_cluster): + query_result = node1.query("select replica_is_active from system.replicas where table = 'test_table'") + assert query_result == '{\'node1\':1,\'node2\':1,\'node3\':1}\n' + + node3.stop() + query_result = node1.query("select replica_is_active from system.replicas where table = 'test_table'") + assert query_result == '{\'node1\':1,\'node2\':1,\'node3\':0}\n' + + node2.stop() + query_result = node1.query("select replica_is_active from system.replicas where table = 'test_table'") + assert query_result == '{\'node1\':1,\'node2\':0,\'node3\':0}\n' From b8558a1716a9c98b9bfdd1fe3e8259f815d5fd30 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 4 Aug 2021 16:44:39 +0300 Subject: [PATCH 123/198] Fix uninitialized memory in functions multiSearch* with empty array --- src/Functions/FunctionsMultiStringPosition.h | 2 +- src/Functions/MultiSearchFirstIndexImpl.h | 2 +- src/Functions/MultiSearchFirstPositionImpl.h | 2 +- src/Functions/MultiSearchImpl.h | 2 +- .../00233_position_function_family.reference | 16 ++++++++++++++++ .../00233_position_function_family.sql | 17 ++++++++++++++++- 6 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/Functions/FunctionsMultiStringPosition.h b/src/Functions/FunctionsMultiStringPosition.h index f36f7639ccd..74496484c92 100644 --- a/src/Functions/FunctionsMultiStringPosition.h +++ b/src/Functions/FunctionsMultiStringPosition.h @@ -109,7 +109,7 @@ public: auto & vec_res = col_res->getData(); auto & offsets_res = col_offsets->getData(); - vec_res.resize(column_haystack_size * refs.size()); + vec_res.resize_fill(column_haystack_size * refs.size()); if (col_haystack_vector) Impl::vectorConstant(col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res); diff --git a/src/Functions/MultiSearchFirstIndexImpl.h b/src/Functions/MultiSearchFirstIndexImpl.h index bb0bb57b9db..146cb551bb3 100644 --- a/src/Functions/MultiSearchFirstIndexImpl.h +++ b/src/Functions/MultiSearchFirstIndexImpl.h @@ -26,7 +26,7 @@ struct MultiSearchFirstIndexImpl { auto searcher = Impl::createMultiSearcherInBigHaystack(needles); const size_t haystack_string_size = haystack_offsets.size(); - res.resize(haystack_string_size); + res.resize_fill(haystack_string_size); size_t iteration = 0; while (searcher.hasMoreToSearch()) { diff --git a/src/Functions/MultiSearchFirstPositionImpl.h b/src/Functions/MultiSearchFirstPositionImpl.h index 4743518e61e..aee4da43958 100644 --- a/src/Functions/MultiSearchFirstPositionImpl.h +++ b/src/Functions/MultiSearchFirstPositionImpl.h @@ -30,7 +30,7 @@ struct MultiSearchFirstPositionImpl }; auto searcher = Impl::createMultiSearcherInBigHaystack(needles); const size_t haystack_string_size = haystack_offsets.size(); - res.resize(haystack_string_size); + res.resize_fill(haystack_string_size); size_t iteration = 0; while (searcher.hasMoreToSearch()) { diff --git a/src/Functions/MultiSearchImpl.h b/src/Functions/MultiSearchImpl.h index 5b881e2fd55..4580eda27e4 100644 --- a/src/Functions/MultiSearchImpl.h +++ b/src/Functions/MultiSearchImpl.h @@ -26,7 +26,7 @@ struct MultiSearchImpl { auto searcher = Impl::createMultiSearcherInBigHaystack(needles); const size_t haystack_string_size = haystack_offsets.size(); - res.resize(haystack_string_size); + res.resize_fill(haystack_string_size); size_t iteration = 0; while (searcher.hasMoreToSearch()) { diff --git a/tests/queries/0_stateless/00233_position_function_family.reference b/tests/queries/0_stateless/00233_position_function_family.reference index 1523094261f..f6cacbc7027 100644 --- a/tests/queries/0_stateless/00233_position_function_family.reference +++ b/tests/queries/0_stateless/00233_position_function_family.reference @@ -23796,3 +23796,19 @@ 1 1 1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/00233_position_function_family.sql b/tests/queries/0_stateless/00233_position_function_family.sql index 8e443f5cd10..36f8a5535d9 100644 --- a/tests/queries/0_stateless/00233_position_function_family.sql +++ b/tests/queries/0_stateless/00233_position_function_family.sql @@ -483,6 +483,22 @@ select 1 = position('abab', materialize('ab')); select 1 = position('abababababababababababab', materialize('abab')); select 1 = position('abababababababababababab', materialize('abababababababababa')); +select 0 = multiSearchAny('\0', CAST([], 'Array(String)')); +select 0 = multiSearchAnyCaseInsensitive('\0', CAST([], 'Array(String)')); +select 0 = multiSearchAnyCaseInsensitiveUTF8('\0', CAST([], 'Array(String)')); +select 0 = multiSearchAnyUTF8('\0', CAST([], 'Array(String)')); +select 0 = multiSearchFirstIndex('\0', CAST([], 'Array(String)')); +select 0 = multiSearchFirstIndexCaseInsensitive('\0', CAST([], 'Array(String)')); +select 0 = multiSearchFirstIndexCaseInsensitiveUTF8('\0', CAST([], 'Array(String)')); +select 0 = multiSearchFirstIndexUTF8('\0', CAST([], 'Array(String)')); +select 0 = multiSearchFirstPosition('\0', CAST([], 'Array(String)')); +select 0 = multiSearchFirstPositionCaseInsensitive('\0', CAST([], 'Array(String)')); +select 0 = multiSearchFirstPositionCaseInsensitiveUTF8('\0', CAST([], 'Array(String)')); +select 0 = multiSearchFirstPositionUTF8('\0', CAST([], 'Array(String)')); +select [] = multiSearchAllPositions('\0', CAST([], 'Array(String)')); +select [] = multiSearchAllPositionsCaseInsensitive('\0', CAST([], 'Array(String)')); +select [] = multiSearchAllPositionsCaseInsensitiveUTF8('\0', CAST([], 'Array(String)')); +select [] = multiSearchAllPositionsUTF8('\0', CAST([], 'Array(String)')); select [2] = multiSearchAllPositions(materialize('abcdefgh'), ['b']); select [2] = multiSearchAllPositions(materialize('abcdefgh'), ['bc']); @@ -1177,7 +1193,6 @@ select 0 = multiSearchAnyCaseInsensitiveUTF8(materialize('ШНвпкфЗвгДж select 1 = multiSearchAnyCaseInsensitiveUTF8(materialize('ФРХгаСлчЧОцкШгзмКЗшФфББвЧ'), ['кзШфФб', 'ГАслЧЧОцкшг', 'ФфббВЧ', 'ЦкШ', '', 'АслчЧОЦКШгзМкЗШффбБвч', 'РХгаслЧчОЦКШГз', 'РхгаслчЧОцКШгзМкзшФфБбВ', 'Шг', 'Ф', 'ХГАслчЧоцКШГзМкзш', 'ШгЗмКЗшфФб']) from system.numbers limit 10; select 1 = multiSearchAnyCaseInsensitiveUTF8(materialize('ЧдйШкхОлалщНйбССХКаФзОМрКЕЙР'), ['бссХкафзОм', 'ХОЛаЛщнйБссХкаФз', 'лаЛщнйБсСХ', 'ЩнЙбСсхКаФЗО', 'йБСсХКАФЗОмР', 'йшкХолаЛЩНйбсСхК', 'С', '', 'ЙшкхОлалщНЙБсСхКаФзом', 'Йр', 'щнЙБссхКАфзоМрК', 'рКе']) from system.numbers limit 10; - select 1 = multiSearchFirstIndex(materialize('alhpvldsiwsydwhfdasqju'), ['sydwh', 'dwh', 'dwhfdasqj', 'w', 'briozrtpq', 'fdasq', 'lnuvpuxdhhuxjbolw', 'vldsiws', 'dasqju', 'uancllygwoifwnnp', 'wfxputfnen', 'hzaclvjumecnmweungz']) from system.numbers limit 10; select 0 = multiSearchFirstIndex(materialize('kcwchxxyujbhrxkxgnomg'), ['jmvqipszutxfnhdfaxqwoxcz', 'nrgzkbsakdtdiiyphozjoauyughyvlz', 'qbszx', 'sllthykcnttqecpequommemygee', 'bvsbdiufrrrjxaxzxgbd', 'hdkpcmpdyjildw', 'frxkyukiywngfcxfzwkcun', 'dmvxf', 'esamivybor', 'eoggdynqwlnlxr']) from system.numbers limit 10; select 0 = multiSearchFirstIndex(materialize('zggbeilrfpkleafjjldgyfgn'), ['rpypxkpgvljhqthneremvabcd', 'qchzlnsctuwkdxqcrjgihvtfxhqxfqsxm', 'vtozkivjyqcqetmqenuihq', 'fixcvjyzbzejmwdivjf', 'lydoolvnuuamwlnzbyuuwpqqjaxf', 'elkodwthxqpcybwezm', 'wpiju', 'wdzuuwumlqfvga', 'iokphkai', 'wkbwdstplhivjyk', 'wxfbhfturuqoymwklohawgwltptytc', 'jehprkzofqvurepbvuwdqj']) from system.numbers limit 10; From b80065ab59f706ff89a4bf6013689b4c9f1c8ae8 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 4 Aug 2021 22:09:23 +0800 Subject: [PATCH 124/198] Try our best to log query_kind --- src/Interpreters/executeQuery.cpp | 6 +++++- src/Parsers/ASTAlterQuery.h | 2 ++ src/Parsers/ASTCreateQuery.h | 2 ++ src/Parsers/ASTDropQuery.h | 2 ++ src/Parsers/ASTGrantQuery.h | 1 + src/Parsers/ASTInsertQuery.h | 2 ++ src/Parsers/ASTRenameQuery.h | 2 ++ src/Parsers/ASTSelectQuery.h | 2 ++ src/Parsers/ASTSelectWithUnionQuery.h | 2 ++ src/Parsers/ASTSystemQuery.h | 2 ++ src/Parsers/IAST.h | 3 +++ 11 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 75b89554820..78104020a77 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -262,7 +262,11 @@ static void onExceptionBeforeStart(const String & query_for_logging, ContextPtr elem.query = query_for_logging; elem.normalized_query_hash = normalizedQueryHash(query_for_logging); - // We don't calculate query_kind, databases, tables and columns when the query isn't able to start + // Try log query_kind if ast is valid + if (ast) + elem.query_kind = ast->getQueryKindString(); + + // We don't calculate databases, tables and columns when the query isn't able to start elem.exception_code = getCurrentExceptionCode(); elem.exception = getCurrentExceptionMessage(false); diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h index 5fc146a3072..0fd6d2805ea 100644 --- a/src/Parsers/ASTAlterQuery.h +++ b/src/Parsers/ASTAlterQuery.h @@ -225,6 +225,8 @@ public: return removeOnCluster(clone(), new_database); } + const char * getQueryKindString() const override { return "Alter"; } + protected: void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index c7be67d9b78..e4f2b628886 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -102,6 +102,8 @@ public: bool isView() const { return is_ordinary_view || is_materialized_view || is_live_view; } + const char * getQueryKindString() const override { return "Create"; } + protected: void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; diff --git a/src/Parsers/ASTDropQuery.h b/src/Parsers/ASTDropQuery.h index b612618aaec..6e5fd5854d8 100644 --- a/src/Parsers/ASTDropQuery.h +++ b/src/Parsers/ASTDropQuery.h @@ -45,6 +45,8 @@ public: return removeOnCluster(clone(), new_database); } + const char * getQueryKindString() const override { return "Drop"; } + protected: void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; diff --git a/src/Parsers/ASTGrantQuery.h b/src/Parsers/ASTGrantQuery.h index 184f2e143ee..b0fb64cb33e 100644 --- a/src/Parsers/ASTGrantQuery.h +++ b/src/Parsers/ASTGrantQuery.h @@ -34,5 +34,6 @@ public: void replaceEmptyDatabase(const String & current_database); void replaceCurrentUserTag(const String & current_user_name) const; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } + const char * getQueryKindString() const override { return is_revoke ? "Revoke" : "Grant"; } }; } diff --git a/src/Parsers/ASTInsertQuery.h b/src/Parsers/ASTInsertQuery.h index 982f310fdb3..a454f46c3f1 100644 --- a/src/Parsers/ASTInsertQuery.h +++ b/src/Parsers/ASTInsertQuery.h @@ -47,6 +47,8 @@ public: return res; } + const char * getQueryKindString() const override { return "Insert"; } + protected: void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; diff --git a/src/Parsers/ASTRenameQuery.h b/src/Parsers/ASTRenameQuery.h index 611f81dc9e9..5739e7839d3 100644 --- a/src/Parsers/ASTRenameQuery.h +++ b/src/Parsers/ASTRenameQuery.h @@ -61,6 +61,8 @@ public: return query_ptr; } + const char * getQueryKindString() const override { return "Rename"; } + protected: void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override { diff --git a/src/Parsers/ASTSelectQuery.h b/src/Parsers/ASTSelectQuery.h index db4d7e76320..9336ec7611b 100644 --- a/src/Parsers/ASTSelectQuery.h +++ b/src/Parsers/ASTSelectQuery.h @@ -95,6 +95,8 @@ public: void setFinal(); + const char * getQueryKindString() const override { return "Select"; } + protected: void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; diff --git a/src/Parsers/ASTSelectWithUnionQuery.h b/src/Parsers/ASTSelectWithUnionQuery.h index ecf03bb6a05..0465bdac3a6 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.h +++ b/src/Parsers/ASTSelectWithUnionQuery.h @@ -16,6 +16,8 @@ public: ASTPtr clone() const override; void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + const char * getQueryKindString() const override { return "Select"; } + enum class Mode { Unspecified, diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index cbe82cd936f..fa7b6ece59a 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -86,6 +86,8 @@ public: return removeOnCluster(clone(), new_database); } + const char * getQueryKindString() const override { return "System"; } + protected: void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index 143094e1d7a..94a7b1a52ab 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -231,6 +231,9 @@ public: void cloneChildren(); + // Return query_kind string representation of this AST query. + virtual const char * getQueryKindString() const { return ""; } + public: /// For syntax highlighting. static const char * hilite_keyword; From 23f8b3d07de2ae4c2d6221f258af34222bd6078f Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 4 Aug 2021 17:42:48 +0300 Subject: [PATCH 125/198] fix part name parsing in system.detached_parts --- src/Storages/MergeTree/DataPartsExchange.cpp | 4 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 3 + src/Storages/MergeTree/MergeTreePartInfo.cpp | 31 +++++++- src/Storages/MergeTree/MergeTreePartInfo.h | 5 ++ tests/integration/test_partition/test.py | 79 +++++++++++++++++++ .../0_stateless/01160_detached_parts.sql | 0 6 files changed, 117 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/01160_detached_parts.sql diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index e30da82416d..6ff9c16dad5 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -712,7 +712,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( MergeTreeData::DataPart::Checksums & checksums, ThrottlerPtr throttler) { - static const String TMP_PREFIX = "tmp_fetch_"; + static const String TMP_PREFIX = "tmp-fetch_"; String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_; /// We will remove directory if it's already exists. Make precautions. @@ -784,7 +784,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( LOG_DEBUG(log, "Downloading Part {} unique id {} metadata onto disk {}.", part_name, part_id, disk->getName()); - static const String TMP_PREFIX = "tmp_fetch_"; + static const String TMP_PREFIX = "tmp-fetch_"; String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_; String part_relative_path = String(to_detached ? "detached/" : "") + tmp_prefix + part_name; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index ea965f10b23..db271cc280b 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1322,6 +1322,9 @@ String IMergeTreeDataPart::getRelativePathForDetachedPart(const String & prefix) { /// Do not allow underscores in the prefix because they are used as separators. assert(prefix.find_first_of('_') == String::npos); + assert(prefix.empty() || std::find(DetachedPartInfo::DETACH_REASONS.begin(), + DetachedPartInfo::DETACH_REASONS.end(), + prefix) != DetachedPartInfo::DETACH_REASONS.end()); return "detached/" + getRelativePathForPrefix(prefix); } diff --git a/src/Storages/MergeTree/MergeTreePartInfo.cpp b/src/Storages/MergeTree/MergeTreePartInfo.cpp index 24c18843935..ccb26a0999e 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -247,13 +247,39 @@ String MergeTreePartInfo::getPartNameV0(DayNum left_date, DayNum right_date) con return wb.str(); } + +const std::vector DetachedPartInfo::DETACH_REASONS = + { + "broken", + "unexpected", + "noquorum", + "ignored", + "broken-on-start", + "clone", + "attaching", + "deleting", + "tmp-fetch", + }; + bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, DetachedPartInfo & part_info, MergeTreeDataFormatVersion format_version) { part_info.dir_name = dir_name; - /// First, try to parse as . - // TODO what if tryParsePartName will parse prefix as partition_id? It can happen if dir_name doesn't contain mutation number at the end + /// First, try to find known prefix and parse dir_name as _. + /// Arbitrary strings are not allowed for partition_id, so known_prefix cannot be confused with partition_id. + for (const auto & known_prefix : DETACH_REASONS) + { + if (dir_name.starts_with(known_prefix) && known_prefix.size() < dir_name.size() && dir_name[known_prefix.size()] == '_') + { + part_info.prefix = known_prefix; + String part_name = dir_name.substr(known_prefix.size() + 1); + bool parsed = MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version); + return part_info.valid_name = parsed; + } + } + + /// Next, try to parse dir_name as . if (MergeTreePartInfo::tryParsePartName(dir_name, &part_info, format_version)) return part_info.valid_name = true; @@ -263,7 +289,6 @@ bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, Detache if (first_separator == String::npos) return part_info.valid_name = false; - // TODO what if contains '_'? const auto part_name = dir_name.substr(first_separator + 1, dir_name.size() - first_separator - 1); if (!MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) diff --git a/src/Storages/MergeTree/MergeTreePartInfo.h b/src/Storages/MergeTree/MergeTreePartInfo.h index 063f234e50f..87f96ed5038 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/src/Storages/MergeTree/MergeTreePartInfo.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -115,6 +116,10 @@ struct DetachedPartInfo : public MergeTreePartInfo /// If false, MergeTreePartInfo is in invalid state (directory name was not successfully parsed). bool valid_name; + static const std::vector DETACH_REASONS; + + /// NOTE: It may parse part info incorrectly. + /// For example, if prefix contain '_' or if DETACH_REASONS doesn't contain prefix. static bool tryParseDetachedPartName(const String & dir_name, DetachedPartInfo & part_info, MergeTreeDataFormatVersion format_version); }; diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py index baac5367c00..b43c85a4d48 100644 --- a/tests/integration/test_partition/test.py +++ b/tests/integration/test_partition/test.py @@ -236,3 +236,82 @@ def test_drop_detached_parts(drop_detached_parts_table): q("ALTER TABLE test.drop_detached DROP DETACHED PARTITION 1", settings=s) detached = q("SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='test' ORDER BY name") assert TSV(detached) == TSV('0_3_3_0\nattaching_0_6_6_0\ndeleting_0_7_7_0') + +def test_system_detached_parts(drop_detached_parts_table): + q("create table sdp_0 (n int, x int) engine=MergeTree order by n") + q("create table sdp_1 (n int, x int) engine=MergeTree order by n partition by x") + q("create table sdp_2 (n int, x String) engine=MergeTree order by n partition by x") + q("create table sdp_3 (n int, x Enum('broken' = 0, 'all' = 1)) engine=MergeTree order by n partition by x") + + for i in range(0, 4): + q("system stop merges sdp_{}".format(i)) + q("insert into sdp_{} values (0, 0)".format(i)) + q("insert into sdp_{} values (1, 1)".format(i)) + for p in q("select distinct partition_id from system.parts where table='sdp_{}'".format(i))[:-1].split('\n'): + q("alter table sdp_{} detach partition id '{}'".format(i, p)) + + path_to_detached = path_to_data + 'data/default/sdp_{}/detached/{}' + for i in range(0, 4): + instance.exec_in_container(['mkdir', path_to_detached.format(i, 'attaching_0_6_6_0')]) + instance.exec_in_container(['mkdir', path_to_detached.format(i, 'deleting_0_7_7_0')]) + instance.exec_in_container(['mkdir', path_to_detached.format(i, 'any_other_name')]) + instance.exec_in_container(['mkdir', path_to_detached.format(i, 'prefix_1_2_2_0_0')]) + + instance.exec_in_container(['mkdir', path_to_detached.format(i, 'ignored_202107_714380_714380_0')]) + instance.exec_in_container(['mkdir', path_to_detached.format(i, 'broken_202107_714380_714380_123')]) + instance.exec_in_container(['mkdir', path_to_detached.format(i, 'clone_all_714380_714380_42')]) + instance.exec_in_container(['mkdir', path_to_detached.format(i, 'clone_all_714380_714380_42_123')]) + instance.exec_in_container(['mkdir', path_to_detached.format(i, 'broken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123')]) + + res = q("select * from system.detached_parts where table like 'sdp_%' order by table, name") + assert res == \ + "default\tsdp_0\tall\tall_1_1_0\tdefault\t\t1\t1\t0\n" \ + "default\tsdp_0\tall\tall_2_2_0\tdefault\t\t2\t2\t0\n" \ + "default\tsdp_0\t\\N\tany_other_name\tdefault\t\\N\t\\N\t\\N\t\\N\n" \ + "default\tsdp_0\t0\tattaching_0_6_6_0\tdefault\tattaching\t6\t6\t0\n" \ + "default\tsdp_0\t6711e2b2592d86d18fc0f260cf33ef2b\tbroken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123\tdefault\tbroken-on-start\t714380\t714380\t42\n" \ + "default\tsdp_0\t202107\tbroken_202107_714380_714380_123\tdefault\tbroken\t714380\t714380\t123\n" \ + "default\tsdp_0\tall\tclone_all_714380_714380_42\tdefault\tclone\t714380\t714380\t42\n" \ + "default\tsdp_0\tall\tclone_all_714380_714380_42_123\tdefault\tclone\t714380\t714380\t42\n" \ + "default\tsdp_0\t0\tdeleting_0_7_7_0\tdefault\tdeleting\t7\t7\t0\n" \ + "default\tsdp_0\t202107\tignored_202107_714380_714380_0\tdefault\tignored\t714380\t714380\t0\n" \ + "default\tsdp_0\t1\tprefix_1_2_2_0_0\tdefault\tprefix\t2\t2\t0\n" \ + "default\tsdp_1\t0\t0_1_1_0\tdefault\t\t1\t1\t0\n" \ + "default\tsdp_1\t1\t1_2_2_0\tdefault\t\t2\t2\t0\n" \ + "default\tsdp_1\t\\N\tany_other_name\tdefault\t\\N\t\\N\t\\N\t\\N\n" \ + "default\tsdp_1\t0\tattaching_0_6_6_0\tdefault\tattaching\t6\t6\t0\n" \ + "default\tsdp_1\t6711e2b2592d86d18fc0f260cf33ef2b\tbroken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123\tdefault\tbroken-on-start\t714380\t714380\t42\n" \ + "default\tsdp_1\t202107\tbroken_202107_714380_714380_123\tdefault\tbroken\t714380\t714380\t123\n" \ + "default\tsdp_1\tall\tclone_all_714380_714380_42\tdefault\tclone\t714380\t714380\t42\n" \ + "default\tsdp_1\tall\tclone_all_714380_714380_42_123\tdefault\tclone\t714380\t714380\t42\n" \ + "default\tsdp_1\t0\tdeleting_0_7_7_0\tdefault\tdeleting\t7\t7\t0\n" \ + "default\tsdp_1\t202107\tignored_202107_714380_714380_0\tdefault\tignored\t714380\t714380\t0\n" \ + "default\tsdp_1\t1\tprefix_1_2_2_0_0\tdefault\tprefix\t2\t2\t0\n" \ + "default\tsdp_2\t58ed7160db50ea45e1c6aa694c8cbfd1\t58ed7160db50ea45e1c6aa694c8cbfd1_1_1_0\tdefault\t\t1\t1\t0\n" \ + "default\tsdp_2\t6711e2b2592d86d18fc0f260cf33ef2b\t6711e2b2592d86d18fc0f260cf33ef2b_2_2_0\tdefault\t\t2\t2\t0\n" \ + "default\tsdp_2\t\\N\tany_other_name\tdefault\t\\N\t\\N\t\\N\t\\N\n" \ + "default\tsdp_2\t0\tattaching_0_6_6_0\tdefault\tattaching\t6\t6\t0\n" \ + "default\tsdp_2\t6711e2b2592d86d18fc0f260cf33ef2b\tbroken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123\tdefault\tbroken-on-start\t714380\t714380\t42\n" \ + "default\tsdp_2\t202107\tbroken_202107_714380_714380_123\tdefault\tbroken\t714380\t714380\t123\n" \ + "default\tsdp_2\tall\tclone_all_714380_714380_42\tdefault\tclone\t714380\t714380\t42\n" \ + "default\tsdp_2\tall\tclone_all_714380_714380_42_123\tdefault\tclone\t714380\t714380\t42\n" \ + "default\tsdp_2\t0\tdeleting_0_7_7_0\tdefault\tdeleting\t7\t7\t0\n" \ + "default\tsdp_2\t202107\tignored_202107_714380_714380_0\tdefault\tignored\t714380\t714380\t0\n" \ + "default\tsdp_2\t1\tprefix_1_2_2_0_0\tdefault\tprefix\t2\t2\t0\n" \ + "default\tsdp_3\t0\t0_1_1_0\tdefault\t\t1\t1\t0\n" \ + "default\tsdp_3\t1\t1_2_2_0\tdefault\t\t2\t2\t0\n" \ + "default\tsdp_3\t\\N\tany_other_name\tdefault\t\\N\t\\N\t\\N\t\\N\n" \ + "default\tsdp_3\t0\tattaching_0_6_6_0\tdefault\tattaching\t6\t6\t0\n" \ + "default\tsdp_3\t6711e2b2592d86d18fc0f260cf33ef2b\tbroken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123\tdefault\tbroken-on-start\t714380\t714380\t42\n" \ + "default\tsdp_3\t202107\tbroken_202107_714380_714380_123\tdefault\tbroken\t714380\t714380\t123\n" \ + "default\tsdp_3\tall\tclone_all_714380_714380_42\tdefault\tclone\t714380\t714380\t42\n" \ + "default\tsdp_3\tall\tclone_all_714380_714380_42_123\tdefault\tclone\t714380\t714380\t42\n" \ + "default\tsdp_3\t0\tdeleting_0_7_7_0\tdefault\tdeleting\t7\t7\t0\n" \ + "default\tsdp_3\t202107\tignored_202107_714380_714380_0\tdefault\tignored\t714380\t714380\t0\n" \ + "default\tsdp_3\t1\tprefix_1_2_2_0_0\tdefault\tprefix\t2\t2\t0\n" + + for i in range(0, 4): + for p in q("select distinct partition_id from system.detached_parts where table='sdp_{}' and partition_id is not null".format(i))[:-1].split('\n'): + q("alter table sdp_{} attach partition id '{}'".format(i, p)) + + assert q("select n, x, count() from merge('default', 'sdp_') group by n, x") == "0\t0\t4\n1\t1\t4\n" diff --git a/tests/queries/0_stateless/01160_detached_parts.sql b/tests/queries/0_stateless/01160_detached_parts.sql new file mode 100644 index 00000000000..e69de29bb2d From 0730e685b0ae976e72155858865a5953ef55bb4b Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 4 Aug 2021 17:45:32 +0300 Subject: [PATCH 126/198] Mark window functions ready for general use. --- .../sql-reference/window-functions/index.md | 3 --- src/Core/Settings.h | 1 - src/Interpreters/ExpressionAnalyzer.cpp | 12 ----------- tests/performance/window_functions.xml | 4 ---- ...mize_distributed_group_by_sharding_key.sql | 2 +- ...568_window_functions_distributed.reference | 1 - .../01568_window_functions_distributed.sql | 2 -- .../01571_window_functions.reference | 2 +- .../0_stateless/01571_window_functions.sql | 1 - .../01591_window_functions.reference | 2 -- .../0_stateless/01591_window_functions.sql | 2 -- .../01592_long_window_functions1.sql | 1 - .../0_stateless/01592_window_functions.sql | 2 -- ...802_rank_corr_mann_whitney_over_window.sql | 2 -- .../window_functions/tests/common.py | 20 ------------------- .../window_functions/tests/feature.py | 3 --- 16 files changed, 2 insertions(+), 58 deletions(-) diff --git a/docs/en/sql-reference/window-functions/index.md b/docs/en/sql-reference/window-functions/index.md index dcab019c9d5..e62808a46bd 100644 --- a/docs/en/sql-reference/window-functions/index.md +++ b/docs/en/sql-reference/window-functions/index.md @@ -5,9 +5,6 @@ toc_title: Window Functions # [experimental] Window Functions -!!! warning "Warning" - This is an experimental feature that is currently in development and is not ready for general use. It will change in unpredictable backwards-incompatible ways in the future releases. Set `allow_experimental_window_functions = 1` to enable it. - ClickHouse supports the standard grammar for defining windows and window functions. The following features are currently supported: | Feature | Support or workaround | diff --git a/src/Core/Settings.h b/src/Core/Settings.h index c8befbb81b2..d311b0b5c60 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -449,7 +449,6 @@ class IColumn; M(Bool, optimize_skip_merged_partitions, false, "Skip partitions with one part with level > 0 in optimize final", 0) \ M(Bool, optimize_on_insert, true, "Do the same transformation for inserted block of data as if merge was done on this block.", 0) \ M(Bool, allow_experimental_map_type, true, "Obsolete setting, does nothing.", 0) \ - M(Bool, allow_experimental_window_functions, false, "Allow experimental window functions", 0) \ M(Bool, allow_experimental_projection_optimization, false, "Enable projection optimization when processing SELECT queries", 0) \ M(Bool, force_optimize_projection, false, "If projection optimization is enabled, SELECT queries need to use projection", 0) \ M(Bool, async_socket_for_remote, true, "Asynchronously read from socket executing remote query", 0) \ diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 875a7bef862..ad1ee703785 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -608,18 +608,6 @@ void makeWindowDescriptionFromAST(const Context & context, void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions) { - // Convenient to check here because at least we have the Context. - if (!syntax->window_function_asts.empty() && - !getContext()->getSettingsRef().allow_experimental_window_functions) - { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, - "The support for window functions is experimental and will change" - " in backwards-incompatible ways in the future releases. Set" - " allow_experimental_window_functions = 1 to enable it." - " While processing '{}'", - syntax->window_function_asts[0]->formatForErrorMessage()); - } - // Window definitions from the WINDOW clause const auto * select_query = query->as(); if (select_query && select_query->window()) diff --git a/tests/performance/window_functions.xml b/tests/performance/window_functions.xml index 6be3d59e2b0..e3d30d96ec3 100644 --- a/tests/performance/window_functions.xml +++ b/tests/performance/window_functions.xml @@ -3,10 +3,6 @@ hits_100m_single - - 1 - -