diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md index 716e774871b..17e7f1f18cc 100644 --- a/docs/en/interfaces/third-party/integrations.md +++ b/docs/en/interfaces/third-party/integrations.md @@ -12,6 +12,7 @@ toc_title: Integrations - Relational database management systems - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/es/interfaces/third-party/integrations.md b/docs/es/interfaces/third-party/integrations.md index 716e774871b..17e7f1f18cc 100644 --- a/docs/es/interfaces/third-party/integrations.md +++ b/docs/es/interfaces/third-party/integrations.md @@ -12,6 +12,7 @@ toc_title: Integrations - Relational database management systems - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/fa/interfaces/third-party/integrations.md b/docs/fa/interfaces/third-party/integrations.md index 657432c7958..df864ef71e6 100644 --- a/docs/fa/interfaces/third-party/integrations.md +++ b/docs/fa/interfaces/third-party/integrations.md @@ -14,6 +14,7 @@ toc_title: "\u06CC\u06A9\u067E\u0627\u0631\u0686\u06AF\u06CC" - سیستم های مدیریت پایگاه داده رابطه ای - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [در حال بارگذاری](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [تاتر-خروجی زیر-داده خوان](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/fr/interfaces/third-party/integrations.md b/docs/fr/interfaces/third-party/integrations.md index f252fd6229b..8332ffe5e59 100644 --- a/docs/fr/interfaces/third-party/integrations.md +++ b/docs/fr/interfaces/third-party/integrations.md @@ -14,6 +14,7 @@ toc_title: "Int\xE9gration" - Systèmes de gestion de bases de données relationnelles - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-lecteur de données](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-réplicateur](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/ja/interfaces/third-party/integrations.md b/docs/ja/interfaces/third-party/integrations.md index 3e38d578093..2ac2ad24410 100644 --- a/docs/ja/interfaces/third-party/integrations.md +++ b/docs/ja/interfaces/third-party/integrations.md @@ -14,6 +14,7 @@ toc_title: "\u7D71\u5408" - リレーショナルデータベース管理システム - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-データリーダー](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-レプリケーター](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/ru/interfaces/third-party/integrations.md b/docs/ru/interfaces/third-party/integrations.md index 39449b54df8..19a72edc4d3 100644 --- a/docs/ru/interfaces/third-party/integrations.md +++ b/docs/ru/interfaces/third-party/integrations.md @@ -7,6 +7,7 @@ - Реляционные системы управления базами данных - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/tools/build.py b/docs/tools/build.py index 95e887f046f..b7ddbc29629 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -220,7 +220,7 @@ if __name__ == '__main__': arg_parser.add_argument('--website-dir', default=website_dir) arg_parser.add_argument('--output-dir', default='build') arg_parser.add_argument('--enable-stable-releases', action='store_true') - arg_parser.add_argument('--stable-releases-limit', type=int, default='4') + arg_parser.add_argument('--stable-releases-limit', type=int, default='3') arg_parser.add_argument('--lts-releases-limit', type=int, default='2') arg_parser.add_argument('--nav-limit', type=int, default='0') arg_parser.add_argument('--version-prefix', type=str, default='') diff --git a/docs/tools/translate/requirements.txt b/docs/tools/translate/requirements.txt index 0c9d44a346e..370554fa90f 100644 --- a/docs/tools/translate/requirements.txt +++ b/docs/tools/translate/requirements.txt @@ -1,7 +1,7 @@ Babel==2.8.0 certifi==2020.4.5.2 chardet==3.0.4 -googletrans==2.4.0 +googletrans==3.0.0 idna==2.9 Jinja2==2.11.2 pandocfilters==1.4.2 diff --git a/docs/tr/interfaces/third-party/integrations.md b/docs/tr/interfaces/third-party/integrations.md index 8a1d5c239f6..a5e5a60c72f 100644 --- a/docs/tr/interfaces/third-party/integrations.md +++ b/docs/tr/interfaces/third-party/integrations.md @@ -14,6 +14,7 @@ toc_title: Entegrasyonlar - İlişkisel veritabanı yönetim sistemleri - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-çoğaltıcı](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/zh/interfaces/third-party/integrations.md b/docs/zh/interfaces/third-party/integrations.md index 014fdc88304..e0f308fecde 100644 --- a/docs/zh/interfaces/third-party/integrations.md +++ b/docs/zh/interfaces/third-party/integrations.md @@ -7,6 +7,7 @@ - 关系数据库管理系统 - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-复制器](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/zh/sql-reference/syntax.md b/docs/zh/sql-reference/syntax.md index b0aa9e7364f..687638a9be6 100644 --- a/docs/zh/sql-reference/syntax.md +++ b/docs/zh/sql-reference/syntax.md @@ -1,156 +1,162 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 -toc_title: "\u8BED\u6CD5" +toc_title: SQL语法 --- -# 语法 {#syntax} - -系统中有两种类型的解析器：完整SQL解析器（递归下降解析器）和数据格式解析器（快速流解析器）。 -在所有情况下，除了 `INSERT` 查询时，只使用完整的SQL解析器。 -该 `INSERT` 查询使用两个解析器: +# SQL语法 {#syntax} +CH有2类解析器：完整SQL解析器（递归式解析器），以及数据格式解析器（快速流式解析器） +除了 `INSERT` 查询，其它情况下仅使用完整SQL解析器。 + `INSERT`查询会同时使用2种解析器： ``` sql INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') ``` -该 `INSERT INTO t VALUES` 片段由完整的解析器解析，并且数据 `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` 由快速流解析器解析。 您也可以通过使用 [input\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) 设置。 当 `input_format_values_interpret_expressions = 1`，ClickHouse首先尝试使用fast stream解析器解析值。 如果失败，ClickHouse将尝试对数据使用完整的解析器，将其视为SQL [表达式](#syntax-expressions). +含`INSERT INTO t VALUES` 的部分由完整SQL解析器处理，包含数据的部分 `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` 交给快速流式解析器解析。通过设置参数 [input\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions)，你也可以对数据部分开启完整SQL解析器。当 `input_format_values_interpret_expressions = 1` 时，CH优先采用快速流式解析器来解析数据。如果失败，CH再尝试用完整SQL解析器来处理，就像处理SQL [expression](#syntax-expressions) 一样。 -数据可以有任何格式。 当接收到查询时，服务器计算不超过 [max\_query\_size](../operations/settings/settings.md#settings-max_query_size) RAM中请求的字节（默认为1MB），其余的是流解析。 -它允许避免与大的问题 `INSERT` 查询。 +数据可以采用任何格式。当CH接受到请求时，服务端先在内存中计算不超过 [max\_query\_size](../operations/settings/settings.md#settings-max_query_size) 字节的请求数据（默认1 mb），然后剩下部分交给快速流式解析器。 -使用时 `Values` 格式为 `INSERT` 查询，它可能看起来数据被解析相同的表达式 `SELECT` 查询，但事实并非如此。 该 `Values` 格式更为有限。 +这将避免在处理大型的 `INSERT`语句时出现问题。 -本文的其余部分将介绍完整的解析器。 有关格式解析器的详细信息，请参阅 [格式](../interfaces/formats.md) 科。 +当 `INSERT` 语句中使用 `Values` 形式时，看起来 数据部分的解析和解析`SELECT` 中的表达式相同，但并不是这样的。 `Values` 形式非常有限。 +该篇的剩余部分涵盖了完整SQL解析器。关于格式解析的更多信息，参见 [Formats](../interfaces/formats.md) 章节。 -## 空间 {#spaces} +## 空字符 {#spaces} -语法结构之间可能有任意数量的空格符号（包括查询的开始和结束）。 空格符号包括空格、制表符、换行符、CR和换页符。 +sql语句中（包含sql的起始和结束）可以有任意的空字符，这些空字符类型包括：空格字符，tab制表符，换行符，CR符，换页符等。 -## 评论 {#comments} +## 注释 {#comments} -ClickHouse支持SQL风格和C风格的注释。 -SQL风格的注释以下开头 `--` 并继续到线的末尾，一个空格后 `--` 可以省略。 -C型是从 `/*` 到 `*/`并且可以是多行，也不需要空格。 +CH支持SQL风格或C语言风格的注释： +- SQL风格的注释以 `--` 开始，直到行末，`--` 后紧跟的空格可以忽略 +- C语言风格的注释以 `/*` 开始，以 `*/` 结束，支持多行形式，同样可以省略 `/*` 后的空格 -## 关键词 {#syntax-keywords} +## 关键字 {#syntax-keywords} -当关键字对应于以下关键字时，不区分大小写: +以下场景的关键字是大小写不敏感的： +- 标准SQL。例如，`SELECT`, `select` 和 `SeLeCt` 都是允许的 +- 在某些流行的RDBMS中被实现的关键字，例如，`DateTime` 和 `datetime`是一样的 -- SQL标准。 例如, `SELECT`, `select` 和 `SeLeCt` 都是有效的。 -- 在一些流行的DBMS（MySQL或Postgres）中实现。 例如, `DateTime` 是一样的 `datetime`. -数据类型名称是否区分大小写可以在 `system.data_type_families` 桌子 +你可以在系统表 [system.data_type_families](../operations/system-tables.md#system_tables-data_type_families) 中检查某个数据类型的名称是否是大小写敏感型。 -与标准SQL相比，所有其他关键字（包括函数名称）都是 **区分大小写**. +和标准SQL相反，所有其它的关键字都是 **大小写敏感的**，包括函数名称。 +In contrast to standard SQL, all other keywords (including functions names) are **case-sensitive**. -不保留关键字；它们仅在相应的上下文中被视为保留关键字。 如果您使用 [标识符](#syntax-identifiers) 使用与关键字相同的名称，将它们括在双引号或反引号中。 例如，查询 `SELECT "FROM" FROM table_name` 是有效的，如果表 `table_name` 具有名称的列 `"FROM"`. +关键字不是保留的；它们仅在相应的上下文中才会被处理。如果你使用和关键字同名的 [变量名](#syntax-identifiers) ，需要使用双引号或转移符将它们包含起来。例如：如果表 `table_name` 包含列 `"FROM"`，那么 `SELECT "FROM" FROM table_name` 是合法的 -## 标识符 {#syntax-identifiers} +## 变量名 {#syntax-identifiers} -标识符是: +变量包括： +Identifiers are: -- 集群、数据库、表、分区和列名称。 -- 功能。 -- 数据类型。 -- [表达式别名](#syntax-expression_aliases). +- 集群，数据库，表，分区，列名称 +- 函数 +- 数据类型 +- 表达式别名 -标识符可以是引号或非引号。 后者是优选的。 +变量名可以使用反引号包含起来 -非引号标识符必须与正则表达式匹配 `^[a-zA-Z_][0-9a-zA-Z_]*$` 并且不能等于 [关键词](#syntax-keywords). 例: `x, _1, X_y__Z123_.` +没有使用反引号包含的变量名，必须匹配正则表达式 `^[a-zA-Z_][0-9a-zA-Z_]*$`，并且不能和 [关键字]相同 -如果要使用与关键字相同的标识符，或者要在标识符中使用其他符号，请使用双引号或反引号对其进行引用，例如, `"id"`, `` `id` ``. +如果想使用和关键字同名的变量名称，或者在变量名称中包含其它符号，你需要通过双引号或转义符号，例如： `"id"`, `` `id` `` -## 文字数 {#literals} +## 字符 {#literals} -有数字，字符串，复合和 `NULL` 文字。 +CH包含数字，字母，括号，NULL值等字符 ### 数字 {#numeric} -数值文字尝试进行分析: +数字类型字符会被做如下解析： +- 首先，当做64位的有符号整数，使用该函数 [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) +- 如果失败，解析成64位无符号整数，同样使用函数 [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) -- 首先，作为一个64位有符号的数字，使用 [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) 功能。 -- 如果不成功，作为64位无符号数，使用 [strtoll](https://en.cppreference.com/w/cpp/string/byte/strtol) 功能。 -- 如果不成功，作为一个浮点数使用 [strtod](https://en.cppreference.com/w/cpp/string/byte/strtof) 功能。 -- 否则，将返回错误。 +- 如果还失败了，试图解析成浮点型数值，使用函数 [strtod](https://en.cppreference.com/w/cpp/string/byte/strtof) +Numeric literal tries to be parsed: -文本值具有该值适合的最小类型。 -例如，1被解析为 `UInt8`，但256被解析为 `UInt16`. 有关详细信息，请参阅 [数据类型](../sql-reference/data-types/index.md). +- 最后，以上情形都不符合时，返回异常 -例: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`. -### 字符串 {#syntax-string-literal} +数字类型的值类型为能容纳该值的最小数据类型。 +例如：1 解析成 `UInt8`型，256 则解析成 `UInt16`。更多信息，参见 [数据类型](../sql-reference/data-types/index.md) -仅支持单引号中的字符串文字。 封闭的字符可以反斜杠转义。 以下转义序列具有相应的特殊值: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. 在所有其他情况下，转义序列的格式为 `\c`，哪里 `c` 是任何字符，被转换为 `c`. 这意味着你可以使用序列 `\'`和`\\`. 该值将具有 [字符串](../sql-reference/data-types/string.md) 类型。 +例如: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`. -在字符串文字中，你至少需要转义 `'` 和 `\`. 单引号可以用单引号，文字转义 `'It\'s'` 和 `'It''s'` 是平等的。 +### 字母 {#syntax-string-literal} +CH只支持用单引号包含的字母。特殊字符可通过反斜杠进行转义。下列转义字符都有相应的实际值： `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`。其它情况下，以 `\c`形式出现的转义字符，当`c`表示任意字符时，转义字符会转换成`c`。这意味着你可以使用 `\'`和`\\`。该值将拥有[String](../sql-reference/data-types/string.md)类型。 -### 化合物 {#compound} -数组使用方括号构造 `[1, 2, 3]`. Nuples用圆括号构造 `(1, 'Hello, world!', 2)`. -从技术上讲，这些不是文字，而是分别具有数组创建运算符和元组创建运算符的表达式。 -数组必须至少包含一个项目，元组必须至少包含两个项目。 -有一个单独的情况下，当元组出现在 `IN` a条款 `SELECT` 查询。 查询结果可以包含元组，但元组不能保存到数据库（除了具有以下内容的表 [记忆](../engines/table-engines/special/memory.md) 发动机）。 +在字符串中，你至少需要对 `'` 和 `\` 进行转义。单引号可以使用单引号转义，例如 `'It\'s'` 和 `'It''s'` 是相同的。 -### NULL {#null-literal} +### 括号 {#compound} +数组都是使用方括号进行构造 `[1, 2, 3]`，元组则使用圆括号 `(1, 'Hello, world!', 2)` -指示该值丢失。 +从技术上来讲，这些都不是字符串，而是包含创建数组和元组运算符的表达式。 -为了存储 `NULL` 在表字段中，它必须是 [可为空](../sql-reference/data-types/nullable.md) 类型。 +创建一个数组必须至少包含一个元素，创建一个元组至少包含2个元素 -根据数据格式（输入或输出), `NULL` 可能有不同的表示。 有关详细信息，请参阅以下文档 [数据格式](../interfaces/formats.md#formats). +当元组出现在 `SELECT` 查询的 `IN` 部分时，是一种例外情形。查询结果可以包含元组，但是元组类型不能保存到数据库中（除非表采用 [内存表](../engines/table-engines/special/memory.md)引擎） -处理有许多细微差别 `NULL`. 例如，如果比较操作的至少一个参数是 `NULL`，此操作的结果也是 `NULL`. 对于乘法，加法和其他操作也是如此。 有关详细信息，请阅读每个操作的文档。 -在查询中，您可以检查 `NULL` 使用 [IS NULL](operators/index.md#operator-is-null) 和 [IS NOT NULL](operators/index.md) 运算符及相关功能 `isNull` 和 `isNotNull`. +### NULL值 {#null-literal} -## 功能 {#functions} +代表不存在的值 -函数调用像一个标识符一样写入，并在圆括号中包含一个参数列表（可能是空的）。 与标准SQL相比，括号是必需的，即使是空的参数列表。 示例: `now()`. -有常规函数和聚合函数（请参阅部分 “Aggregate functions”). 某些聚合函数可以包含括号中的两个参数列表。 示例: `quantile (0.9) (x)`. 这些聚合函数被调用 “parametric” 函数，并在第一个列表中的参数被调用 “parameters”. 不带参数的聚合函数的语法与常规函数的语法相同。 +为了能在表字段中存储NULL值，该字段必须声明为 [空值](../sql-reference/data-types/nullable.md) 类型 +根据数据的格式（输入或输出），NULL值有不同的表现形式。更多信息参见文档 [数据格式](../interfaces/formats.md#formats) -## 运营商 {#operators} +在处理 `NULL`时存在很多细微差别。例如，比较运算的至少一个参数为 `NULL` ，该结果也是 `NULL` 。与之类似的还有乘法运算, 加法运算,以及其它运算。更多信息，请参阅每种运算的文档部分。 -在查询解析过程中，运算符会转换为相应的函数，同时考虑它们的优先级和关联性。 -例如，表达式 `1 + 2 * 3 + 4` 转化为 `plus(plus(1, multiply(2, 3)), 4)`. +在语句中，可以通过 [是否为NULL](operators/index.md#operator-is-null) 以及 [是否不为NULL](operators/index.md) 运算符，以及 `isNull` 、 `isNotNull` 函数来检查 `NULL` 值 -## 数据类型和数据库表引擎 {#data_types-and-database-table-engines} +## 函数 {#functions} +函数调用的写法，类似于变量并带有被圆括号包含的参数列表（可能为空）。与标准SQL不同，圆括号是必须的，不管参数列表是否为空。例如： `now()`。 -数据类型和表引擎 `CREATE` 查询的编写方式与标识符或函数相同。 换句话说，它们可能包含也可能不包含括号中的参数列表。 有关详细信息，请参阅部分 “Data types,” “Table engines,” 和 “CREATE”. +函数分为常规函数和聚合函数（参见“Aggregate functions”一章）。有些聚合函数包含2个参数列表，第一个参数列表中的参数被称为“parameters”。不包含“parameters”的聚合函数语法和常规函数是一样的。 + + +## 运算符 {#operators} + +在查询解析阶段，运算符会被转换成对应的函数，使用时请注意它们的优先级。例如： +表达式 `1 + 2 * 3 + 4` 会被解析成 `plus(plus(1, multiply(2, 3)), 4)`. + + +## 数据类型及数据库/表引擎 {#data_types-and-database-table-engines} + +`CREATE` 语句中的数据类型和表引擎写法与变量或函数类似。 +换句话说，它们可以用括号包含参数列表。更多信息，参见“数据类型,” “数据表引擎” 和 “CREATE语句”等章节 ## 表达式别名 {#syntax-expression_aliases} -别名是查询中表达式的用户定义名称。 +别名是用户对表达式的自定义名称 ``` sql expr AS alias ``` -- `AS` — The keyword for defining aliases. You can define the alias for a table name or a column name in a `SELECT` 子句不使用 `AS` 关键字。 +- `AS` — 用于定义别名的关键字。可以对表或select语句中的列定义别名(`AS` 可以省略） + 例如, `SELECT table_name_alias.column_name FROM table_name table_name_alias`. - For example, `SELECT table_name_alias.column_name FROM table_name table_name_alias`. + 在 [CAST函数](sql_reference/functions/type_conversion_functions.md#type_conversion_function-cast) 中，`AS`有其它含义。请参见该函数的说明部分。 - In the [CAST](sql_reference/functions/type_conversion_functions.md#type_conversion_function-cast) function, the `AS` keyword has another meaning. See the description of the function. -- `expr` — Any expression supported by ClickHouse. +- `expr` — 任意CH支持的表达式. - For example, `SELECT column_name * 2 AS double FROM some_table`. + 例如, `SELECT column_name * 2 AS double FROM some_table`. -- `alias` — Name for `expr`. 别名应符合 [标识符](#syntax-identifiers) 语法 +- `alias` — `expr` 的名称。别名必须符合 [变量名]](#syntax-identifiers) 语法. - For example, `SELECT "table t".column_name FROM table_name AS "table t"`. + 例如, `SELECT "table t".column_name FROM table_name AS "table t"`. -### 使用注意事项 {#notes-on-usage} +### 用法注意 {#notes-on-usage} -别名对于查询或子查询是全局的，您可以在查询的任何部分中为任何表达式定义别名。 例如, `SELECT (1 AS n) + 2, n`. +别名在当前查询或子查询中是全局可见的，你可以在查询语句的任何位置对表达式定义别名 -别名在子查询和子查询之间不可见。 例如，在执行查询时 `SELECT (SELECT sum(b.a) + num FROM b) - a.a AS num FROM a` ClickHouse生成异常 `Unknown identifier: num`. +别名在当前查询的子查询及不同子查询中是不可见的。例如，执行如下查询SQL: `SELECT (SELECT sum(b.a) + num FROM b) - a.a AS num FROM a` ,CH会提示异常 `Unknown identifier: num`. -如果为结果列定义了别名 `SELECT` 子查询的子句，这些列在外部查询中可见。 例如, `SELECT n + m FROM (SELECT 1 AS n, 2 AS m)`. - -小心使用与列或表名相同的别名。 让我们考虑以下示例: +如果给select子查询语句的结果列定义其别名，那么在外层可以使用该别名。例如, `SELECT n + m FROM (SELECT 1 AS n, 2 AS m)`. +注意列的别名和表的别名相同时的情形，考虑如下示例： ``` sql CREATE TABLE t ( @@ -172,16 +178,18 @@ Received exception from server (version 18.14.17): Code: 184. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: Aggregate function sum(b) is found inside another aggregate function in query. ``` -在这个例子中，我们声明表 `t` 带柱 `b`. 然后，在选择数据时，我们定义了 `sum(b) AS b` 别名 由于别名是全局的，ClickHouse替换了文字 `b` 在表达式中 `argMax(a, b)` 用表达式 `sum(b)`. 这种替换导致异常。 +在这个示例中，先声明了表 `t` 以及列 `b`。然后，在查询数据时，又定义了别名 `sum(b) AS b`。由于别名是全局的，CH使用表达式 `sum(b)` 来替换表达式 `argMax(a, b)` 中的变量 `b`。这种替换导致出现异常。 ## 星号 {#asterisk} -在一个 `SELECT` 查询中，星号可以替换表达式。 有关详细信息，请参阅部分 “SELECT”. +select查询中，星号可以代替表达式使用。详情请参见“select”部分 + ## 表达式 {#syntax-expressions} -表达式是函数、标识符、文字、运算符的应用程序、括号中的表达式、子查询或星号。 它还可以包含别名。 -表达式列表是一个或多个用逗号分隔的表达式。 -函数和运算符，反过来，可以有表达式作为参数。 -[原始文章](https://clickhouse.tech/docs/en/sql_reference/syntax/) +An expression is a function, identifier, literal, application of an operator, expression in brackets, subquery, or asterisk. It can also contain an alias. +A list of expressions is one or more expressions separated by commas. +Functions and operators, in turn, can have expressions as arguments. + +[原始文档](https://clickhouse.tech/docs/en/sql_reference/syntax/) diff --git a/src/Common/Arena.h b/src/Common/Arena.h index d203a92d4a3..44a9b444ff2 100644 --- a/src/Common/Arena.h +++ b/src/Common/Arena.h @@ -4,10 +4,10 @@ #include #include #include -#if __has_include() +#include +#if __has_include() && defined(ADDRESS_SANITIZER) # include #endif -#include #include #include #include diff --git a/src/Common/ArenaWithFreeLists.h b/src/Common/ArenaWithFreeLists.h index 6092f03ce19..1284c3586c0 100644 --- a/src/Common/ArenaWithFreeLists.h +++ b/src/Common/ArenaWithFreeLists.h @@ -1,9 +1,9 @@ #pragma once -#if __has_include() +#include +#if __has_include() && defined(ADDRESS_SANITIZER) # include #endif -#include #include #include diff --git a/src/Common/FieldVisitors.h b/src/Common/FieldVisitors.h index 90f80974ab1..ddeddb8fbf6 100644 --- a/src/Common/FieldVisitors.h +++ b/src/Common/FieldVisitors.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include @@ -14,7 +13,6 @@ namespace DB namespace ErrorCodes { extern const int CANNOT_CONVERT_TYPE; - extern const int BAD_TYPE_OF_FIELD; extern const int LOGICAL_ERROR; } @@ -177,243 +175,6 @@ template <> constexpr bool isDecimalField>() { return tr template <> constexpr bool isDecimalField>() { return true; } -/** More precise comparison, used for index. - * Differs from Field::operator< and Field::operator== in that it also compares values of different types. - * Comparison rules are same as in FunctionsComparison (to be consistent with expression evaluation in query). - */ -class FieldVisitorAccurateEquals : public StaticVisitor -{ -public: - bool operator() (const UInt64 &, const Null &) const { return false; } - bool operator() (const UInt64 & l, const UInt64 & r) const { return l == r; } - bool operator() (const UInt64 & l, const UInt128 & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const Int64 & r) const { return accurate::equalsOp(l, r); } - bool operator() (const UInt64 & l, const Float64 & r) const { return accurate::equalsOp(l, r); } - bool operator() (const UInt64 & l, const String & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const Array & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const Tuple & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - - bool operator() (const Int64 &, const Null &) const { return false; } - bool operator() (const Int64 & l, const UInt64 & r) const { return accurate::equalsOp(l, r); } - bool operator() (const Int64 & l, const UInt128 & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const Int64 & r) const { return l == r; } - bool operator() (const Int64 & l, const Float64 & r) const { return accurate::equalsOp(l, r); } - bool operator() (const Int64 & l, const String & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const Array & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const Tuple & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - - bool operator() (const Float64 &, const Null &) const { return false; } - bool operator() (const Float64 & l, const UInt64 & r) const { return accurate::equalsOp(l, r); } - bool operator() (const Float64 & l, const UInt128 & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const Int64 & r) const { return accurate::equalsOp(l, r); } - bool operator() (const Float64 & l, const Float64 & r) const { return l == r; } - bool operator() (const Float64 & l, const String & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const Array & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const Tuple & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - - template - bool operator() (const Null &, const T &) const - { - return std::is_same_v; - } - - template - bool operator() (const String & l, const T & r) const - { - if constexpr (std::is_same_v) - return l == r; - if constexpr (std::is_same_v) - return stringToUUID(l) == r; - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template - bool operator() (const UInt128 & l, const T & r) const - { - if constexpr (std::is_same_v) - return l == r; - if constexpr (std::is_same_v) - return l == stringToUUID(r); - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template - bool operator() (const Array & l, const T & r) const - { - if constexpr (std::is_same_v) - return l == r; - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template - bool operator() (const Tuple & l, const T & r) const - { - if constexpr (std::is_same_v) - return l == r; - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template - bool operator() (const DecimalField & l, const U & r) const - { - if constexpr (isDecimalField()) - return l == r; - if constexpr (std::is_same_v || std::is_same_v) - return l == DecimalField(r, 0); - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template bool operator() (const UInt64 & l, const DecimalField & r) const { return DecimalField(l, 0) == r; } - template bool operator() (const Int64 & l, const DecimalField & r) const { return DecimalField(l, 0) == r; } - template bool operator() (const Float64 & l, const DecimalField & r) const { return cantCompare(l, r); } - - template - bool operator() (const AggregateFunctionStateData & l, const T & r) const - { - if constexpr (std::is_same_v) - return l == r; - return cantCompare(l, r); - } - -private: - template - bool cantCompare(const T &, const U &) const - { - if constexpr (std::is_same_v) - return false; - throw Exception("Cannot compare " + demangle(typeid(T).name()) + " with " + demangle(typeid(U).name()), - ErrorCodes::BAD_TYPE_OF_FIELD); - } -}; - -class FieldVisitorAccurateLess : public StaticVisitor -{ -public: - bool operator() (const UInt64 &, const Null &) const { return false; } - bool operator() (const UInt64 & l, const UInt64 & r) const { return l < r; } - bool operator() (const UInt64 & l, const UInt128 & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const Int64 & r) const { return accurate::lessOp(l, r); } - bool operator() (const UInt64 & l, const Float64 & r) const { return accurate::lessOp(l, r); } - bool operator() (const UInt64 & l, const String & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const Array & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const Tuple & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - - bool operator() (const Int64 &, const Null &) const { return false; } - bool operator() (const Int64 & l, const UInt64 & r) const { return accurate::lessOp(l, r); } - bool operator() (const Int64 & l, const UInt128 & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const Int64 & r) const { return l < r; } - bool operator() (const Int64 & l, const Float64 & r) const { return accurate::lessOp(l, r); } - bool operator() (const Int64 & l, const String & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const Array & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const Tuple & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - - bool operator() (const Float64 &, const Null &) const { return false; } - bool operator() (const Float64 & l, const UInt64 & r) const { return accurate::lessOp(l, r); } - bool operator() (const Float64 & l, const UInt128 & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const Int64 & r) const { return accurate::lessOp(l, r); } - bool operator() (const Float64 & l, const Float64 & r) const { return l < r; } - bool operator() (const Float64 & l, const String & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const Array & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const Tuple & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - - template - bool operator() (const Null &, const T &) const - { - return !std::is_same_v; - } - - template - bool operator() (const String & l, const T & r) const - { - if constexpr (std::is_same_v) - return l < r; - if constexpr (std::is_same_v) - return stringToUUID(l) < r; - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template - bool operator() (const UInt128 & l, const T & r) const - { - if constexpr (std::is_same_v) - return l < r; - if constexpr (std::is_same_v) - return l < stringToUUID(r); - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template - bool operator() (const Array & l, const T & r) const - { - if constexpr (std::is_same_v) - return l < r; - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template - bool operator() (const Tuple & l, const T & r) const - { - if constexpr (std::is_same_v) - return l < r; - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template - bool operator() (const DecimalField & l, const U & r) const - { - if constexpr (isDecimalField()) - return l < r; - if constexpr (std::is_same_v || std::is_same_v) - return l < DecimalField(r, 0); - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template bool operator() (const UInt64 & l, const DecimalField & r) const { return DecimalField(l, 0) < r; } - template bool operator() (const Int64 & l, const DecimalField & r) const { return DecimalField(l, 0) < r; } - template bool operator() (const Float64 &, const DecimalField &) const { return false; } - - template - bool operator() (const AggregateFunctionStateData & l, const T & r) const - { - return cantCompare(l, r); - } - -private: - template - bool cantCompare(const T &, const U &) const - { - throw Exception("Cannot compare " + demangle(typeid(T).name()) + " with " + demangle(typeid(U).name()), - ErrorCodes::BAD_TYPE_OF_FIELD); - } -}; - - /** Implements `+=` operation. * Returns false if the result is zero. */ diff --git a/src/Common/FieldVisitorsAccurateComparison.h b/src/Common/FieldVisitorsAccurateComparison.h new file mode 100644 index 00000000000..91fa4bf28de --- /dev/null +++ b/src/Common/FieldVisitorsAccurateComparison.h @@ -0,0 +1,142 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_TYPE_OF_FIELD; +} + +/** More precise comparison, used for index. + * Differs from Field::operator< and Field::operator== in that it also compares values of different types. + * Comparison rules are same as in FunctionsComparison (to be consistent with expression evaluation in query). + */ +class FieldVisitorAccurateEquals : public StaticVisitor +{ +public: + template + bool operator() (const T & l, const U & r) const + { + if constexpr (std::is_same_v || std::is_same_v) + return std::is_same_v; + else + { + if constexpr (std::is_same_v) + return l == r; + + if constexpr (std::is_arithmetic_v && std::is_arithmetic_v) + return accurate::equalsOp(l, r); + + if constexpr (isDecimalField() && isDecimalField()) + return l == r; + + if constexpr (isDecimalField() && std::is_arithmetic_v) + return l == DecimalField(r, 0); + + if constexpr (std::is_arithmetic_v && isDecimalField()) + return DecimalField(l, 0) == r; + + if constexpr (std::is_same_v) + { + if constexpr (std::is_same_v) + return stringToUUID(l) == r; + + if constexpr (std::is_arithmetic_v) + { + ReadBufferFromString in(l); + T parsed; + readText(parsed, in); + return operator()(parsed, r); + } + } + + if constexpr (std::is_same_v) + { + if constexpr (std::is_same_v) + return l == stringToUUID(r); + + if constexpr (std::is_arithmetic_v) + { + ReadBufferFromString in(r); + T parsed; + readText(parsed, in); + return operator()(l, parsed); + } + } + } + + throw Exception("Cannot compare " + demangle(typeid(T).name()) + " with " + demangle(typeid(U).name()), + ErrorCodes::BAD_TYPE_OF_FIELD); + } +}; + + +class FieldVisitorAccurateLess : public StaticVisitor +{ +public: + template + bool operator() (const T & l, const U & r) const + { + if constexpr (std::is_same_v || std::is_same_v) + return false; + else + { + if constexpr (std::is_same_v) + return l < r; + + if constexpr (std::is_arithmetic_v && std::is_arithmetic_v) + return accurate::lessOp(l, r); + + if constexpr (isDecimalField() && isDecimalField()) + return l < r; + + if constexpr (isDecimalField() && std::is_arithmetic_v) + return l < DecimalField(r, 0); + + if constexpr (std::is_arithmetic_v && isDecimalField()) + return DecimalField(l, 0) < r; + + if constexpr (std::is_same_v) + { + if constexpr (std::is_same_v) + return stringToUUID(l) < r; + + if constexpr (std::is_arithmetic_v) + { + ReadBufferFromString in(l); + T parsed; + readText(parsed, in); + return operator()(parsed, r); + } + } + + if constexpr (std::is_same_v) + { + if constexpr (std::is_same_v) + return l < stringToUUID(r); + + if constexpr (std::is_arithmetic_v) + { + ReadBufferFromString in(r); + T parsed; + readText(parsed, in); + return operator()(l, parsed); + } + } + } + + throw Exception("Cannot compare " + demangle(typeid(T).name()) + " with " + demangle(typeid(U).name()), + ErrorCodes::BAD_TYPE_OF_FIELD); + } +}; + +} diff --git a/src/Core/Defines.h b/src/Core/Defines.h index 13070c565b4..8b26f486c9d 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -87,7 +87,7 @@ #define DBMS_DISTRIBUTED_SIGNATURE_HEADER 0xCAFEDACEull #define DBMS_DISTRIBUTED_SIGNATURE_HEADER_OLD_FORMAT 0xCAFECABEull -#if !__has_include() +#if !__has_include() || !defined(ADDRESS_SANITIZER) # define ASAN_UNPOISON_MEMORY_REGION(a, b) # define ASAN_POISON_MEMORY_REGION(a, b) #endif diff --git a/src/Core/Settings.h b/src/Core/Settings.h index cd9de5abec3..adc804c3a28 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -360,6 +360,7 @@ struct Settings : public SettingsCollection M(SettingBool, optimize_trivial_count_query, true, "Process trivial 'SELECT count() FROM table' query from metadata.", 0) \ M(SettingUInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \ M(SettingBool, optimize_arithmetic_operations_in_aggregate_functions, true, "Move arithmetic operations out of aggregation functions", 0) \ + M(SettingBool, optimize_duplicate_order_by_and_distinct, true, "Remove duplicate ORDER BY and DISTINCT if it's possible", 0) \ M(SettingBool, optimize_if_chain_to_miltiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \ M(SettingBool, allow_experimental_alter_materialized_view_structure, false, "Allow atomic alter on Materialized views. Work in progress.", 0) \ M(SettingBool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there're constants there", 0) \ @@ -376,6 +377,7 @@ struct Settings : public SettingsCollection M(SettingBool, materialize_ttl_after_modify, true, "Apply TTL for old data, after ALTER MODIFY TTL query", 0) \ \ M(SettingBool, allow_experimental_geo_types, false, "Allow geo data types such as Point, Ring, Polygon, MultiPolygon", 0) \ + M(SettingBool, data_type_default_nullable, false, "Data types without NULL or NOT NULL will make Nullable", 0) \ \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index fab1332cbda..50214ee790f 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Interpreters/BloomFilterHash.h b/src/Interpreters/BloomFilterHash.h index e7411433781..43f5d7b5e87 100644 --- a/src/Interpreters/BloomFilterHash.h +++ b/src/Interpreters/BloomFilterHash.h @@ -196,18 +196,17 @@ struct BloomFilterHash const ColumnString::Chars & data = index_column->getChars(); const ColumnString::Offsets & offsets = index_column->getOffsets(); - ColumnString::Offset current_offset = pos; for (size_t index = 0, size = vec.size(); index < size; ++index) { + ColumnString::Offset current_offset = offsets[index + pos - 1]; + size_t length = offsets[index + pos] - current_offset - 1 /* terminating zero */; UInt64 city_hash = CityHash_v1_0_2::CityHash64( - reinterpret_cast(&data[current_offset]), offsets[index + pos] - current_offset - 1); + reinterpret_cast(&data[current_offset]), length); if constexpr (is_first) vec[index] = city_hash; else vec[index] = CityHash_v1_0_2::Hash128to64(CityHash_v1_0_2::uint128(vec[index], city_hash)); - - current_offset = offsets[index + pos]; } } else if (const auto * fixed_string_index_column = typeid_cast(column)) diff --git a/src/Interpreters/CrossToInnerJoinVisitor.cpp b/src/Interpreters/CrossToInnerJoinVisitor.cpp index b2f3f56be4d..5ebebae2578 100644 --- a/src/Interpreters/CrossToInnerJoinVisitor.cpp +++ b/src/Interpreters/CrossToInnerJoinVisitor.cpp @@ -202,11 +202,11 @@ private: { std::optional left_table_pos = IdentifierSemantic::getMembership(left); if (!left_table_pos) - left_table_pos = IdentifierSemantic::chooseTable(left, tables); + left_table_pos = IdentifierSemantic::chooseTableColumnMatch(left, tables); std::optional right_table_pos = IdentifierSemantic::getMembership(right); if (!right_table_pos) - right_table_pos = IdentifierSemantic::chooseTable(right, tables); + right_table_pos = IdentifierSemantic::chooseTableColumnMatch(right, tables); if (left_table_pos && right_table_pos && (*left_table_pos != *right_table_pos)) { diff --git a/src/Interpreters/DuplicateDistinctVisitor.h b/src/Interpreters/DuplicateDistinctVisitor.h new file mode 100644 index 00000000000..9ce2624f5bd --- /dev/null +++ b/src/Interpreters/DuplicateDistinctVisitor.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +/// Removes duplicate DISTINCT from queries. +class DuplicateDistinctMatcher +{ +public: + struct Data + { + bool is_distinct; + std::vector last_ids; + }; + + static void visit(const ASTPtr & ast, Data & data) + { + auto * select_query = ast->as(); + if (select_query) + visit(*select_query, data); + } + + static void visit(ASTSelectQuery & select_query, Data & data) + { + if (!select_query.distinct || !select_query.select()) + return; + + /// Optimize shouldn't work for distributed tables + for (const auto & elem : select_query.children) + { + if (elem->as() && !elem->as()->is_standalone) + return; + } + + auto expression_list = select_query.select(); + std::vector current_ids; + + if (expression_list->children.empty()) + return; + + current_ids.reserve(expression_list->children.size()); + for (const auto & id : expression_list->children) + current_ids.push_back(id->getColumnName()); + + if (data.is_distinct && current_ids == data.last_ids) + select_query.distinct = false; + + data.is_distinct = true; + data.last_ids = std::move(current_ids); + } + + static bool needChildVisit(const ASTPtr &, const ASTPtr &) + { + return true; + } + +}; + +using DuplicateDistinctVisitor = InDepthNodeVisitor; + +} diff --git a/src/Interpreters/DuplicateOrderByVisitor.h b/src/Interpreters/DuplicateOrderByVisitor.h new file mode 100644 index 00000000000..85f34377e54 --- /dev/null +++ b/src/Interpreters/DuplicateOrderByVisitor.h @@ -0,0 +1,127 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +/// Checks if SELECT has stateful functions +class ASTFunctionStatefulData +{ +public: + using TypeToVisit = ASTFunction; + + const Context & context; + bool & is_stateful; + void visit(ASTFunction & ast_function, ASTPtr &) + { + if (ast_function.name == "any" || ast_function.name == "groupArray") + { + is_stateful = true; + return; + } + + const auto & function = FunctionFactory::instance().tryGet(ast_function.name, context); + + if (function && function->isStateful()) + { + is_stateful = true; + return; + } + } +}; + +using ASTFunctionStatefulMatcher = OneTypeMatcher; +using ASTFunctionStatefulVisitor = InDepthNodeVisitor; + + +/// Erases unnecessary ORDER BY from subquery +class DuplicateOrderByFromSubqueriesData +{ +public: + using TypeToVisit = ASTSelectQuery; + + bool done = false; + + void visit(ASTSelectQuery & select_query, ASTPtr &) + { + if (done) + return; + + if (select_query.orderBy() && !select_query.limitBy() && !select_query.limitByOffset() && + !select_query.limitByLength() && !select_query.limitLength() && !select_query.limitOffset()) + { + select_query.setExpression(ASTSelectQuery::Expression::ORDER_BY, nullptr); + } + + done = true; + } +}; + +using DuplicateOrderByFromSubqueriesMatcher = OneTypeMatcher; +using DuplicateOrderByFromSubqueriesVisitor = InDepthNodeVisitor; + + +/// Finds SELECT that can be optimized +class DuplicateOrderByData +{ +public: + using TypeToVisit = ASTSelectQuery; + + const Context & context; + bool done = false; + + void visit(ASTSelectQuery & select_query, ASTPtr &) + { + if (done) + return; + + /// Disable optimization for distributed tables + for (const auto & elem : select_query.children) + { + if (elem->as() && !elem->as()->is_standalone) + return; + } + + if (select_query.orderBy() || select_query.groupBy()) + { + for (auto & elem : select_query.children) + { + if (elem->as()) + { + bool is_stateful = false; + ASTFunctionStatefulVisitor::Data data{context, is_stateful}; + ASTFunctionStatefulVisitor(data).visit(elem); + if (is_stateful) + return; + } + } + + if (auto select_table_ptr = select_query.tables()) + { + if (auto * select_table = select_table_ptr->as()) + { + if (!select_table->children.empty()) + { + DuplicateOrderByFromSubqueriesVisitor::Data data{false}; + DuplicateOrderByFromSubqueriesVisitor(data).visit(select_table->children[0]); + } + } + } + } + } +}; + +using DuplicateOrderByMatcher = OneTypeMatcher; +using DuplicateOrderByVisitor = InDepthNodeVisitor; + +} diff --git a/src/Interpreters/FillingRow.cpp b/src/Interpreters/FillingRow.cpp index dc48b5347c4..7e32d9514a6 100644 --- a/src/Interpreters/FillingRow.cpp +++ b/src/Interpreters/FillingRow.cpp @@ -1,4 +1,6 @@ #include +#include + namespace DB { diff --git a/src/Interpreters/FillingRow.h b/src/Interpreters/FillingRow.h index 1753508e139..0e1d60d0d7a 100644 --- a/src/Interpreters/FillingRow.h +++ b/src/Interpreters/FillingRow.h @@ -1,7 +1,7 @@ #pragma once #include #include -#include + namespace DB { diff --git a/src/Interpreters/IdentifierSemantic.cpp b/src/Interpreters/IdentifierSemantic.cpp index 8f254b50400..f661ec2ae71 100644 --- a/src/Interpreters/IdentifierSemantic.cpp +++ b/src/Interpreters/IdentifierSemantic.cpp @@ -16,7 +16,8 @@ namespace { template -std::optional tryChooseTable(const ASTIdentifier & identifier, const std::vector & tables, bool allow_ambiguous) +std::optional tryChooseTable(const ASTIdentifier & identifier, const std::vector & tables, + bool allow_ambiguous, bool column_match [[maybe_unused]] = false) { using ColumnMatch = IdentifierSemantic::ColumnMatch; @@ -27,6 +28,13 @@ std::optional tryChooseTable(const ASTIdentifier & identifier, const std for (size_t i = 0; i < tables.size(); ++i) { auto match = IdentifierSemantic::canReferColumnToTable(identifier, tables[i]); + + if constexpr (std::is_same_v) + { + if (column_match && match == ColumnMatch::NoMatch && identifier.isShort() && tables[i].hasColumn(identifier.shortName())) + match = ColumnMatch::ColumnName; + } + if (match != ColumnMatch::NoMatch) { if (match > best_match) @@ -125,12 +133,17 @@ std::optional IdentifierSemantic::chooseTable(const ASTIdentifier & iden return tryChooseTable(identifier, tables, ambiguous); } -std::optional IdentifierSemantic::chooseTable(const ASTIdentifier & identifier, const std::vector & tables, - bool ambiguous) +std::optional IdentifierSemantic::chooseTable(const ASTIdentifier & identifier, const TablesWithColumns & tables, bool ambiguous) { return tryChooseTable(identifier, tables, ambiguous); } +std::optional IdentifierSemantic::chooseTableColumnMatch(const ASTIdentifier & identifier, const TablesWithColumns & tables, + bool ambiguous) +{ + return tryChooseTable(identifier, tables, ambiguous, true); +} + StorageID IdentifierSemantic::extractDatabaseAndTable(const ASTIdentifier & identifier) { if (identifier.name_parts.size() > 2) @@ -191,14 +204,9 @@ IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const } IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const ASTIdentifier & identifier, - const TableWithColumnNamesAndTypes & db_and_table) + const TableWithColumnNamesAndTypes & table_with_columns) { - ColumnMatch match = canReferColumnToTable(identifier, db_and_table.table); -#if 0 - if (match == ColumnMatch::NoMatch && identifier.isShort() && db_and_table.hasColumn(identifier.shortName())) - match = ColumnMatch::ColumnName; -#endif - return match; + return canReferColumnToTable(identifier, table_with_columns.table); } /// Strip qualificators from left side of column name. diff --git a/src/Interpreters/IdentifierSemantic.h b/src/Interpreters/IdentifierSemantic.h index 7e84e10a26f..0aef297c734 100644 --- a/src/Interpreters/IdentifierSemantic.h +++ b/src/Interpreters/IdentifierSemantic.h @@ -41,7 +41,7 @@ struct IdentifierSemantic static std::optional extractNestedName(const ASTIdentifier & identifier, const String & table_name); static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); - static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const TableWithColumnNamesAndTypes & db_and_table); + static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const TableWithColumnNamesAndTypes & table_with_columns); static void setColumnShortName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); static void setColumnLongName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); @@ -52,7 +52,9 @@ struct IdentifierSemantic static std::optional getMembership(const ASTIdentifier & identifier); static std::optional chooseTable(const ASTIdentifier &, const std::vector & tables, bool allow_ambiguous = false); - static std::optional chooseTable(const ASTIdentifier &, const std::vector & tables, + static std::optional chooseTable(const ASTIdentifier &, const TablesWithColumns & tables, + bool allow_ambiguous = false); + static std::optional chooseTableColumnMatch(const ASTIdentifier &, const TablesWithColumns & tables, bool allow_ambiguous = false); private: diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 8a7b9a245e4..5d8c43aed0d 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -71,6 +72,7 @@ namespace ErrorCodes extern const int BAD_DATABASE_FOR_TEMPORARY_TABLE; extern const int SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY; extern const int DICTIONARY_ALREADY_EXISTS; + extern const int ILLEGAL_SYNTAX_FOR_DATA_TYPE; extern const int ILLEGAL_COLUMN; } @@ -276,6 +278,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( /** all default_expressions as a single expression list, * mixed with conversion-columns for each explicitly specified type */ + ASTPtr default_expr_list = std::make_shared(); NamesAndTypesList column_names_and_types; @@ -284,9 +287,23 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( const auto & col_decl = ast->as(); DataTypePtr column_type = nullptr; + if (col_decl.type) { column_type = DataTypeFactory::instance().get(col_decl.type); + + if (col_decl.null_modifier) + { + if (column_type->isNullable()) + throw Exception("Cant use [NOT] NULL modifier with Nullable type", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE); + if (*col_decl.null_modifier) + column_type = makeNullable(column_type); + } + else if (context.getSettingsRef().data_type_default_nullable) + { + column_type = makeNullable(column_type); + } + column_names_and_types.emplace_back(col_decl.name, column_type); } else diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index dacd7ca5f20..4890287e81e 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -4,20 +4,15 @@ #include #include #include -#include -#include #include +#include #include #include -#include #include -#include #include #include -#include +#include -#include -#include #include #include @@ -31,56 +26,30 @@ namespace { struct Data { - bool analyzed = false; const Context & context; }; - static bool needChildVisit(ASTPtr &, ASTPtr &) { return true; } + static bool needChildVisit(ASTPtr & node, ASTPtr &) + { + return !node->as(); + } static void visit(ASTPtr & ast, Data & data) { - if (auto * select_query = ast->as()) - visit(*select_query, ast, data); - if (auto * union_select_query = ast->as()) - visit(*union_select_query, ast, data); + if (auto * select = ast->as()) + visit(*select, ast, data); } - static void visit(ASTSelectQuery & select_query, ASTPtr &, Data & data) + static void visit(ASTSelectQuery & select, ASTPtr & node, Data & data) { - if (!select_query.tables()) - return; + InterpreterSelectQuery interpreter( + node, data.context, SelectQueryOptions(QueryProcessingStage::FetchColumns).analyze().modify()); - for (const auto & child : select_query.tables()->children) + const SelectQueryInfo & query_info = interpreter.getQueryInfo(); + if (query_info.view_query) { - auto * tables_element = child->as(); - - if (tables_element && tables_element->table_expression) - visit(*tables_element->table_expression->as(), select_query, data); - } - } - - static void visit(ASTSelectWithUnionQuery &, ASTPtr & node, Data & data) - { - if (!data.analyzed) - { - data.analyzed = true; - InterpreterSelectWithUnionQuery interpreter( - node, data.context, SelectQueryOptions(QueryProcessingStage::FetchColumns).analyze().modify()); - } - } - - static void visit(ASTTableExpression & expression, ASTSelectQuery & select_query, Data & data) - { - if (data.context.getSettingsRef().enable_optimize_predicate_expression && expression.database_and_table_name) - { - if (const auto * identifier = expression.database_and_table_name->as()) - { - auto table_id = data.context.resolveStorageID(*identifier); - const auto & storage = DatabaseCatalog::instance().getTable(table_id, data.context); - - if (auto * storage_view = dynamic_cast(storage.get())) - storage_view->getRuntimeViewQuery(&select_query, data.context, true); - } + ASTPtr tmp; + StorageView::replaceWithSubquery(select, query_info.view_query->clone(), tmp); } } }; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 98cf36cc30b..ac17a3042d8 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include @@ -37,7 +38,7 @@ #include #include #include -#include +#include #include #include #include @@ -186,6 +187,26 @@ static Context getSubqueryContext(const Context & context) return subquery_context; } +static void rewriteMultipleJoins(ASTPtr & query, const TablesWithColumns & tables, const String & database, const Settings & settings) +{ + ASTSelectQuery & select = query->as(); + + Aliases aliases; + if (ASTPtr with = select.with()) + QueryAliasesNoSubqueriesVisitor(aliases).visit(with); + QueryAliasesNoSubqueriesVisitor(aliases).visit(select.select()); + + CrossToInnerJoinVisitor::Data cross_to_inner{tables, aliases, database}; + CrossToInnerJoinVisitor(cross_to_inner).visit(query); + + size_t rewriter_version = settings.multiple_joins_rewriter_version; + if (!rewriter_version || rewriter_version > 2) + throw Exception("Bad multiple_joins_rewriter_version setting value: " + settings.multiple_joins_rewriter_version.toString(), + ErrorCodes::INVALID_SETTING_VALUE); + JoinToSubqueryTransformVisitor::Data join_to_subs_data{tables, aliases, rewriter_version}; + JoinToSubqueryTransformVisitor(join_to_subs_data).visit(query); +} + InterpreterSelectQuery::InterpreterSelectQuery( const ASTPtr & query_ptr_, const Context & context_, @@ -242,29 +263,14 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// Rewrite JOINs if (!has_input && joined_tables.tablesCount() > 1) { - ASTSelectQuery & select = getSelectQuery(); + rewriteMultipleJoins(query_ptr, joined_tables.tablesWithColumns(), context->getCurrentDatabase(), settings); - Aliases aliases; - if (ASTPtr with = select.with()) - QueryAliasesNoSubqueriesVisitor(aliases).visit(with); - QueryAliasesNoSubqueriesVisitor(aliases).visit(select.select()); - - CrossToInnerJoinVisitor::Data cross_to_inner{joined_tables.tablesWithColumns(), aliases, context->getCurrentDatabase()}; - CrossToInnerJoinVisitor(cross_to_inner).visit(query_ptr); - - size_t rewriter_version = settings.multiple_joins_rewriter_version; - if (!rewriter_version || rewriter_version > 2) - throw Exception("Bad multiple_joins_rewriter_version setting value: " + settings.multiple_joins_rewriter_version.toString(), - ErrorCodes::INVALID_SETTING_VALUE); - JoinToSubqueryTransformVisitor::Data join_to_subs_data{joined_tables.tablesWithColumns(), aliases, rewriter_version}; - JoinToSubqueryTransformVisitor(join_to_subs_data).visit(query_ptr); - - joined_tables.reset(select); + joined_tables.reset(getSelectQuery()); joined_tables.resolveTables(); if (storage && joined_tables.isLeftTableSubquery()) { - /// Rewritten with subquery. Free storage here locks here. + /// Rewritten with subquery. Free storage locks here. storage = {}; table_lock.release(); table_id = StorageID::createEmpty(); @@ -288,12 +294,28 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (storage) row_policy_filter = context->getRowPolicyCondition(table_id.getDatabaseName(), table_id.getTableName(), RowPolicy::SELECT_FILTER); + StorageView * view = nullptr; + if (storage) + view = dynamic_cast(storage.get()); + auto analyze = [&] (bool try_move_to_prewhere) { + /// Allow push down and other optimizations for VIEW: replace with subquery and rewrite it. + ASTPtr view_table; + if (view) + view->replaceWithSubquery(getSelectQuery(), view_table); + syntax_analyzer_result = SyntaxAnalyzer(*context).analyzeSelect( query_ptr, SyntaxAnalyzerResult(source_header.getNamesAndTypesList(), storage), options, joined_tables.tablesWithColumns(), required_result_column_names, table_join); + if (view) + { + /// Restore original view name. Save rewritten subquery for future usage in StorageView. + query_info.view_query = view->restoreViewName(getSelectQuery(), view_table); + view = nullptr; + } + if (try_move_to_prewhere && storage && !row_policy_filter && query.where() && !query.prewhere() && !query.final()) { /// PREWHERE optimization: transfer some condition from WHERE to PREWHERE if enabled and viable diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index c60451d5f4a..8ed775f60ae 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -88,6 +88,8 @@ public: size_t getMaxStreams() const { return max_streams; } + const SelectQueryInfo & getQueryInfo() const { return query_info; } + private: InterpreterSelectQuery( const ASTPtr & query_ptr_, diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 7ed1bb9d1bb..9bfa9e1e98b 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -585,8 +585,9 @@ std::vector normalizeColumnNamesExtractNeeded( for (ASTIdentifier * ident : identifiers) { bool got_alias = aliases.count(ident->name); + bool allow_ambiguous = got_alias; /// allow ambiguous column overridden by an alias - if (auto table_pos = IdentifierSemantic::chooseTable(*ident, tables)) + if (auto table_pos = IdentifierSemantic::chooseTableColumnMatch(*ident, tables, allow_ambiguous)) { if (!ident->isShort()) { diff --git a/src/Interpreters/JoinedTables.h b/src/Interpreters/JoinedTables.h index 55244e1225c..2591b49527b 100644 --- a/src/Interpreters/JoinedTables.h +++ b/src/Interpreters/JoinedTables.h @@ -34,7 +34,8 @@ public: void makeFakeTable(StoragePtr storage, const Block & source_header); std::shared_ptr makeTableJoin(const ASTSelectQuery & select_query); - const std::vector & tablesWithColumns() const { return tables_with_columns; } + const TablesWithColumns & tablesWithColumns() const { return tables_with_columns; } + TablesWithColumns moveTablesWithColumns() { return std::move(tables_with_columns); } bool isLeftTableSubquery() const; bool isLeftTableFunction() const; @@ -49,7 +50,7 @@ public: private: Context context; std::vector table_expressions; - std::vector tables_with_columns; + TablesWithColumns tables_with_columns; /// Legacy (duplicated left table values) ASTPtr left_table_expression; diff --git a/src/Interpreters/SyntaxAnalyzer.cpp b/src/Interpreters/SyntaxAnalyzer.cpp index 8f6d368e6ad..4bfae18f9a5 100644 --- a/src/Interpreters/SyntaxAnalyzer.cpp +++ b/src/Interpreters/SyntaxAnalyzer.cpp @@ -23,12 +23,15 @@ #include #include #include +#include +#include #include #include #include #include #include +#include #include #include @@ -370,6 +373,18 @@ void optimizeOrderBy(const ASTSelectQuery * select_query) elems = std::move(unique_elems); } +/// Optimize duplicate ORDER BY and DISTINCT +void optimizeDuplicateOrderByAndDistinct(ASTPtr & query, bool optimize_duplicate_order_by_and_distinct, const Context & context) +{ + if (optimize_duplicate_order_by_and_distinct) + { + DuplicateOrderByVisitor::Data order_by_data{context, false}; + DuplicateOrderByVisitor(order_by_data).visit(query); + DuplicateDistinctVisitor::Data distinct_data{}; + DuplicateDistinctVisitor(distinct_data).visit(query); + } +} + /// Remove duplicate items from LIMIT BY. void optimizeLimitBy(const ASTSelectQuery * select_query) { @@ -831,6 +846,9 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect( /// Remove duplicate items from ORDER BY. optimizeOrderBy(select_query); + /// Remove duplicate ORDER BY and DISTINCT from subqueries. + optimizeDuplicateOrderByAndDistinct(query, settings.optimize_duplicate_order_by_and_distinct, context); + /// Remove duplicated elements from LIMIT BY clause. optimizeLimitBy(select_query); diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index 73bfe9896cc..730e892f8f7 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -58,6 +58,12 @@ void ASTColumnDeclaration::formatImpl(const FormatSettings & settings, FormatSta type->formatImpl(settings, state, frame); } + if (null_modifier) + { + settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") + << (*null_modifier ? "" : "NOT ") << "NULL" << (settings.hilite ? hilite_none : ""); + } + if (default_expression) { settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << default_specifier << (settings.hilite ? hilite_none : "") << ' '; diff --git a/src/Parsers/ASTColumnDeclaration.h b/src/Parsers/ASTColumnDeclaration.h index ad23e0669bc..ea17a8b4dfa 100644 --- a/src/Parsers/ASTColumnDeclaration.h +++ b/src/Parsers/ASTColumnDeclaration.h @@ -13,6 +13,7 @@ class ASTColumnDeclaration : public IAST public: String name; ASTPtr type; + std::optional null_modifier; String default_specifier; ASTPtr default_expression; ASTPtr comment; diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index f8c137fb679..d515fe0084f 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -157,7 +157,7 @@ bool ParserTablePropertyDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expecte ParserIndexDeclaration index_p; ParserConstraintDeclaration constraint_p; - ParserColumnDeclaration column_p; + ParserColumnDeclaration column_p{true, true}; ASTPtr new_node = nullptr; diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index 19410a78dd2..a4fc60a2393 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -92,7 +92,8 @@ template class IParserColumnDeclaration : public IParserBase { public: - explicit IParserColumnDeclaration(bool require_type_ = true) : require_type(require_type_) + explicit IParserColumnDeclaration(bool require_type_ = true, bool allow_null_modifiers_ = false) + : require_type(require_type_), allow_null_modifiers(allow_null_modifiers_) { } @@ -104,6 +105,7 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; bool require_type = true; + bool allow_null_modifiers = false; }; using ParserColumnDeclaration = IParserColumnDeclaration; @@ -115,6 +117,8 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E NameParser name_parser; ParserIdentifierWithOptionalParameters type_parser; ParserKeyword s_default{"DEFAULT"}; + ParserKeyword s_null{"NULL"}; + ParserKeyword s_not{"NOT"}; ParserKeyword s_materialized{"MATERIALIZED"}; ParserKeyword s_alias{"ALIAS"}; ParserKeyword s_comment{"COMMENT"}; @@ -135,6 +139,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E */ ASTPtr type; String default_specifier; + std::optional null_modifier; ASTPtr default_expression; ASTPtr comment_expression; ASTPtr codec_expression; @@ -163,6 +168,17 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E if (require_type && !type && !default_expression) return false; /// reject column name without type + if (type && allow_null_modifiers) + { + if (s_not.ignore(pos, expected)) + { + if (!s_null.ignore(pos, expected)) + return false; + null_modifier.emplace(false); + } + else if (s_null.ignore(pos, expected)) + null_modifier.emplace(true); + } if (s_comment.ignore(pos, expected)) { @@ -193,6 +209,8 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E column_declaration->children.push_back(std::move(type)); } + column_declaration->null_modifier = null_modifier; + if (default_expression) { column_declaration->default_specifier = default_specifier; diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index dad73b6a003..281f8511a59 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include @@ -826,8 +826,8 @@ bool KeyCondition::tryParseAtomFromAST(const ASTPtr & node, const Context & cont } bool cast_not_needed = - is_set_const /// Set args are already casted inside Set::createFromAST - || (isNativeNumber(key_expr_type) && isNativeNumber(const_type)); /// Numbers are accurately compared without cast. + is_set_const /// Set args are already casted inside Set::createFromAST + || (isNativeNumber(key_expr_type) && isNativeNumber(const_type)); /// Numbers are accurately compared without cast. if (!cast_not_needed) castValueToType(key_expr_type, const_value, const_type, node); diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index c4cd1035ea7..26b318f107b 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -70,6 +71,7 @@ using ReadInOrderOptimizerPtr = std::shared_ptr; struct SelectQueryInfo { ASTPtr query; + ASTPtr view_query; /// Optimized VIEW query SyntaxAnalyzerResultPtr syntax_analyzer_result; diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index 97403a359c3..745fc823703 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -1,8 +1,6 @@ #include #include -#include #include -#include #include #include @@ -30,7 +28,6 @@ namespace ErrorCodes { extern const int INCORRECT_QUERY; extern const int LOGICAL_ERROR; - extern const int ALIAS_REQUIRED; } @@ -60,9 +57,12 @@ Pipes StorageView::read( Pipes pipes; ASTPtr current_inner_query = inner_query; - - if (context.getSettings().enable_optimize_predicate_expression) - current_inner_query = getRuntimeViewQuery(*query_info.query->as(), context); + if (query_info.view_query) + { + if (!query_info.view_query->as()) + throw Exception("Unexpected optimized VIEW query", ErrorCodes::LOGICAL_ERROR); + current_inner_query = query_info.view_query->clone(); + } InterpreterSelectWithUnionQuery interpreter(current_inner_query, context, {}, column_names); @@ -87,60 +87,52 @@ Pipes StorageView::read( return pipes; } -ASTPtr StorageView::getRuntimeViewQuery(const ASTSelectQuery & outer_query, const Context & context) +static ASTTableExpression * getFirstTableExpression(ASTSelectQuery & select_query) { - auto temp_outer_query = outer_query.clone(); - auto * new_outer_select = temp_outer_query->as(); - return getRuntimeViewQuery(new_outer_select, context, false); -} - - -static void replaceTableNameWithSubquery(ASTSelectQuery * select_query, ASTPtr & subquery) -{ - auto * select_element = select_query->tables()->children[0]->as(); + auto * select_element = select_query.tables()->children[0]->as(); if (!select_element->table_expression) throw Exception("Logical error: incorrect table expression", ErrorCodes::LOGICAL_ERROR); - auto * table_expression = select_element->table_expression->as(); + return select_element->table_expression->as(); +} + +void StorageView::replaceWithSubquery(ASTSelectQuery & outer_query, ASTPtr view_query, ASTPtr & view_name) +{ + ASTTableExpression * table_expression = getFirstTableExpression(outer_query); if (!table_expression->database_and_table_name) throw Exception("Logical error: incorrect table expression", ErrorCodes::LOGICAL_ERROR); - const auto alias = table_expression->database_and_table_name->tryGetAlias(); + DatabaseAndTableWithAlias db_table(table_expression->database_and_table_name); + String alias = db_table.alias.empty() ? db_table.table : db_table.alias; + + view_name = table_expression->database_and_table_name; table_expression->database_and_table_name = {}; table_expression->subquery = std::make_shared(); - table_expression->subquery->children.push_back(subquery); - table_expression->children.push_back(table_expression->subquery); - if (!alias.empty()) - table_expression->subquery->setAlias(alias); + table_expression->subquery->children.push_back(view_query); + table_expression->subquery->setAlias(alias); + + for (auto & child : table_expression->children) + if (child.get() == view_name.get()) + child = view_query; } - -ASTPtr StorageView::getRuntimeViewQuery(ASTSelectQuery * outer_query, const Context & context, bool normalize) +ASTPtr StorageView::restoreViewName(ASTSelectQuery & select_query, const ASTPtr & view_name) { - auto runtime_view_query = inner_query->clone(); + ASTTableExpression * table_expression = getFirstTableExpression(select_query); - /// TODO: remove getTableExpressions and getTablesWithColumns - { - const auto & table_expressions = getTableExpressions(*outer_query); - const auto & tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context); + if (!table_expression->subquery) + throw Exception("Logical error: incorrect table expression", ErrorCodes::LOGICAL_ERROR); - replaceTableNameWithSubquery(outer_query, runtime_view_query); - if (context.getSettingsRef().joined_subquery_requires_alias && tables_with_columns.size() > 1) - { - for (const auto & pr : tables_with_columns) - if (pr.table.table.empty() && pr.table.alias.empty()) - throw Exception("Not unique subquery in FROM requires an alias (or joined_subquery_requires_alias=0 to disable restriction).", - ErrorCodes::ALIAS_REQUIRED); - } + ASTPtr subquery = table_expression->subquery; + table_expression->subquery = {}; + table_expression->database_and_table_name = view_name; - if (PredicateExpressionsOptimizer(context, tables_with_columns, context.getSettings()).optimize(*outer_query) && normalize) - InterpreterSelectWithUnionQuery( - runtime_view_query, context, SelectQueryOptions(QueryProcessingStage::FetchColumns).analyze().modify(), {}); - } - - return runtime_view_query; + for (auto & child : table_expression->children) + if (child.get() == subquery.get()) + child = view_name; + return subquery->children[0]; } void registerStorageView(StorageFactory & factory) diff --git a/src/Storages/StorageView.h b/src/Storages/StorageView.h index 86550db83ce..61885460249 100644 --- a/src/Storages/StorageView.h +++ b/src/Storages/StorageView.h @@ -29,9 +29,13 @@ public: size_t max_block_size, unsigned num_streams) override; - ASTPtr getRuntimeViewQuery(const ASTSelectQuery & outer_query, const Context & context); + void replaceWithSubquery(ASTSelectQuery & select_query, ASTPtr & view_name) const + { + replaceWithSubquery(select_query, inner_query->clone(), view_name); + } - ASTPtr getRuntimeViewQuery(ASTSelectQuery * outer_query, const Context & context, bool normalize); + static void replaceWithSubquery(ASTSelectQuery & outer_query, ASTPtr view_query, ASTPtr & view_name); + static ASTPtr restoreViewName(ASTSelectQuery & select_query, const ASTPtr & view_name); private: ASTPtr inner_query; diff --git a/tests/performance/duplicate_order_by_and_distinct.xml b/tests/performance/duplicate_order_by_and_distinct.xml new file mode 100644 index 00000000000..0c05af3fc56 --- /dev/null +++ b/tests/performance/duplicate_order_by_and_distinct.xml @@ -0,0 +1,10 @@ + + + hits_10m_single + + + SELECT * FROM (SELECT CounterID, EventDate FROM hits_10m_single ORDER BY CounterID DESC) ORDER BY EventDate, CounterID FORMAT Null + SELECT DISTINCT * FROM (SELECT DISTINCT CounterID, EventDate FROM hits_10m_single) FORMAT Null + SELECT DISTINCT * FROM (SELECT DISTINCT CounterID, EventDate FROM hits_10m_single ORDER BY CounterID DESC) ORDER BY toStartOfWeek(EventDate) FORMAT Null + + diff --git a/tests/queries/0_stateless/00945_bloom_filter_index.sql b/tests/queries/0_stateless/00945_bloom_filter_index.sql index 6f93ae89a42..d509b99229a 100755 --- a/tests/queries/0_stateless/00945_bloom_filter_index.sql +++ b/tests/queries/0_stateless/00945_bloom_filter_index.sql @@ -43,7 +43,7 @@ SELECT COUNT() FROM bloom_filter_types_test WHERE f32 = 1 SETTINGS max_rows_to_r SELECT COUNT() FROM bloom_filter_types_test WHERE f64 = 1 SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_types_test WHERE date = '1970-01-02' SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_types_test WHERE date_time = toDateTime('1970-01-01 03:00:01', 'Europe/Moscow') SETTINGS max_rows_to_read = 6; -SELECT COUNT() FROM bloom_filter_types_test WHERE str = '1' SETTINGS max_rows_to_read = 6; +SELECT COUNT() FROM bloom_filter_types_test WHERE str = '1' SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_types_test WHERE fixed_string = toFixedString('1', 5) SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_types_test WHERE str IN ( SELECT str FROM bloom_filter_types_test); @@ -122,7 +122,7 @@ SELECT COUNT() FROM bloom_filter_null_types_test WHERE f32 = 1 SETTINGS max_rows SELECT COUNT() FROM bloom_filter_null_types_test WHERE f64 = 1 SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_null_types_test WHERE date = '1970-01-02' SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_null_types_test WHERE date_time = toDateTime('1970-01-01 03:00:01', 'Europe/Moscow') SETTINGS max_rows_to_read = 6; -SELECT COUNT() FROM bloom_filter_null_types_test WHERE str = '1' SETTINGS max_rows_to_read = 6; +SELECT COUNT() FROM bloom_filter_null_types_test WHERE str = '1' SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_null_types_test WHERE fixed_string = toFixedString('1', 5) SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_null_types_test WHERE isNull(i8); @@ -150,7 +150,7 @@ CREATE TABLE bloom_filter_lc_null_types_test (order_key UInt64, str LowCardinali INSERT INTO bloom_filter_lc_null_types_test SELECT number AS order_key, toString(number) AS str, toFixedString(toString(number), 5) AS fixed_string FROM system.numbers LIMIT 100; INSERT INTO bloom_filter_lc_null_types_test SELECT 0 AS order_key, NULL AS str, NULL AS fixed_string; -SELECT COUNT() FROM bloom_filter_lc_null_types_test WHERE str = '1' SETTINGS max_rows_to_read = 6; +SELECT COUNT() FROM bloom_filter_lc_null_types_test WHERE str = '1' SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_lc_null_types_test WHERE fixed_string = toFixedString('1', 5) SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_lc_null_types_test WHERE isNull(str); diff --git a/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference b/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference index e2c3b5dab4a..5cc0a546e27 100644 --- a/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference +++ b/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference @@ -1,4 +1,4 @@ -SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM default.test\n WHERE id = 1\n)\nWHERE id = 1 -SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM default.test\n WHERE id = 2\n)\nWHERE id = 2 -SELECT id\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM default.test\n WHERE id = 1\n)\nWHERE id = 1 -SELECT id\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM default.test\n WHERE id = 1\n) AS s\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT *\n FROM default.test\n HAVING id = 1\n) AS test_view\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT *\n FROM default.test\n HAVING id = 2\n) AS test_view\nWHERE id = 2 +SELECT id\nFROM \n(\n SELECT *\n FROM default.test\n HAVING id = 1\n) AS test_view\nWHERE id = 1 +SELECT id\nFROM \n(\n SELECT *\n FROM default.test\n HAVING id = 1\n) AS s\nWHERE id = 1 diff --git a/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.reference b/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.reference new file mode 100644 index 00000000000..461a50ea880 --- /dev/null +++ b/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.reference @@ -0,0 +1,3 @@ +1 1 1 +2 2 0 +1 val11 val21 val31 diff --git a/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.sql b/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.sql new file mode 100644 index 00000000000..c90d01ff76d --- /dev/null +++ b/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.sql @@ -0,0 +1,35 @@ +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +DROP TABLE IF EXISTS view1; + +CREATE TABLE t1 (id UInt32, value1 String) ENGINE MergeTree() ORDER BY id; +CREATE TABLE t2 (id UInt32, value2 String) ENGINE MergeTree() ORDER BY id; +CREATE TABLE t3 (id UInt32, value3 String) ENGINE MergeTree() ORDER BY id; + +INSERT INTO t1 (id, value1) VALUES (1, 'val11'); +INSERT INTO t2 (id, value2) VALUES (1, 'val21'); +INSERT INTO t3 (id, value3) VALUES (1, 'val31'); + +SET multiple_joins_rewriter_version = 2; +SET enable_optimize_predicate_expression = 1; + +SELECT t1.id, t2.id as id, t3.id as value +FROM (select number as id, 42 as value from numbers(4)) t1 +LEFT JOIN (select number as id, 42 as value from numbers(3)) t2 ON t1.id = t2.id +LEFT JOIN (select number as id, 42 as value from numbers(2)) t3 ON t1.id = t3.id +WHERE id > 0 AND value < 42; + +CREATE VIEW IF NOT EXISTS view1 AS + SELECT t1.id AS id, t1.value1 AS value1, t2.value2 AS value2, t3.value3 AS value3 + FROM t1 + LEFT JOIN t2 ON t1.id = t2.id + LEFT JOIN t3 ON t1.id = t3.id + WHERE t1.id > 0; + +SELECT * FROM view1 WHERE id = 1; + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +DROP TABLE IF EXISTS view1; diff --git a/tests/queries/0_stateless/01269_create_with_null.reference b/tests/queries/0_stateless/01269_create_with_null.reference new file mode 100644 index 00000000000..739063af67f --- /dev/null +++ b/tests/queries/0_stateless/01269_create_with_null.reference @@ -0,0 +1,4 @@ +Nullable(Int32) Int32 Nullable(Int32) Int32 +CREATE TABLE default.data_null\n(\n `a` Nullable(Int32), \n `b` Int32, \n `c` Nullable(Int32), \n `d` Int32\n)\nENGINE = Memory() +Nullable(Int32) Int32 Nullable(Int32) Nullable(Int32) +CREATE TABLE default.set_null\n(\n `a` Nullable(Int32), \n `b` Int32, \n `c` Nullable(Int32), \n `d` Nullable(Int32)\n)\nENGINE = Memory() diff --git a/tests/queries/0_stateless/01269_create_with_null.sql b/tests/queries/0_stateless/01269_create_with_null.sql new file mode 100644 index 00000000000..856b6ea75f4 --- /dev/null +++ b/tests/queries/0_stateless/01269_create_with_null.sql @@ -0,0 +1,50 @@ +DROP TABLE IF EXISTS data_null; +DROP TABLE IF EXISTS set_null; + +SET data_type_default_nullable='false'; + +CREATE TABLE data_null ( + a INT NULL, + b INT NOT NULL, + c Nullable(INT), + d INT +) engine=Memory(); + + +INSERT INTO data_null VALUES (NULL, 2, NULL, 4); + +SELECT toTypeName(a), toTypeName(b), toTypeName(c), toTypeName(d) FROM data_null; + +SHOW CREATE TABLE data_null; + +CREATE TABLE data_null_error ( + a Nullable(INT) NULL, + b INT NOT NULL, + c Nullable(INT) +) engine=Memory(); --{serverError 377} + + +CREATE TABLE data_null_error ( + a INT NULL, + b Nullable(INT) NOT NULL, + c Nullable(INT) +) engine=Memory(); --{serverError 377} + +SET data_type_default_nullable='true'; + +CREATE TABLE set_null ( + a INT NULL, + b INT NOT NULL, + c Nullable(INT), + d INT +) engine=Memory(); + + +INSERT INTO set_null VALUES (NULL, 2, NULL, NULL); + +SELECT toTypeName(a), toTypeName(b), toTypeName(c), toTypeName(d) FROM set_null; + +SHOW CREATE TABLE set_null; + +DROP TABLE data_null; +DROP TABLE set_null; diff --git a/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.reference b/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.reference new file mode 100644 index 00000000000..208f3d1abe5 --- /dev/null +++ b/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.reference @@ -0,0 +1,14 @@ +SELECT number\nFROM \n(\n SELECT number\n FROM \n (\n SELECT DISTINCT number\n FROM numbers(3)\n )\n)\nORDER BY number ASC +0 +1 +2 +SELECT DISTINCT number\nFROM \n(\n SELECT DISTINCT number\n FROM \n (\n SELECT DISTINCT number\n FROM numbers(3)\n ORDER BY number ASC\n )\n ORDER BY number ASC\n)\nORDER BY number ASC +0 +1 +2 +SELECT number\nFROM \n(\n SELECT DISTINCT number\n FROM \n (\n SELECT DISTINCT number % 2 AS number\n FROM numbers(3)\n )\n)\nORDER BY number ASC +0 +1 +SELECT DISTINCT number\nFROM \n(\n SELECT DISTINCT number\n FROM \n (\n SELECT DISTINCT number % 2 AS number\n FROM numbers(3)\n ORDER BY number ASC\n )\n ORDER BY number ASC\n)\nORDER BY number ASC +0 +1 diff --git a/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.sql b/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.sql new file mode 100644 index 00000000000..a660e5f0b77 --- /dev/null +++ b/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.sql @@ -0,0 +1,124 @@ +set enable_debug_queries = 1; +set optimize_duplicate_order_by_and_distinct = 1; + +analyze SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT * + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; + +SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT * + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; + +set optimize_duplicate_order_by_and_distinct = 0; + +analyze SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT * + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; + +SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT * + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; + +set optimize_duplicate_order_by_and_distinct = 1; + +analyze SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT number % 2 + AS number + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; + +SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT number % 2 + AS number + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; + +set optimize_duplicate_order_by_and_distinct = 0; + +analyze SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT number % 2 + AS number + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; + +SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT number % 2 + AS number + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; diff --git a/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.reference b/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.reference new file mode 100644 index 00000000000..aa47d0d46d4 --- /dev/null +++ b/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.reference @@ -0,0 +1,2 @@ +0 +0 diff --git a/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql b/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql new file mode 100644 index 00000000000..e1467bacf2f --- /dev/null +++ b/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql @@ -0,0 +1,20 @@ +set optimize_duplicate_order_by_and_distinct = 1; +SELECT DISTINCT number +FROM +( + SELECT DISTINCT number + FROM remote('127.0.0.{1,2}', system.numbers) + LIMIT 1 + SETTINGS distributed_group_by_no_merge = 1 +); + +set optimize_duplicate_order_by_and_distinct = 0; +SELECT DISTINCT number +FROM +( + SELECT DISTINCT number + FROM remote('127.0.0.{1,2}', system.numbers) + LIMIT 1 + SETTINGS distributed_group_by_no_merge = 1 +); + diff --git a/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.reference b/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.reference new file mode 100644 index 00000000000..98fb6a68656 --- /dev/null +++ b/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.reference @@ -0,0 +1,4 @@ +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.sql b/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.sql new file mode 100644 index 00000000000..832f7140af2 --- /dev/null +++ b/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.sql @@ -0,0 +1,8 @@ +DROP TABLE IF EXISTS test_01307; +CREATE TABLE test_01307 (id UInt64, val String, INDEX ind val TYPE bloom_filter() GRANULARITY 1) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 2; +INSERT INTO test_01307 (id, val) select number as id, toString(number) as val from numbers(4); +SELECT count() FROM test_01307 WHERE identity(val) = '2'; +SELECT count() FROM test_01307 WHERE val = '2'; +OPTIMIZE TABLE test_01307 FINAL; +SELECT count() FROM test_01307 WHERE identity(val) = '2'; +SELECT count() FROM test_01307 WHERE val = '2'; diff --git a/tests/queries/0_stateless/01307_multiple_leaders.reference b/tests/queries/0_stateless/01307_multiple_leaders.reference index 576441b288d..62cda31dff8 100644 --- a/tests/queries/0_stateless/01307_multiple_leaders.reference +++ b/tests/queries/0_stateless/01307_multiple_leaders.reference @@ -1,2 +1,2 @@ -2000 1999000 -2000 1999000 +400 79800 +400 79800 diff --git a/tests/queries/0_stateless/01307_multiple_leaders.sh b/tests/queries/0_stateless/01307_multiple_leaders.sh index 0bf5e0b13bf..e19a10bcecb 100755 --- a/tests/queries/0_stateless/01307_multiple_leaders.sh +++ b/tests/queries/0_stateless/01307_multiple_leaders.sh @@ -22,8 +22,8 @@ function thread() } -thread 0 1000 & -thread 1 1000 & +thread 0 200 & +thread 1 200 & wait diff --git a/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.reference b/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.reference new file mode 100644 index 00000000000..ee98bdf033b --- /dev/null +++ b/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.reference @@ -0,0 +1,12 @@ +1 +999999 +100000 +899999 +100001 +900000 +1 +999999 +100000 +899999 +100001 +900000 diff --git a/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql b/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql new file mode 100644 index 00000000000..e37f647e81f --- /dev/null +++ b/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql @@ -0,0 +1,32 @@ +DROP TABLE IF EXISTS test; +CREATE TABLE test (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS index_granularity = 1000; +INSERT INTO test SELECT * FROM numbers(1000000); +OPTIMIZE TABLE test; + +SET max_rows_to_read = 2000; +SELECT count() FROM test WHERE x = 100000; +SET max_rows_to_read = 1000000; +SELECT count() FROM test WHERE x != 100000; +SET max_rows_to_read = 101000; +SELECT count() FROM test WHERE x < 100000; +SET max_rows_to_read = 900000; +SELECT count() FROM test WHERE x > 100000; +SET max_rows_to_read = 101000; +SELECT count() FROM test WHERE x <= 100000; +SET max_rows_to_read = 901000; +SELECT count() FROM test WHERE x >= 100000; + +SET max_rows_to_read = 2000; +SELECT count() FROM test WHERE x = '100000'; +SET max_rows_to_read = 1000000; +SELECT count() FROM test WHERE x != '100000'; +SET max_rows_to_read = 101000; +SELECT count() FROM test WHERE x < '100000'; +SET max_rows_to_read = 900000; +SELECT count() FROM test WHERE x > '100000'; +SET max_rows_to_read = 101000; +SELECT count() FROM test WHERE x <= '100000'; +SET max_rows_to_read = 901000; +SELECT count() FROM test WHERE x >= '100000'; + +DROP TABLE test;