Merge branch 'master' into refactoring

dbms/src/Formats/FormatFactory.cpp

@@ -83,7 +83,17 @@ BlockInputStreamPtr FormatFactory::getInput(
         return std::make_shared<NativeBlockInputStream>(buf, sample, 0);
 
     if (!getCreators(name).input_processor_creator)
-        return getInput(name, buf, sample, context, max_block_size, rows_portion_size, std::move(callback));
+    {
+        const auto & input_getter = getCreators(name).inout_creator;
+        if (!input_getter)
+            throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT);
+
+        const Settings & settings = context.getSettingsRef();
+        FormatSettings format_settings = getInputFormatSetting(settings);
+
+        return input_getter(
+                buf, sample, context, max_block_size, rows_portion_size, callback ? callback : ReadCallback(), format_settings);
+    }
 
     auto format = getInputFormat(name, buf, sample, context, max_block_size, rows_portion_size, std::move(callback));
     return std::make_shared<InputStreamFromInputFormat>(std::move(format));
@@ -106,13 +116,22 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, WriteBuffer &
     }
 
     if (!getCreators(name).output_processor_creator)
-        return getOutput(name, buf, sample, context);
+    {
+        const auto & output_getter = getCreators(name).output_creator;
+        if (!output_getter)
+            throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
+
+        const Settings & settings = context.getSettingsRef();
+        FormatSettings format_settings = getOutputFormatSetting(settings);
+
+        /** Materialization is needed, because formats can use the functions `IDataType`,
+          *  which only work with full columns.
+          */
+        return std::make_shared<MaterializingBlockOutputStream>(
+                output_getter(buf, sample, context, format_settings), sample);
+    }
 
     auto format = getOutputFormat(name, buf, sample, context);
-
-    /** Materialization is needed, because formats can use the functions `IDataType`,
-      *  which only work with full columns.
-      */
     return std::make_shared<MaterializingBlockOutputStream>(std::make_shared<OutputStreamToOutputFormat>(format), sample);
 }
 

dbms/src/Interpreters/ExpressionAnalyzer.cpp

@@ -24,7 +24,6 @@
 
 #include <Interpreters/ExpressionAnalyzer.h>
 #include <Interpreters/ExpressionActions.h>
-#include <Interpreters/ActionsVisitor.h>
 #include <Interpreters/InJoinSubqueriesPreprocessor.h>
 #include <Interpreters/LogicalExpressionsOptimizer.h>
 #include <Interpreters/PredicateExpressionsOptimizer.h>
@@ -55,6 +54,7 @@
 #include <Interpreters/interpretSubquery.h>
 #include <Interpreters/DatabaseAndTableWithAlias.h>
 #include <Interpreters/QueryNormalizer.h>
+#include <Interpreters/ActionsVisitor.h>
 
 #include <Interpreters/ExternalTablesVisitor.h>
 #include <Interpreters/GlobalSubqueriesVisitor.h>

dbms/src/Processors/Executors/PipelineExecutor.h

@@ -177,7 +177,7 @@ private:
     /// Prepare processor with pid number.
     /// Check parents and children of current processor and push them to stacks if they also need to be prepared.
     /// If processor wants to be expanded, ExpandPipelineTask from thread_number's execution context will be used.
-    bool prepareProcessor(size_t pid, Stack & children, Stack & parents, size_t thread_number, bool async);
+    bool prepareProcessor(UInt64 pid, Stack & children, Stack & parents, size_t thread_number, bool async);
     void doExpandPipeline(ExpandPipelineTask * task, bool processing);
 
     void executeImpl(size_t num_threads);

dbms/tests/queries/0_stateless/00990_request_splitting.sql

@@ -0,0 +1 @@
+SELECT * FROM url('http://127.0.0.1:1337/? HTTP/1.1\r\nTest: test', CSV, 'column1 String'); -- { serverError 1000 }

docs/en/interfaces/cli.md

@@ -65,6 +65,31 @@ You can cancel a long query by pressing Ctrl+C. However, you will still need to
 
 The command-line client allows passing external data (external temporary tables) for querying. For more information, see the section "External data for query processing".
 
+### Queries with Parameters {#cli-queries-with-parameters}
+
+You can create a query with parameters, and pass values for these parameters with the parameters of the client app. For example:
+
+```bash
+clickhouse-client --param_parName="[1, 2]"  -q "SELECT * FROM table WHERE a = {parName:Array(UInt16)}"
+```
+
+#### Syntax of a Query {#cli-queries-with-parameters-syntax}
+
+Format a query by the standard method. Values that you want to put into the query from the app parameters place in braces and format as follows:
+
+```
+{<name>:<data type>}
+```
+
+- `name` — Identifier of a placeholder that should be used in app parameter as `--param_name = value`.
+- `data type` — A data type of app parameter value. For example, data structure like `(integer, ('string', integer))` can have a data type `Tuple(UInt8, Tuple(String, UInt8))` (also you can use another [integer](../data_types/int_uint.md) types).
+
+#### Example
+
+```bash
+clickhouse-client --param_tuple_in_tuple="(10, ('dt', 10))" -q "SELECT * FROM table WHERE val = {tuple_in_tuple:Tuple(UInt8, Tuple(String, UInt8))}"
+```
+
 ## Configuring {#interfaces_cli_configuration}
 
 You can pass parameters to `clickhouse-client` (all parameters have a default value) using:

docs/en/interfaces/http.md

@@ -244,5 +244,14 @@ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wa
 
 Use buffering to avoid situations where a query processing error occurred after the response code and HTTP headers were sent to the client. In this situation, an error message is written at the end of the response body, and on the client side, the error can only be detected at the parsing stage.
 
+### Queries with Parameters {#cli-queries-with-parameters}
+
+You can create a query with parameters, and pass values for these parameters with the parameters of the HTTP request. For more information, see [CLI Formatted Queries](cli.md#cli-queries-with-parameters).
+
+### Example
+
+```bash
+curl -sS "<address>?param_id=2&param_phrase=test" -d "SELECT * FROM table WHERE int_column = {id:UInt8} and string_column = {phrase:String}"
+```
 
 [Original article](https://clickhouse.yandex/docs/en/interfaces/http_interface/) <!--hide-->

docs/en/query_language/agg_functions/combinators.md

@@ -44,5 +44,71 @@ Merges the intermediate aggregation states in the same way as the -Merge combina
 
 Converts an aggregate function for tables into an aggregate function for arrays that aggregates the corresponding array items and returns an array of results. For example, `sumForEach` for the arrays `[1, 2]`, `[3, 4, 5]`and`[6, 7]`returns the result `[10, 13, 5]` after adding together the corresponding array items.
 
+## -Resample
+
+Allows to divide data by groups, and then separately aggregates the data in those groups. Groups are created by splitting the values of one of the columns into intervals.
+
+```
+<aggFunction>Resample(start, end, step)(<aggFunction_params>, resampling_key)
+```
+
+**Parameters**
+
+- `start` — Starting value of the whole required interval for the values of `resampling_key`. 
+- `stop` — Ending value of the whole required interval for the values of `resampling_key`. The whole interval doesn't include the `stop` value `[start, stop)`.
+- `step` — Step for separating the whole interval by subintervals. The `aggFunction` is executed over each of those subintervals independently.
+- `resampling_key` — Column, which values are used for separating data by intervals.
+- `aggFunction_params` — Parameters of `aggFunction`.
+
+
+**Returned values**
+
+- Array of `aggFunction` results for each of subintervals.
+
+**Example**
+
+Consider the `people` table with the following data:
+
+```text
+┌─name───┬─age─┬─wage─┐
+│ John   │  16 │   10 │
+│ Alice  │  30 │   15 │
+│ Mary   │  35 │    8 │
+│ Evelyn │  48 │ 11.5 │
+│ David  │  62 │  9.9 │
+│ Brian  │  60 │   16 │
+└────────┴─────┴──────┘
+```
+
+Let's get the names of the persons which age lies in the intervals of `[30,60)` and `[60,75)`. As we use integer representation of age, then there are ages of `[30, 59]` and `[60,74]`.
+
+For aggregating names into the array, we use the aggregate function [groupArray](reference.md#agg_function-grouparray). It takes a single argument. For our case, it is the `name` column. The `groupArrayResample` function should use the `age` column to aggregate names by age. To define required intervals, we pass the `(30, 75, 30)` arguments into the `groupArrayResample` function.
+
+```sql
+SELECT groupArrayResample(30, 75, 30)(name, age) from people
+```
+```text
+┌─groupArrayResample(30, 75, 30)(name, age)─────┐
+│ [['Alice','Mary','Evelyn'],['David','Brian']] │
+└───────────────────────────────────────────────┘
+```
+
+Consider the results.
+
+`Jonh` is out of the sample because he is too young. Other people are distributed according to the specified age intervals.
+
+Now, let's count the total number of people and their average wage in the specified age intervals.
+
+```sql
+SELECT
+    countResample(30, 75, 30)(name, age) AS amount,
+    avgResample(30, 75, 30)(wage, age) AS avg_wage
+FROM people
+```
+```text
+┌─amount─┬─avg_wage──────────────────┐
+│ [3,2]  │ [11.5,12.949999809265137] │
+└────────┴───────────────────────────┘
+```
 
 [Original article](https://clickhouse.yandex/docs/en/query_language/agg_functions/combinators/) <!--hide-->

docs/en/query_language/agg_functions/reference.md

@@ -650,7 +650,7 @@ The function takes a variable number of parameters. Parameters can be `Tuple`, `
 - [uniqHLL12](#agg_function-uniqhll12)
 
 
-## groupArray(x), groupArray(max_size)(x)
+## groupArray(x), groupArray(max_size)(x) {#agg_function-grouparray}
 
 Creates an array of argument values.
 Values can be added to the array in any (indeterminate) order.

docs/toc_zh.yml

@@ -47,26 +47,26 @@ nav:
   - 'AggregateFunction(name, types_of_arguments...)': 'data_types/nested_data_structures/aggregatefunction.md'
   - 'Tuple(T1, T2, ...)': 'data_types/tuple.md'
   - 'Nullable': 'data_types/nullable.md'
-  - 'Nested data structures':
+  - '嵌套数据结构':
     - 'hidden': 'data_types/nested_data_structures/index.md'
     - 'Nested(Name1 Type1, Name2 Type2, ...)': 'data_types/nested_data_structures/nested.md'
-  - 'Special data types':
+  - '特殊数据类型':
     - 'hidden': 'data_types/special_data_types/index.md'
     - 'Expression': 'data_types/special_data_types/expression.md'
     - 'Set': 'data_types/special_data_types/set.md'
     - 'Nothing': 'data_types/special_data_types/nothing.md'
-  - 'Domains':
-    - 'Overview': 'data_types/domains/overview.md'
+  - 'Domain类型':
+    - '介绍': 'data_types/domains/overview.md'
     - 'IPv4': 'data_types/domains/ipv4.md'
     - 'IPv6': 'data_types/domains/ipv6.md'
 
-- 'Database Engines':
-    - 'Introduction': 'database_engines/index.md'
+- '数据库引擎':
+    - '介绍': 'database_engines/index.md'
     - 'MySQL': 'database_engines/mysql.md'
 
-- 'Table Engines':
-  - 'Introduction': 'operations/table_engines/index.md'
-  - 'MergeTree Family':
+- '表引擎':
+  - '介绍': 'operations/table_engines/index.md'
+  - 'MergeTree':
     - 'MergeTree': 'operations/table_engines/mergetree.md'
     - 'Data Replication': 'operations/table_engines/replication.md'
     - 'Custom Partitioning Key': 'operations/table_engines/custom_partitioning_key.md'
@@ -76,17 +76,17 @@ nav:
     - 'CollapsingMergeTree': 'operations/table_engines/collapsingmergetree.md'
     - 'VersionedCollapsingMergeTree': 'operations/table_engines/versionedcollapsingmergetree.md'
     - 'GraphiteMergeTree': 'operations/table_engines/graphitemergetree.md'
-  - 'Log Family':
-    - 'Introduction': 'operations/table_engines/log_family.md'
+  - 'Log':
+    - '介绍': 'operations/table_engines/log_family.md'
     - 'StripeLog': 'operations/table_engines/stripelog.md'
     - 'Log': 'operations/table_engines/log.md'
     - 'TinyLog': 'operations/table_engines/tinylog.md'
-  - 'Integrations':
+  - '外部表引擎':
     - 'Kafka': 'operations/table_engines/kafka.md'
     - 'MySQL': 'operations/table_engines/mysql.md'
     - 'JDBC': 'operations/table_engines/jdbc.md'
     - 'ODBC': 'operations/table_engines/odbc.md'
-  - 'Special':
+  - '其他表引擎':
     - 'Distributed': 'operations/table_engines/distributed.md'
     - 'External data': 'operations/table_engines/external_data.md'
     - 'Dictionary': 'operations/table_engines/dictionary.md'
@@ -108,8 +108,8 @@ nav:
   - 'CREATE': 'query_language/create.md'
   - 'ALTER': 'query_language/alter.md'
   - 'SYSTEM': 'query_language/system.md'
-  - 'Other kinds of queries': 'query_language/misc.md'
-  - 'Functions':
+  - '其他类型的查询': 'query_language/misc.md'
+  - '函数':
     - '介绍': 'query_language/functions/index.md'
     - '算术函数': 'query_language/functions/arithmetic_functions.md'
     - '比较函数': 'query_language/functions/comparison_functions.md'
@@ -142,13 +142,13 @@ nav:
     - 'Nullable处理函数': 'query_language/functions/functions_for_nulls.md'
     - '机器学习函数': 'query_language/functions/machine_learning_functions.md'
     - '其他函数': 'query_language/functions/other_functions.md'
-  - 'Aggregate functions':
-    - 'Introduction': 'query_language/agg_functions/index.md'
-    - 'Function reference': 'query_language/agg_functions/reference.md'
-    - 'Aggregate function combinators': 'query_language/agg_functions/combinators.md'
-    - 'Parametric aggregate functions': 'query_language/agg_functions/parametric_functions.md'
-  - 'Table functions':
-    - 'Introduction': 'query_language/table_functions/index.md'
+  - '聚合函数':
+    - '介绍': 'query_language/agg_functions/index.md'
+    - '函数列表': 'query_language/agg_functions/reference.md'
+    - '聚合函数组合子': 'query_language/agg_functions/combinators.md'
+    - '参数化聚合函数': 'query_language/agg_functions/parametric_functions.md'
+  - '表引擎函数':
+    - '介绍': 'query_language/table_functions/index.md'
     - 'file': 'query_language/table_functions/file.md'
     - 'merge': 'query_language/table_functions/merge.md'
     - 'numbers': 'query_language/table_functions/numbers.md'
@@ -157,44 +157,44 @@ nav:
     - 'mysql': 'query_language/table_functions/mysql.md'
     - 'jdbc': 'query_language/table_functions/jdbc.md'
     - 'odbc': 'query_language/table_functions/odbc.md'
-  - 'Dictionaries':
-    - 'Introduction': 'query_language/dicts/index.md'
-    - 'External dictionaries':
-      - 'General description': 'query_language/dicts/external_dicts.md'
-      - 'Configuring an external dictionary': 'query_language/dicts/external_dicts_dict.md'
-      - 'Storing dictionaries in memory': 'query_language/dicts/external_dicts_dict_layout.md'
-      - 'Dictionary updates': 'query_language/dicts/external_dicts_dict_lifetime.md'
-      - 'Sources of external dictionaries': 'query_language/dicts/external_dicts_dict_sources.md'
-      - 'Dictionary key and fields': 'query_language/dicts/external_dicts_dict_structure.md'
-    - 'Internal dictionaries': 'query_language/dicts/internal_dicts.md'
-  - 'Operators': 'query_language/operators.md'
-  - 'General syntax': 'query_language/syntax.md'
+  - '字典':
+    - '介绍': 'query_language/dicts/index.md'
+    - '外部字典':
+      - '介绍': 'query_language/dicts/external_dicts.md'
+      - '配置外部字典': 'query_language/dicts/external_dicts_dict.md'
+      - '字典的内存布局': 'query_language/dicts/external_dicts_dict_layout.md'
+      - '字典的刷新策略': 'query_language/dicts/external_dicts_dict_lifetime.md'
+      - '字典的外部数据源': 'query_language/dicts/external_dicts_dict_sources.md'
+      - '字典的键和字段值': 'query_language/dicts/external_dicts_dict_structure.md'
+    - '内部字典': 'query_language/dicts/internal_dicts.md'
+  - '操作符': 'query_language/operators.md'
+  - '语法说明': 'query_language/syntax.md'
 
 - '运维':
-  - 'Introduction': 'operations/index.md'
-  - 'Requirements': 'operations/requirements.md'
-  - 'Monitoring': 'operations/monitoring.md'
-  - 'Troubleshooting': 'operations/troubleshooting.md'
-  - 'Usage recommendations': 'operations/tips.md'
-  - 'ClickHouse Update': 'operations/update.md'
-  - 'Access rights': 'operations/access_rights.md'
-  - 'Data backup': 'operations/backup.md'
-  - 'Configuration files': 'operations/configuration_files.md'
-  - 'Quotas': 'operations/quotas.md'
-  - 'System tables': 'operations/system_tables.md'
-  - 'Server configuration parameters':
-    - 'Introduction': 'operations/server_settings/index.md'
-    - 'Server settings': 'operations/server_settings/settings.md'
-  - 'Settings':
-    - 'Introduction': 'operations/settings/index.md'
-    - 'Permissions for queries': 'operations/settings/permissions_for_queries.md'
-    - 'Restrictions on query complexity': 'operations/settings/query_complexity.md'
-    - 'Settings': 'operations/settings/settings.md'
-    - 'Settings profiles': 'operations/settings/settings_profiles.md'
-    - 'User Settings': 'operations/settings/settings_users.md'
-    - 'Constraints on Settings': 'operations/settings/constraints_on_settings.md'
-  - 'Utilities':
-    - 'Overview': 'operations/utils/index.md'
+  - '介绍': 'operations/index.md'
+  - '环境要求': 'operations/requirements.md'
+  - '监控': 'operations/monitoring.md'
+  - '故障排查': 'operations/troubleshooting.md'
+  - '使用建议': 'operations/tips.md'
+  - '版本升级': 'operations/update.md'
+  - '访问权限控制': 'operations/access_rights.md'
+  - '数据备份': 'operations/backup.md'
+  - '配置文件': 'operations/configuration_files.md'
+  - '配额': 'operations/quotas.md'
+  - '系统表': 'operations/system_tables.md'
+  - 'Server参数配置':
+    - '介绍': 'operations/server_settings/index.md'
+    - 'Server参数说明': 'operations/server_settings/settings.md'
+  - 'Settings配置':
+    - '介绍': 'operations/settings/index.md'
+    - '查询权限管理': 'operations/settings/permissions_for_queries.md'
+    - '查询复杂性的限制': 'operations/settings/query_complexity.md'
+    - 'Setting列表': 'operations/settings/settings.md'
+    - 'Setting配置组': 'operations/settings/settings_profiles.md'
+    - '用户配置': 'operations/settings/settings_users.md'
+    - 'Settings的约束': 'operations/settings/constraints_on_settings.md'
+  - '常用工具':
+    - '介绍': 'operations/utils/index.md'
     - 'clickhouse-copier': 'operations/utils/clickhouse-copier.md'
     - 'clickhouse-local': 'operations/utils/clickhouse-local.md'