Merge branch 'master' into refactoring

dbms/src/Formats/FormatFactory.cpp

@@ -83,7 +83,17 @@ BlockInputStreamPtr FormatFactory::getInput(
         return std::make_shared<NativeBlockInputStream>(buf, sample, 0);
 
     if (!getCreators(name).input_processor_creator)
-        return getInput(name, buf, sample, context, max_block_size, rows_portion_size, std::move(callback));
+    {
+        const auto & input_getter = getCreators(name).inout_creator;
+        if (!input_getter)
+            throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT);
+
+        const Settings & settings = context.getSettingsRef();
+        FormatSettings format_settings = getInputFormatSetting(settings);
+
+        return input_getter(
+                buf, sample, context, max_block_size, rows_portion_size, callback ? callback : ReadCallback(), format_settings);
+    }
 
     auto format = getInputFormat(name, buf, sample, context, max_block_size, rows_portion_size, std::move(callback));
     return std::make_shared<InputStreamFromInputFormat>(std::move(format));
@@ -106,13 +116,22 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, WriteBuffer &
     }
 
     if (!getCreators(name).output_processor_creator)
-        return getOutput(name, buf, sample, context);
+    {
+        const auto & output_getter = getCreators(name).output_creator;
+        if (!output_getter)
+            throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
+
+        const Settings & settings = context.getSettingsRef();
+        FormatSettings format_settings = getOutputFormatSetting(settings);
+
+        /** Materialization is needed, because formats can use the functions `IDataType`,
+          *  which only work with full columns.
+          */
+        return std::make_shared<MaterializingBlockOutputStream>(
+                output_getter(buf, sample, context, format_settings), sample);
+    }
 
     auto format = getOutputFormat(name, buf, sample, context);
-
-    /** Materialization is needed, because formats can use the functions `IDataType`,
-      *  which only work with full columns.
-      */
     return std::make_shared<MaterializingBlockOutputStream>(std::make_shared<OutputStreamToOutputFormat>(format), sample);
 }
 

dbms/src/Interpreters/ExpressionAnalyzer.cpp

@@ -24,7 +24,6 @@
 
 #include <Interpreters/ExpressionAnalyzer.h>
 #include <Interpreters/ExpressionActions.h>
-#include <Interpreters/ActionsVisitor.h>
 #include <Interpreters/InJoinSubqueriesPreprocessor.h>
 #include <Interpreters/LogicalExpressionsOptimizer.h>
 #include <Interpreters/PredicateExpressionsOptimizer.h>
@@ -55,6 +54,7 @@
 #include <Interpreters/interpretSubquery.h>
 #include <Interpreters/DatabaseAndTableWithAlias.h>
 #include <Interpreters/QueryNormalizer.h>
+#include <Interpreters/ActionsVisitor.h>
 
 #include <Interpreters/ExternalTablesVisitor.h>
 #include <Interpreters/GlobalSubqueriesVisitor.h>

dbms/src/Processors/Executors/PipelineExecutor.h

@@ -177,7 +177,7 @@ private:
     /// Prepare processor with pid number.
     /// Check parents and children of current processor and push them to stacks if they also need to be prepared.
     /// If processor wants to be expanded, ExpandPipelineTask from thread_number's execution context will be used.
-    bool prepareProcessor(size_t pid, Stack & children, Stack & parents, size_t thread_number, bool async);
+    bool prepareProcessor(UInt64 pid, Stack & children, Stack & parents, size_t thread_number, bool async);
     void doExpandPipeline(ExpandPipelineTask * task, bool processing);
 
     void executeImpl(size_t num_threads);

dbms/tests/queries/0_stateless/00990_request_splitting.sql

@@ -0,0 +1 @@
+SELECT * FROM url('http://127.0.0.1:1337/? HTTP/1.1\r\nTest: test', CSV, 'column1 String'); -- { serverError 1000 }

docs/en/interfaces/cli.md

@@ -65,6 +65,31 @@ You can cancel a long query by pressing Ctrl+C. However, you will still need to
 
 The command-line client allows passing external data (external temporary tables) for querying. For more information, see the section "External data for query processing".
 
+### Queries with Parameters {#cli-queries-with-parameters}
+
+You can create a query with parameters, and pass values for these parameters with the parameters of the client app. For example:
+
+```bash
+clickhouse-client --param_parName="[1, 2]"  -q "SELECT * FROM table WHERE a = {parName:Array(UInt16)}"
+```
+
+#### Syntax of a Query {#cli-queries-with-parameters-syntax}
+
+Format a query by the standard method. Values that you want to put into the query from the app parameters place in braces and format as follows:
+
+```
+{<name>:<data type>}
+```
+
+- `name` — Identifier of a placeholder that should be used in app parameter as `--param_name = value`.
+- `data type` — A data type of app parameter value. For example, data structure like `(integer, ('string', integer))` can have a data type `Tuple(UInt8, Tuple(String, UInt8))` (also you can use another [integer](../data_types/int_uint.md) types).
+
+#### Example
+
+```bash
+clickhouse-client --param_tuple_in_tuple="(10, ('dt', 10))" -q "SELECT * FROM table WHERE val = {tuple_in_tuple:Tuple(UInt8, Tuple(String, UInt8))}"
+```
+
 ## Configuring {#interfaces_cli_configuration}
 
 You can pass parameters to `clickhouse-client` (all parameters have a default value) using:

docs/en/interfaces/http.md

@@ -244,5 +244,14 @@ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wa
 
 Use buffering to avoid situations where a query processing error occurred after the response code and HTTP headers were sent to the client. In this situation, an error message is written at the end of the response body, and on the client side, the error can only be detected at the parsing stage.
 
+### Queries with Parameters {#cli-queries-with-parameters}
+
+You can create a query with parameters, and pass values for these parameters with the parameters of the HTTP request. For more information, see [CLI Formatted Queries](cli.md#cli-queries-with-parameters).
+
+### Example
+
+```bash
+curl -sS "<address>?param_id=2&param_phrase=test" -d "SELECT * FROM table WHERE int_column = {id:UInt8} and string_column = {phrase:String}"
+```
 
 [Original article](https://clickhouse.yandex/docs/en/interfaces/http_interface/) <!--hide-->

docs/en/query_language/agg_functions/combinators.md

@@ -44,5 +44,71 @@ Merges the intermediate aggregation states in the same way as the -Merge combina
 
 Converts an aggregate function for tables into an aggregate function for arrays that aggregates the corresponding array items and returns an array of results. For example, `sumForEach` for the arrays `[1, 2]`, `[3, 4, 5]`and`[6, 7]`returns the result `[10, 13, 5]` after adding together the corresponding array items.
 
+## -Resample
+
+Allows to divide data by groups, and then separately aggregates the data in those groups. Groups are created by splitting the values of one of the columns into intervals.
+
+```
+<aggFunction>Resample(start, end, step)(<aggFunction_params>, resampling_key)
+```
+
+**Parameters**
+
+- `start` — Starting value of the whole required interval for the values of `resampling_key`. 
+- `stop` — Ending value of the whole required interval for the values of `resampling_key`. The whole interval doesn't include the `stop` value `[start, stop)`.
+- `step` — Step for separating the whole interval by subintervals. The `aggFunction` is executed over each of those subintervals independently.
+- `resampling_key` — Column, which values are used for separating data by intervals.
+- `aggFunction_params` — Parameters of `aggFunction`.
+
+
+**Returned values**
+
+- Array of `aggFunction` results for each of subintervals.
+
+**Example**
+
+Consider the `people` table with the following data:
+
+```text
+┌─name───┬─age─┬─wage─┐
+│ John   │  16 │   10 │
+│ Alice  │  30 │   15 │
+│ Mary   │  35 │    8 │
+│ Evelyn │  48 │ 11.5 │
+│ David  │  62 │  9.9 │
+│ Brian  │  60 │   16 │
+└────────┴─────┴──────┘
+```
+
+Let's get the names of the persons which age lies in the intervals of `[30,60)` and `[60,75)`. As we use integer representation of age, then there are ages of `[30, 59]` and `[60,74]`.
+
+For aggregating names into the array, we use the aggregate function [groupArray](reference.md#agg_function-grouparray). It takes a single argument. For our case, it is the `name` column. The `groupArrayResample` function should use the `age` column to aggregate names by age. To define required intervals, we pass the `(30, 75, 30)` arguments into the `groupArrayResample` function.
+
+```sql
+SELECT groupArrayResample(30, 75, 30)(name, age) from people
+```
+```text
+┌─groupArrayResample(30, 75, 30)(name, age)─────┐
+│ [['Alice','Mary','Evelyn'],['David','Brian']] │
+└───────────────────────────────────────────────┘
+```
+
+Consider the results.
+
+`Jonh` is out of the sample because he is too young. Other people are distributed according to the specified age intervals.
+
+Now, let's count the total number of people and their average wage in the specified age intervals.
+
+```sql
+SELECT
+    countResample(30, 75, 30)(name, age) AS amount,
+    avgResample(30, 75, 30)(wage, age) AS avg_wage
+FROM people
+```
+```text
+┌─amount─┬─avg_wage──────────────────┐
+│ [3,2]  │ [11.5,12.949999809265137] │
+└────────┴───────────────────────────┘
+```
 
 [Original article](https://clickhouse.yandex/docs/en/query_language/agg_functions/combinators/) <!--hide-->

docs/en/query_language/agg_functions/reference.md

@@ -650,7 +650,7 @@ The function takes a variable number of parameters. Parameters can be `Tuple`, `
 - [uniqHLL12](#agg_function-uniqhll12)
 
 
-## groupArray(x), groupArray(max_size)(x)
+## groupArray(x), groupArray(max_size)(x) {#agg_function-grouparray}
 
 Creates an array of argument values.
 Values can be added to the array in any (indeterminate) order.

docs/toc_zh.yml

@@ -47,26 +47,26 @@ nav:
   - 'AggregateFunction(name, types_of_arguments...)': 'data_types/nested_data_structures/aggregatefunction.md'
   - 'Tuple(T1, T2, ...)': 'data_types/tuple.md'
   - 'Nullable': 'data_types/nullable.md'
-  - 'Nested data structures':
+  - '嵌套数据结构':
     - 'hidden': 'data_types/nested_data_structures/index.md'
     - 'Nested(Name1 Type1, Name2 Type2, ...)': 'data_types/nested_data_structures/nested.md'
-  - 'Special data types':
+  - '特殊数据类型':
     - 'hidden': 'data_types/special_data_types/index.md'
     - 'Expression': 'data_types/special_data_types/expression.md'
     - 'Set': 'data_types/special_data_types/set.md'
     - 'Nothing': 'data_types/special_data_types/nothing.md'
-  - 'Domains':
+  - 'Domain类型':
-    - 'Overview': 'data_types/domains/overview.md'
+    - '介绍': 'data_types/domains/overview.md'
     - 'IPv4': 'data_types/domains/ipv4.md'
     - 'IPv6': 'data_types/domains/ipv6.md'
 
-- 'Database Engines':
+- '数据库引擎':
-    - 'Introduction': 'database_engines/index.md'
+    - '介绍': 'database_engines/index.md'
     - 'MySQL': 'database_engines/mysql.md'
 
-- 'Table Engines':
+- '表引擎':
-  - 'Introduction': 'operations/table_engines/index.md'
+  - '介绍': 'operations/table_engines/index.md'
-  - 'MergeTree Family':
+  - 'MergeTree':
     - 'MergeTree': 'operations/table_engines/mergetree.md'
     - 'Data Replication': 'operations/table_engines/replication.md'
     - 'Custom Partitioning Key': 'operations/table_engines/custom_partitioning_key.md'
@@ -76,17 +76,17 @@ nav:
     - 'CollapsingMergeTree': 'operations/table_engines/collapsingmergetree.md'
     - 'VersionedCollapsingMergeTree': 'operations/table_engines/versionedcollapsingmergetree.md'
     - 'GraphiteMergeTree': 'operations/table_engines/graphitemergetree.md'
-  - 'Log Family':
+  - 'Log':
-    - 'Introduction': 'operations/table_engines/log_family.md'
+    - '介绍': 'operations/table_engines/log_family.md'
     - 'StripeLog': 'operations/table_engines/stripelog.md'
     - 'Log': 'operations/table_engines/log.md'
     - 'TinyLog': 'operations/table_engines/tinylog.md'
-  - 'Integrations':
+  - '外部表引擎':
     - 'Kafka': 'operations/table_engines/kafka.md'
     - 'MySQL': 'operations/table_engines/mysql.md'
     - 'JDBC': 'operations/table_engines/jdbc.md'
     - 'ODBC': 'operations/table_engines/odbc.md'
-  - 'Special':
+  - '其他表引擎':
     - 'Distributed': 'operations/table_engines/distributed.md'
     - 'External data': 'operations/table_engines/external_data.md'
     - 'Dictionary': 'operations/table_engines/dictionary.md'
@@ -108,8 +108,8 @@ nav:
   - 'CREATE': 'query_language/create.md'
   - 'ALTER': 'query_language/alter.md'
   - 'SYSTEM': 'query_language/system.md'
-  - 'Other kinds of queries': 'query_language/misc.md'
+  - '其他类型的查询': 'query_language/misc.md'
-  - 'Functions':
+  - '函数':
     - '介绍': 'query_language/functions/index.md'
     - '算术函数': 'query_language/functions/arithmetic_functions.md'
     - '比较函数': 'query_language/functions/comparison_functions.md'
@@ -142,13 +142,13 @@ nav:
     - 'Nullable处理函数': 'query_language/functions/functions_for_nulls.md'
     - '机器学习函数': 'query_language/functions/machine_learning_functions.md'
     - '其他函数': 'query_language/functions/other_functions.md'
-  - 'Aggregate functions':
+  - '聚合函数':
-    - 'Introduction': 'query_language/agg_functions/index.md'
+    - '介绍': 'query_language/agg_functions/index.md'
-    - 'Function reference': 'query_language/agg_functions/reference.md'
+    - '函数列表': 'query_language/agg_functions/reference.md'
-    - 'Aggregate function combinators': 'query_language/agg_functions/combinators.md'
+    - '聚合函数组合子': 'query_language/agg_functions/combinators.md'
-    - 'Parametric aggregate functions': 'query_language/agg_functions/parametric_functions.md'
+    - '参数化聚合函数': 'query_language/agg_functions/parametric_functions.md'
-  - 'Table functions':
+  - '表引擎函数':
-    - 'Introduction': 'query_language/table_functions/index.md'
+    - '介绍': 'query_language/table_functions/index.md'
     - 'file': 'query_language/table_functions/file.md'
     - 'merge': 'query_language/table_functions/merge.md'
     - 'numbers': 'query_language/table_functions/numbers.md'
@@ -157,44 +157,44 @@ nav:
     - 'mysql': 'query_language/table_functions/mysql.md'
     - 'jdbc': 'query_language/table_functions/jdbc.md'
     - 'odbc': 'query_language/table_functions/odbc.md'
-  - 'Dictionaries':
+  - '字典':
-    - 'Introduction': 'query_language/dicts/index.md'
+    - '介绍': 'query_language/dicts/index.md'
-    - 'External dictionaries':
+    - '外部字典':
-      - 'General description': 'query_language/dicts/external_dicts.md'
+      - '介绍': 'query_language/dicts/external_dicts.md'
-      - 'Configuring an external dictionary': 'query_language/dicts/external_dicts_dict.md'
+      - '配置外部字典': 'query_language/dicts/external_dicts_dict.md'
-      - 'Storing dictionaries in memory': 'query_language/dicts/external_dicts_dict_layout.md'
+      - '字典的内存布局': 'query_language/dicts/external_dicts_dict_layout.md'
-      - 'Dictionary updates': 'query_language/dicts/external_dicts_dict_lifetime.md'
+      - '字典的刷新策略': 'query_language/dicts/external_dicts_dict_lifetime.md'
-      - 'Sources of external dictionaries': 'query_language/dicts/external_dicts_dict_sources.md'
+      - '字典的外部数据源': 'query_language/dicts/external_dicts_dict_sources.md'
-      - 'Dictionary key and fields': 'query_language/dicts/external_dicts_dict_structure.md'
+      - '字典的键和字段值': 'query_language/dicts/external_dicts_dict_structure.md'
-    - 'Internal dictionaries': 'query_language/dicts/internal_dicts.md'
+    - '内部字典': 'query_language/dicts/internal_dicts.md'
-  - 'Operators': 'query_language/operators.md'
+  - '操作符': 'query_language/operators.md'
-  - 'General syntax': 'query_language/syntax.md'
+  - '语法说明': 'query_language/syntax.md'
 
 - '运维':
-  - 'Introduction': 'operations/index.md'
+  - '介绍': 'operations/index.md'
-  - 'Requirements': 'operations/requirements.md'
+  - '环境要求': 'operations/requirements.md'
-  - 'Monitoring': 'operations/monitoring.md'
+  - '监控': 'operations/monitoring.md'
-  - 'Troubleshooting': 'operations/troubleshooting.md'
+  - '故障排查': 'operations/troubleshooting.md'
-  - 'Usage recommendations': 'operations/tips.md'
+  - '使用建议': 'operations/tips.md'
-  - 'ClickHouse Update': 'operations/update.md'
+  - '版本升级': 'operations/update.md'
-  - 'Access rights': 'operations/access_rights.md'
+  - '访问权限控制': 'operations/access_rights.md'
-  - 'Data backup': 'operations/backup.md'
+  - '数据备份': 'operations/backup.md'
-  - 'Configuration files': 'operations/configuration_files.md'
+  - '配置文件': 'operations/configuration_files.md'
-  - 'Quotas': 'operations/quotas.md'
+  - '配额': 'operations/quotas.md'
-  - 'System tables': 'operations/system_tables.md'
+  - '系统表': 'operations/system_tables.md'
-  - 'Server configuration parameters':
+  - 'Server参数配置':
-    - 'Introduction': 'operations/server_settings/index.md'
+    - '介绍': 'operations/server_settings/index.md'
-    - 'Server settings': 'operations/server_settings/settings.md'
+    - 'Server参数说明': 'operations/server_settings/settings.md'
-  - 'Settings':
+  - 'Settings配置':
-    - 'Introduction': 'operations/settings/index.md'
+    - '介绍': 'operations/settings/index.md'
-    - 'Permissions for queries': 'operations/settings/permissions_for_queries.md'
+    - '查询权限管理': 'operations/settings/permissions_for_queries.md'
-    - 'Restrictions on query complexity': 'operations/settings/query_complexity.md'
+    - '查询复杂性的限制': 'operations/settings/query_complexity.md'
-    - 'Settings': 'operations/settings/settings.md'
+    - 'Setting列表': 'operations/settings/settings.md'
-    - 'Settings profiles': 'operations/settings/settings_profiles.md'
+    - 'Setting配置组': 'operations/settings/settings_profiles.md'
-    - 'User Settings': 'operations/settings/settings_users.md'
+    - '用户配置': 'operations/settings/settings_users.md'
-    - 'Constraints on Settings': 'operations/settings/constraints_on_settings.md'
+    - 'Settings的约束': 'operations/settings/constraints_on_settings.md'
-  - 'Utilities':
+  - '常用工具':
-    - 'Overview': 'operations/utils/index.md'
+    - '介绍': 'operations/utils/index.md'
     - 'clickhouse-copier': 'operations/utils/clickhouse-copier.md'
     - 'clickhouse-local': 'operations/utils/clickhouse-local.md'