mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-19 16:20:50 +00:00
Merge branch 'master' of github.com:ClickHouse/ClickHouse into divanik/add_local_object_storage
This commit is contained in:
divanik
commit
e2b13411fd
4
.github/workflows/backport_branches.yml
vendored
4
.github/workflows/backport_branches.yml
vendored
@ -36,10 +36,6 @@ jobs:
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
echo "Testing the main ci directory"
|
||||
python3 -m unittest discover -s . -p 'test_*.py'
|
||||
for dir in *_lambda/; do
|
||||
echo "Testing $dir"
|
||||
python3 -m unittest discover -s "$dir" -p 'test_*.py'
|
||||
done
|
||||
- name: PrepareRunConfig
|
||||
id: runconfig
|
||||
run: |
|
||||
|
||||
4
.github/workflows/master.yml
vendored
4
.github/workflows/master.yml
vendored
@ -33,10 +33,6 @@ jobs:
|
||||
# cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
# echo "Testing the main ci directory"
|
||||
# python3 -m unittest discover -s . -p 'test_*.py'
|
||||
# for dir in *_lambda/; do
|
||||
# echo "Testing $dir"
|
||||
# python3 -m unittest discover -s "$dir" -p 'test_*.py'
|
||||
# done
|
||||
- name: PrepareRunConfig
|
||||
id: runconfig
|
||||
run: |
|
||||
|
||||
4
.github/workflows/merge_queue.yml
vendored
4
.github/workflows/merge_queue.yml
vendored
@ -30,10 +30,6 @@ jobs:
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
echo "Testing the main ci directory"
|
||||
python3 -m unittest discover -s . -p 'test_*.py'
|
||||
for dir in *_lambda/; do
|
||||
echo "Testing $dir"
|
||||
python3 -m unittest discover -s "$dir" -p 'test_*.py'
|
||||
done
|
||||
- name: PrepareRunConfig
|
||||
id: runconfig
|
||||
run: |
|
||||
|
||||
4
.github/workflows/pull_request.yml
vendored
4
.github/workflows/pull_request.yml
vendored
@ -48,10 +48,6 @@ jobs:
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
echo "Testing the main ci directory"
|
||||
python3 -m unittest discover -s . -p 'test_*.py'
|
||||
for dir in *_lambda/; do
|
||||
echo "Testing $dir"
|
||||
python3 -m unittest discover -s "$dir" -p 'test_*.py'
|
||||
done
|
||||
- name: PrepareRunConfig
|
||||
id: runconfig
|
||||
run: |
|
||||
|
||||
4
.github/workflows/release_branches.yml
vendored
4
.github/workflows/release_branches.yml
vendored
@ -33,10 +33,6 @@ jobs:
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
echo "Testing the main ci directory"
|
||||
python3 -m unittest discover -s . -p 'test_*.py'
|
||||
for dir in *_lambda/; do
|
||||
echo "Testing $dir"
|
||||
python3 -m unittest discover -s "$dir" -p 'test_*.py'
|
||||
done
|
||||
- name: PrepareRunConfig
|
||||
id: runconfig
|
||||
run: |
|
||||
|
||||
@ -26,7 +26,10 @@ RUN apt-get update \
|
||||
zstd \
|
||||
--yes --no-install-recommends \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \
|
||||
&& groupadd --system --gid 1000 clickhouse \
|
||||
&& useradd --system --gid 1000 --uid 1000 -m clickhouse
|
||||
# ^ For some reason, groupadd and useradd are needed for tests with 'expect', but I don't know, why.
|
||||
|
||||
COPY requirements.txt /
|
||||
RUN pip3 install --no-cache-dir -r /requirements.txt
|
||||
|
||||
@ -9,7 +9,7 @@ trap 'kill $(jobs -pr) ||:' EXIT
|
||||
stage=${stage:-}
|
||||
|
||||
# Compiler version, normally set by Dockerfile
|
||||
export LLVM_VERSION=${LLVM_VERSION:-17}
|
||||
export LLVM_VERSION=${LLVM_VERSION:-18}
|
||||
|
||||
# A variable to pass additional flags to CMake.
|
||||
# Here we explicitly default it to nothing so that bash doesn't complain about
|
||||
|
||||
@ -17,6 +17,7 @@ ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=18
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install \
|
||||
sudo \
|
||||
apt-transport-https \
|
||||
apt-utils \
|
||||
ca-certificates \
|
||||
|
||||
@ -185,6 +185,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
|
||||
- `--format, -f` – Use the specified default format to output the result.
|
||||
- `--vertical, -E` – If specified, use the [Vertical format](../interfaces/formats.md#vertical) by default to output the result. This is the same as `–format=Vertical`. In this format, each value is printed on a separate line, which is helpful when displaying wide tables.
|
||||
- `--time, -t` – If specified, print the query execution time to ‘stderr’ in non-interactive mode.
|
||||
- `--memory-usage` – If specified, print memory usage to ‘stderr’ in non-interactive mode]. Possible values: 'none' - do not print memory usage, 'default' - print number of bytes, 'readable' - print memory usage in human-readable format.
|
||||
- `--stacktrace` – If specified, also print the stack trace if an exception occurs.
|
||||
- `--config-file` – The name of the configuration file.
|
||||
- `--secure` – If specified, will connect to server over secure connection (TLS). You might need to configure your CA certificates in the [configuration file](#configuration_files). The available configuration settings are the same as for [server-side TLS configuration](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-openssl).
|
||||
@ -339,7 +340,7 @@ clickhouse-client clickhouse://some_user%40some_mail.com@localhost:9000
|
||||
Connect to one of provides hosts: `192.168.1.15`, `192.168.1.25`.
|
||||
|
||||
``` bash
|
||||
clickhouse-client clickhouse://192.168.1.15,192.168.1.25
|
||||
clickhouse-client clickhouse://192.168.1.15,192.168.1.25
|
||||
```
|
||||
|
||||
### Configuration Files {#configuration_files}
|
||||
@ -367,7 +368,7 @@ Example of a config file:
|
||||
```
|
||||
|
||||
Or the same config in a YAML format:
|
||||
|
||||
|
||||
```yaml
|
||||
user: username
|
||||
password: 'password'
|
||||
|
||||
@ -67,6 +67,7 @@ The supported formats are:
|
||||
| [Prometheus](#prometheus) | ✗ | ✔ |
|
||||
| [Protobuf](#protobuf) | ✔ | ✔ |
|
||||
| [ProtobufSingle](#protobufsingle) | ✔ | ✔ |
|
||||
| [ProtobufList](#protobuflist) | ✔ | ✔ |
|
||||
| [Avro](#data-format-avro) | ✔ | ✔ |
|
||||
| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ |
|
||||
| [Parquet](#data-format-parquet) | ✔ | ✔ |
|
||||
@ -1952,6 +1953,35 @@ SYSTEM DROP FORMAT SCHEMA CACHE FOR Protobuf
|
||||
|
||||
Same as [Protobuf](#protobuf) but for storing/parsing single Protobuf message without length delimiters.
|
||||
|
||||
## ProtobufList {#protobuflist}
|
||||
|
||||
Similar to Protobuf but rows are represented as a sequence of sub-messages contained in a message with fixed name "Envelope".
|
||||
|
||||
Usage example:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM test.table FORMAT ProtobufList SETTINGS format_schema = 'schemafile:MessageType'
|
||||
```
|
||||
|
||||
``` bash
|
||||
cat protobuflist_messages.bin | clickhouse-client --query "INSERT INTO test.table FORMAT ProtobufList SETTINGS format_schema='schemafile:MessageType'"
|
||||
```
|
||||
|
||||
where the file `schemafile.proto` looks like this:
|
||||
|
||||
``` capnp
|
||||
syntax = "proto3";
|
||||
message Envelope {
|
||||
message MessageType {
|
||||
string name = 1;
|
||||
string surname = 2;
|
||||
uint32 birthDate = 3;
|
||||
repeated string phoneNumbers = 4;
|
||||
};
|
||||
MessageType row = 1;
|
||||
};
|
||||
```
|
||||
|
||||
## Avro {#data-format-avro}
|
||||
|
||||
[Apache Avro](https://avro.apache.org/) is a row-oriented data serialization framework developed within Apache’s Hadoop project.
|
||||
|
||||
35
docs/en/operations/system-tables/detached_tables.md
Normal file
35
docs/en/operations/system-tables/detached_tables.md
Normal file
@ -0,0 +1,35 @@
|
||||
---
|
||||
slug: /en/operations/system-tables/detached_tables
|
||||
---
|
||||
# detached_tables
|
||||
|
||||
Contains information of each detached table.
|
||||
|
||||
Columns:
|
||||
|
||||
- `database` ([String](../../sql-reference/data-types/string.md)) — The name of the database the table is in.
|
||||
|
||||
- `table` ([String](../../sql-reference/data-types/string.md)) — Table name.
|
||||
|
||||
- `uuid` ([UUID](../../sql-reference/data-types/uuid.md)) — Table uuid (Atomic database).
|
||||
|
||||
- `metadata_path` ([String](../../sql-reference/data-types/string.md)) - Path to the table metadata in the file system.
|
||||
|
||||
- `is_permanently` ([UInt8](../../sql-reference/data-types/int-uint.md)) - Flag indicates that the table was detached PERMANENTLY.
|
||||
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT * FROM system.detached_tables FORMAT Vertical;
|
||||
```
|
||||
|
||||
```text
|
||||
Row 1:
|
||||
──────
|
||||
database: base
|
||||
table: t1
|
||||
uuid: 81b1c20a-b7c6-4116-a2ce-7583fb6b6736
|
||||
metadata_path: /var/lib/clickhouse/store/461/461cf698-fd0b-406d-8c01-5d8fd5748a91/t1.sql
|
||||
is_permanently: 1
|
||||
```
|
||||
@ -9,7 +9,6 @@ Columns:
|
||||
|
||||
- `name` ([String](../../sql-reference/data-types/string.md)) – The name of the function.
|
||||
- `is_aggregate` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Whether the function is an aggregate function.
|
||||
- `is_deterministic` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md))) - Whether the function is deterministic.
|
||||
- `case_insensitive`, ([UInt8](../../sql-reference/data-types/int-uint.md)) - Whether the function name can be used case-insensitively.
|
||||
- `alias_to`, ([String](../../sql-reference/data-types/string.md)) - The original function name, if the function name is an alias.
|
||||
- `create_query`, ([String](../../sql-reference/data-types/enum.md)) - Unused.
|
||||
|
||||
@ -56,7 +56,6 @@ Functions:
|
||||
|
||||
## Related content
|
||||
|
||||
- [Reducing ClickHouse Storage Cost with the Low Cardinality Type – Lessons from an Instana Engineer](https://altinity.com/blog/2020-5-20-reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer)
|
||||
- [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/ClickHouse/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf)
|
||||
- Blog: [Optimizing ClickHouse with Schemas and Codecs](https://clickhouse.com/blog/optimize-clickhouse-codecs-compression-schema)
|
||||
- Blog: [Working with time series data in ClickHouse](https://clickhouse.com/blog/working-with-time-series-data-and-functions-ClickHouse)
|
||||
- [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/ClickHouse/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf)
|
||||
|
||||
@ -76,7 +76,7 @@ WHERE macro = 'test';
|
||||
└───────┴──────────────┘
|
||||
```
|
||||
|
||||
## FQDN
|
||||
## fqdn
|
||||
|
||||
Returns the fully qualified domain name of the ClickHouse server.
|
||||
|
||||
@ -86,7 +86,7 @@ Returns the fully qualified domain name of the ClickHouse server.
|
||||
fqdn();
|
||||
```
|
||||
|
||||
Aliases: `fullHostName`, 'FQDN'.
|
||||
Aliases: `fullHostName`, `FQDN`.
|
||||
|
||||
**Returned value**
|
||||
|
||||
|
||||
@ -567,12 +567,13 @@ While no standard or recommendation exists for the epoch of Snowflake IDs, imple
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
generateSnowflakeID([expr])
|
||||
generateSnowflakeID([expr, [machine_id]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `expr` — An arbitrary [expression](../../sql-reference/syntax.md#syntax-expressions) used to bypass [common subexpression elimination](../../sql-reference/functions/index.md#common-subexpression-elimination) if the function is called multiple times in a query. The value of the expression has no effect on the returned Snowflake ID. Optional.
|
||||
- `machine_id` — A machine ID, the lowest 10 bits are used. [Int64](../data-types/int-uint.md). Optional.
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -608,6 +609,16 @@ SELECT generateSnowflakeID(1), generateSnowflakeID(2);
|
||||
└────────────────────────┴────────────────────────┘
|
||||
```
|
||||
|
||||
**Example with expression and a machine ID**
|
||||
|
||||
```
|
||||
SELECT generateSnowflakeID('expr', 1);
|
||||
|
||||
┌─generateSnowflakeID('expr', 1)─┐
|
||||
│ 7201148511606784002 │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
|
||||
## snowflakeToDateTime
|
||||
|
||||
:::warning
|
||||
|
||||
@ -356,7 +356,7 @@ sidebar_label: "\u53D8\u66F4\u65E5\u5FD7"
|
||||
|
||||
#### 新功能 {#new-feature-1}
|
||||
|
||||
- 添加 `deduplicate_blocks_in_dependent_materialized_views` 用于控制具有实例化视图的表中幂等插入的行为的选项。 这个新功能是由Altinity的特殊要求添加到错误修正版本中的。
|
||||
- 添加 `deduplicate_blocks_in_dependent_materialized_views` 用于控制具有实例化视图的表中幂等插入的行为的选项。
|
||||
[#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy)
|
||||
|
||||
### ClickHouse版本v20.1.2.4,2020-01-22 {#clickhouse-release-v20-1-2-4-2020-01-22}
|
||||
|
||||
@ -423,6 +423,7 @@ void LocalServer::connect()
|
||||
{
|
||||
connection_parameters = ConnectionParameters(getClientConfiguration(), "localhost");
|
||||
|
||||
/// This is needed for table function input(...).
|
||||
ReadBuffer * in;
|
||||
auto table_file = getClientConfiguration().getString("table-file", "-");
|
||||
if (table_file == "-" || table_file == "stdin")
|
||||
|
||||
@ -2731,8 +2731,7 @@ void Server::createInterserverServers(
|
||||
|
||||
void Server::stopServers(
|
||||
std::vector<ProtocolServerAdapter> & servers,
|
||||
const ServerType & server_type
|
||||
) const
|
||||
const ServerType & server_type) const
|
||||
{
|
||||
LoggerRawPtr log = &logger();
|
||||
|
||||
|
||||
@ -129,8 +129,7 @@ private:
|
||||
|
||||
void stopServers(
|
||||
std::vector<ProtocolServerAdapter> & servers,
|
||||
const ServerType & server_type
|
||||
) const;
|
||||
const ServerType & server_type) const;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -118,10 +118,10 @@ AggregateFunctionPtr createAggregateFunctionAnalysisOfVariance(const std::string
|
||||
void registerAggregateFunctionAnalysisOfVariance(AggregateFunctionFactory & factory)
|
||||
{
|
||||
AggregateFunctionProperties properties = { .is_order_dependent = false };
|
||||
factory.registerFunction("analysisOfVariance", {createAggregateFunctionAnalysisOfVariance, properties}, AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("analysisOfVariance", {createAggregateFunctionAnalysisOfVariance, properties}, AggregateFunctionFactory::Case::Insensitive);
|
||||
|
||||
/// This is widely used term
|
||||
factory.registerAlias("anova", "analysisOfVariance", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("anova", "analysisOfVariance", AggregateFunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -361,9 +361,9 @@ void registerAggregateFunctionsAny(AggregateFunctionFactory & factory)
|
||||
AggregateFunctionProperties default_properties = {.returns_default_when_only_null = false, .is_order_dependent = true};
|
||||
|
||||
factory.registerFunction("any", {createAggregateFunctionAny, default_properties});
|
||||
factory.registerAlias("any_value", "any", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("first_value", "any", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("any_value", "any", AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("first_value", "any", AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction("anyLast", {createAggregateFunctionAnyLast, default_properties});
|
||||
factory.registerAlias("last_value", "anyLast", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("last_value", "anyLast", AggregateFunctionFactory::Case::Insensitive);
|
||||
}
|
||||
}
|
||||
|
||||
@ -221,11 +221,11 @@ void registerAggregateFunctionsAnyRespectNulls(AggregateFunctionFactory & factor
|
||||
= {.returns_default_when_only_null = false, .is_order_dependent = true, .is_window_function = true};
|
||||
|
||||
factory.registerFunction("any_respect_nulls", {createAggregateFunctionAnyRespectNulls, default_properties_for_respect_nulls});
|
||||
factory.registerAlias("any_value_respect_nulls", "any_respect_nulls", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("first_value_respect_nulls", "any_respect_nulls", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("any_value_respect_nulls", "any_respect_nulls", AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("first_value_respect_nulls", "any_respect_nulls", AggregateFunctionFactory::Case::Insensitive);
|
||||
|
||||
factory.registerFunction("anyLast_respect_nulls", {createAggregateFunctionAnyLastRespectNulls, default_properties_for_respect_nulls});
|
||||
factory.registerAlias("last_value_respect_nulls", "anyLast_respect_nulls", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("last_value_respect_nulls", "anyLast_respect_nulls", AggregateFunctionFactory::Case::Insensitive);
|
||||
|
||||
/// Must happen after registering any and anyLast
|
||||
factory.registerNullsActionTransformation("any", "any_respect_nulls");
|
||||
|
||||
@ -46,6 +46,6 @@ AggregateFunctionPtr createAggregateFunctionAvg(const std::string & name, const
|
||||
|
||||
void registerAggregateFunctionAvg(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("avg", createAggregateFunctionAvg, AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("avg", createAggregateFunctionAvg, AggregateFunctionFactory::Case::Insensitive);
|
||||
}
|
||||
}
|
||||
|
||||
@ -234,9 +234,9 @@ void registerAggregateFunctionsBitwise(AggregateFunctionFactory & factory)
|
||||
factory.registerFunction("groupBitXor", createAggregateFunctionBitwise<AggregateFunctionGroupBitXorData>);
|
||||
|
||||
/// Aliases for compatibility with MySQL.
|
||||
factory.registerAlias("BIT_OR", "groupBitOr", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("BIT_AND", "groupBitAnd", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("BIT_XOR", "groupBitXor", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("BIT_OR", "groupBitOr", AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("BIT_AND", "groupBitAnd", AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("BIT_XOR", "groupBitXor", AggregateFunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -9,7 +9,7 @@ template <typename T1, typename T2> using AggregateFunctionCorr = AggregateFunct
|
||||
|
||||
void registerAggregateFunctionsStatisticsCorr(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("corr", createAggregateFunctionStatisticsBinary<AggregateFunctionCorr, StatisticsFunctionKind::corr>, AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("corr", createAggregateFunctionStatisticsBinary<AggregateFunctionCorr, StatisticsFunctionKind::corr>, AggregateFunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -37,7 +37,7 @@ AggregateFunctionPtr createAggregateFunctionCount(const std::string & name, cons
|
||||
void registerAggregateFunctionCount(AggregateFunctionFactory & factory)
|
||||
{
|
||||
AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = false };
|
||||
factory.registerFunction("count", {createAggregateFunctionCount, properties}, AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("count", {createAggregateFunctionCount, properties}, AggregateFunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -13,8 +13,8 @@ void registerAggregateFunctionsStatisticsCovar(AggregateFunctionFactory & factor
|
||||
factory.registerFunction("covarPop", createAggregateFunctionStatisticsBinary<AggregateFunctionCovar, StatisticsFunctionKind::covarPop>);
|
||||
|
||||
/// Synonyms for compatibility.
|
||||
factory.registerAlias("COVAR_SAMP", "covarSamp", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("COVAR_POP", "covarPop", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("COVAR_SAMP", "covarSamp", AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("COVAR_POP", "covarPop", AggregateFunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -29,7 +29,7 @@ const String & getAggregateFunctionCanonicalNameIfAny(const String & name)
|
||||
return AggregateFunctionFactory::instance().getCanonicalNameIfAny(name);
|
||||
}
|
||||
|
||||
void AggregateFunctionFactory::registerFunction(const String & name, Value creator_with_properties, CaseSensitiveness case_sensitiveness)
|
||||
void AggregateFunctionFactory::registerFunction(const String & name, Value creator_with_properties, Case case_sensitiveness)
|
||||
{
|
||||
if (creator_with_properties.creator == nullptr)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "AggregateFunctionFactory: "
|
||||
@ -39,7 +39,7 @@ void AggregateFunctionFactory::registerFunction(const String & name, Value creat
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "AggregateFunctionFactory: the aggregate function name '{}' is not unique",
|
||||
name);
|
||||
|
||||
if (case_sensitiveness == CaseInsensitive)
|
||||
if (case_sensitiveness == Case::Insensitive)
|
||||
{
|
||||
auto key = Poco::toLower(name);
|
||||
if (!case_insensitive_aggregate_functions.emplace(key, creator_with_properties).second)
|
||||
|
||||
@ -60,7 +60,7 @@ public:
|
||||
void registerFunction(
|
||||
const String & name,
|
||||
Value creator,
|
||||
CaseSensitiveness case_sensitiveness = CaseSensitive);
|
||||
Case case_sensitiveness = Case::Sensitive);
|
||||
|
||||
/// Register how to transform from one aggregate function to other based on NullsAction
|
||||
/// Registers them both ways:
|
||||
|
||||
@ -840,8 +840,8 @@ void registerAggregateFunctionGroupArray(AggregateFunctionFactory & factory)
|
||||
AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true };
|
||||
|
||||
factory.registerFunction("groupArray", { createAggregateFunctionGroupArray<false>, properties });
|
||||
factory.registerAlias("array_agg", "groupArray", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAliasUnchecked("array_concat_agg", "groupArrayArray", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("array_agg", "groupArray", AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerAliasUnchecked("array_concat_agg", "groupArrayArray", AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction("groupArraySample", { createAggregateFunctionGroupArraySample, properties });
|
||||
factory.registerFunction("groupArrayLast", { createAggregateFunctionGroupArray<true>, properties });
|
||||
}
|
||||
|
||||
@ -150,8 +150,18 @@ public:
|
||||
|
||||
void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional<size_t> /* version */, Arena *) const override
|
||||
{
|
||||
readVarUInt(this->data(place).version, buf);
|
||||
this->data(place).value.read(buf);
|
||||
auto & set = this->data(place).value;
|
||||
auto & version = this->data(place).version;
|
||||
size_t size;
|
||||
readVarUInt(version, buf);
|
||||
readVarUInt(size, buf);
|
||||
set.reserve(size);
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
int key;
|
||||
readIntBinary(key, buf);
|
||||
set.insert(key);
|
||||
}
|
||||
}
|
||||
|
||||
void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
|
||||
@ -292,7 +302,7 @@ public:
|
||||
}
|
||||
return new_map;
|
||||
};
|
||||
auto new_map = rhs_value.size() < set.size() ? create_new_map(rhs_value, set) : create_new_map(set, rhs_value);
|
||||
auto new_map = create_new_map(set, rhs_value);
|
||||
set = std::move(new_map);
|
||||
}
|
||||
}
|
||||
@ -316,11 +326,9 @@ public:
|
||||
readVarUInt(version, buf);
|
||||
readVarUInt(size, buf);
|
||||
set.reserve(size);
|
||||
UInt64 elem_version;
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
auto key = readStringBinaryInto(*arena, buf);
|
||||
readVarUInt(elem_version, buf);
|
||||
set.insert(key);
|
||||
}
|
||||
}
|
||||
|
||||
@ -277,7 +277,7 @@ void registerAggregateFunctionGroupConcat(AggregateFunctionFactory & factory)
|
||||
AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true };
|
||||
|
||||
factory.registerFunction("groupConcat", { createAggregateFunctionGroupConcat, properties });
|
||||
factory.registerAlias("group_concat", "groupConcat", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("group_concat", "groupConcat", AggregateFunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -350,7 +350,7 @@ AggregateFunctionPtr createAggregateFunctionKolmogorovSmirnovTest(
|
||||
|
||||
void registerAggregateFunctionKolmogorovSmirnovTest(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("kolmogorovSmirnovTest", createAggregateFunctionKolmogorovSmirnovTest, AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("kolmogorovSmirnovTest", createAggregateFunctionKolmogorovSmirnovTest, AggregateFunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -15,11 +15,11 @@ void registerAggregateFunctionsStatisticsSecondMoment(AggregateFunctionFactory &
|
||||
factory.registerFunction("stddevPop", createAggregateFunctionStatisticsUnary<AggregateFunctionSecondMoment, StatisticsFunctionKind::stddevPop>);
|
||||
|
||||
/// Synonyms for compatibility.
|
||||
factory.registerAlias("VAR_SAMP", "varSamp", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("VAR_POP", "varPop", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("STDDEV_SAMP", "stddevSamp", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("STDDEV_POP", "stddevPop", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("STD", "stddevPop", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("VAR_SAMP", "varSamp", AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("VAR_POP", "varPop", AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("STDDEV_SAMP", "stddevSamp", AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("STDDEV_POP", "stddevPop", AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("STD", "stddevPop", AggregateFunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -72,7 +72,7 @@ AggregateFunctionPtr createAggregateFunctionSum(const std::string & name, const
|
||||
|
||||
void registerAggregateFunctionSum(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("sum", createAggregateFunctionSum<AggregateFunctionSumSimple>, AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("sum", createAggregateFunctionSum<AggregateFunctionSumSimple>, AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction("sumWithOverflow", createAggregateFunctionSum<AggregateFunctionSumWithOverflow>);
|
||||
factory.registerFunction("sumKahan", createAggregateFunctionSum<AggregateFunctionSumKahan>);
|
||||
}
|
||||
|
||||
@ -535,9 +535,9 @@ void registerAggregateFunctionTopK(AggregateFunctionFactory & factory)
|
||||
|
||||
factory.registerFunction("topK", { createAggregateFunctionTopK<false, false>, properties });
|
||||
factory.registerFunction("topKWeighted", { createAggregateFunctionTopK<true, false>, properties });
|
||||
factory.registerFunction("approx_top_k", { createAggregateFunctionTopK<false, true>, properties }, AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("approx_top_sum", { createAggregateFunctionTopK<true, true>, properties }, AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("approx_top_count", "approx_top_k", AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("approx_top_k", { createAggregateFunctionTopK<false, true>, properties }, AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction("approx_top_sum", { createAggregateFunctionTopK<true, true>, properties }, AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("approx_top_count", "approx_top_k", AggregateFunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -195,8 +195,8 @@ AggregateFunctionPtr createAggregateFunctionMinMax(
|
||||
|
||||
void registerAggregateFunctionsMinMax(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("min", createAggregateFunctionMinMax<true>, AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("max", createAggregateFunctionMinMax<false>, AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("min", createAggregateFunctionMinMax<true>, AggregateFunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction("max", createAggregateFunctionMinMax<false>, AggregateFunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -2919,6 +2919,17 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
|
||||
|
||||
resolveExpressionNode(in_second_argument, scope, false /*allow_lambda_expression*/, true /*allow_table_expression*/);
|
||||
}
|
||||
|
||||
/// Edge case when the first argument of IN is scalar subquery.
|
||||
auto & in_first_argument = function_in_arguments_nodes[0];
|
||||
auto first_argument_type = in_first_argument->getNodeType();
|
||||
if (first_argument_type == QueryTreeNodeType::QUERY || first_argument_type == QueryTreeNodeType::UNION)
|
||||
{
|
||||
IdentifierResolveScope subquery_scope(in_first_argument, &scope /*parent_scope*/);
|
||||
subquery_scope.subquery_depth = scope.subquery_depth + 1;
|
||||
|
||||
evaluateScalarSubqueryIfNeeded(in_first_argument, subquery_scope);
|
||||
}
|
||||
}
|
||||
|
||||
/// Initialize function argument columns
|
||||
|
||||
@ -21,6 +21,7 @@
|
||||
#include <Common/StringUtils.h>
|
||||
#include <Common/filesystemHelpers.h>
|
||||
#include <Common/NetException.h>
|
||||
#include <Common/SignalHandlers.h>
|
||||
#include <Common/tryGetFileNameByFileDescriptor.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
@ -302,7 +303,13 @@ public:
|
||||
};
|
||||
|
||||
|
||||
ClientBase::~ClientBase() = default;
|
||||
ClientBase::~ClientBase()
|
||||
{
|
||||
writeSignalIDtoSignalPipe(SignalListener::StopThread);
|
||||
signal_listener_thread.join();
|
||||
HandledSignals::instance().reset();
|
||||
}
|
||||
|
||||
ClientBase::ClientBase(
|
||||
int in_fd_,
|
||||
int out_fd_,
|
||||
@ -2070,9 +2077,18 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
|
||||
progress_indication.writeFinalProgress();
|
||||
output_stream << std::endl << std::endl;
|
||||
}
|
||||
else if (getClientConfiguration().getBool("print-time-to-stderr", false))
|
||||
else
|
||||
{
|
||||
error_stream << progress_indication.elapsedSeconds() << "\n";
|
||||
const auto & config = getClientConfiguration();
|
||||
if (config.getBool("print-time-to-stderr", false))
|
||||
error_stream << progress_indication.elapsedSeconds() << "\n";
|
||||
|
||||
const auto & print_memory_mode = config.getString("print-memory-to-stderr", "");
|
||||
auto peak_memeory_usage = std::max<Int64>(progress_indication.getMemoryUsage().peak, 0);
|
||||
if (print_memory_mode == "default")
|
||||
error_stream << peak_memeory_usage << "\n";
|
||||
else if (print_memory_mode == "readable")
|
||||
error_stream << formatReadableSizeWithBinarySuffix(peak_memeory_usage) << "\n";
|
||||
}
|
||||
|
||||
if (!is_interactive && getClientConfiguration().getBool("print-num-processed-rows", false))
|
||||
@ -3036,6 +3052,7 @@ void ClientBase::init(int argc, char ** argv)
|
||||
("disable_suggestion,A", "Disable loading suggestion data. Note that suggestion data is loaded asynchronously through a second connection to ClickHouse server. Also it is reasonable to disable suggestion if you want to paste a query with TAB characters. Shorthand option -A is for those who get used to mysql client.")
|
||||
("wait_for_suggestions_to_load", "Load suggestion data synchonously.")
|
||||
("time,t", "print query execution time to stderr in non-interactive mode (for benchmarks)")
|
||||
("memory-usage", po::value<std::string>()->implicit_value("default")->default_value("none"), "print memory usage to stderr in non-interactive mode (for benchmarks). Values: 'none', 'default', 'readable'")
|
||||
|
||||
("echo", "in batch mode, print query before execution")
|
||||
|
||||
@ -3062,6 +3079,8 @@ void ClientBase::init(int argc, char ** argv)
|
||||
("max_memory_usage_in_client", po::value<std::string>(), "Set memory limit in client/local server")
|
||||
|
||||
("fuzzer-args", po::value<std::string>(), "Command line arguments for the LLVM's libFuzzer driver. Only relevant if the application is compiled with libFuzzer.")
|
||||
|
||||
("client_logs_file", po::value<std::string>(), "Path to a file for writing client logs. Currently we only have fatal logs (when the client crashes)")
|
||||
;
|
||||
|
||||
addOptions(options_description);
|
||||
@ -3121,6 +3140,14 @@ void ClientBase::init(int argc, char ** argv)
|
||||
/// Output execution time to stderr in batch mode.
|
||||
if (options.count("time"))
|
||||
getClientConfiguration().setBool("print-time-to-stderr", true);
|
||||
if (options.count("memory-usage"))
|
||||
{
|
||||
const auto & memory_usage_mode = options["memory-usage"].as<std::string>();
|
||||
if (memory_usage_mode != "none" && memory_usage_mode != "default" && memory_usage_mode != "readable")
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown memory-usage mode: {}", memory_usage_mode);
|
||||
getClientConfiguration().setString("print-memory-to-stderr", memory_usage_mode);
|
||||
}
|
||||
|
||||
if (options.count("query"))
|
||||
queries = options["query"].as<std::vector<std::string>>();
|
||||
if (options.count("query_id"))
|
||||
@ -3218,6 +3245,25 @@ void ClientBase::init(int argc, char ** argv)
|
||||
total_memory_tracker.setDescription("(total)");
|
||||
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
|
||||
}
|
||||
|
||||
/// Print stacktrace in case of crash
|
||||
HandledSignals::instance().setupTerminateHandler();
|
||||
HandledSignals::instance().setupCommonDeadlySignalHandlers();
|
||||
/// We don't setup signal handlers for SIGINT, SIGQUIT, SIGTERM because we don't
|
||||
/// have an option for client to shutdown gracefully.
|
||||
|
||||
fatal_channel_ptr = new Poco::SplitterChannel;
|
||||
fatal_console_channel_ptr = new Poco::ConsoleChannel;
|
||||
fatal_channel_ptr->addChannel(fatal_console_channel_ptr);
|
||||
if (options.count("client_logs_file"))
|
||||
{
|
||||
fatal_file_channel_ptr = new Poco::SimpleFileChannel(options["client_logs_file"].as<std::string>());
|
||||
fatal_channel_ptr->addChannel(fatal_file_channel_ptr);
|
||||
}
|
||||
|
||||
fatal_log = createLogger("ClientBase", fatal_channel_ptr.get(), Poco::Message::PRIO_FATAL);
|
||||
signal_listener = std::make_unique<SignalListener>(nullptr, fatal_log);
|
||||
signal_listener_thread.start(*signal_listener);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -12,6 +12,9 @@
|
||||
#include <Core/ExternalTable.h>
|
||||
#include <Core/Settings.h>
|
||||
#include <Poco/Util/Application.h>
|
||||
#include <Poco/ConsoleChannel.h>
|
||||
#include <Poco/SimpleFileChannel.h>
|
||||
#include <Poco/SplitterChannel.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Client/Suggest.h>
|
||||
#include <boost/program_options.hpp>
|
||||
@ -212,6 +215,13 @@ protected:
|
||||
SharedContextHolder shared_context;
|
||||
ContextMutablePtr global_context;
|
||||
|
||||
LoggerPtr fatal_log;
|
||||
Poco::AutoPtr<Poco::SplitterChannel> fatal_channel_ptr;
|
||||
Poco::AutoPtr<Poco::Channel> fatal_console_channel_ptr;
|
||||
Poco::AutoPtr<Poco::Channel> fatal_file_channel_ptr;
|
||||
Poco::Thread signal_listener_thread;
|
||||
std::unique_ptr<Poco::Runnable> signal_listener;
|
||||
|
||||
bool is_interactive = false; /// Use either interactive line editing interface or batch mode.
|
||||
bool is_multiquery = false;
|
||||
bool delayed_interactive = false;
|
||||
|
||||
@ -54,8 +54,6 @@ public:
|
||||
|
||||
struct ReplicaInfo
|
||||
{
|
||||
bool collaborate_with_initiator{false};
|
||||
size_t all_replicas_count{0};
|
||||
size_t number_of_current_replica{0};
|
||||
};
|
||||
|
||||
|
||||
@ -175,4 +175,5 @@ private:
|
||||
|
||||
ReadBuffer * in;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -142,13 +142,12 @@ void MultiplexedConnections::sendQuery(
|
||||
modified_settings.group_by_two_level_threshold = 0;
|
||||
modified_settings.group_by_two_level_threshold_bytes = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (replica_info)
|
||||
{
|
||||
client_info.collaborate_with_initiator = true;
|
||||
client_info.count_participating_replicas = replica_info->all_replicas_count;
|
||||
client_info.number_of_current_replica = replica_info->number_of_current_replica;
|
||||
}
|
||||
if (replica_info)
|
||||
{
|
||||
client_info.collaborate_with_initiator = true;
|
||||
client_info.number_of_current_replica = replica_info->number_of_current_replica;
|
||||
}
|
||||
|
||||
/// FIXME: Remove once we will make `allow_experimental_analyzer` obsolete setting.
|
||||
|
||||
@ -1101,4 +1101,10 @@ void ColumnObject::finalize()
|
||||
checkObjectHasNoAmbiguosPaths(getKeys());
|
||||
}
|
||||
|
||||
void ColumnObject::updateHashFast(SipHash & hash) const
|
||||
{
|
||||
for (const auto & entry : subcolumns)
|
||||
for (auto & part : entry->data.data)
|
||||
part->updateHashFast(hash);
|
||||
}
|
||||
}
|
||||
|
||||
@ -253,7 +253,7 @@ public:
|
||||
const char * skipSerializedInArena(const char *) const override { throwMustBeConcrete(); }
|
||||
void updateHashWithValue(size_t, SipHash &) const override { throwMustBeConcrete(); }
|
||||
void updateWeakHash32(WeakHash32 &) const override { throwMustBeConcrete(); }
|
||||
void updateHashFast(SipHash &) const override { throwMustBeConcrete(); }
|
||||
void updateHashFast(SipHash & hash) const override;
|
||||
void expand(const Filter &, bool) override { throwMustBeConcrete(); }
|
||||
bool hasEqualValues() const override { throwMustBeConcrete(); }
|
||||
size_t byteSizeAt(size_t) const override { throwMustBeConcrete(); }
|
||||
|
||||
@ -187,12 +187,9 @@ void * Allocator<clear_memory_, populate>::realloc(void * buf, size_t old_size,
|
||||
#if USE_GWP_ASAN
|
||||
if (unlikely(GWPAsan::GuardedAlloc.shouldSample()))
|
||||
{
|
||||
auto trace_alloc = CurrentMemoryTracker::alloc(new_size);
|
||||
if (void * ptr = GWPAsan::GuardedAlloc.allocate(new_size, alignment))
|
||||
{
|
||||
auto trace_free = CurrentMemoryTracker::free(old_size);
|
||||
auto trace_alloc = CurrentMemoryTracker::alloc(new_size);
|
||||
trace_free.onFree(buf, old_size);
|
||||
|
||||
memcpy(ptr, buf, std::min(old_size, new_size));
|
||||
free(buf, old_size);
|
||||
trace_alloc.onAlloc(buf, new_size);
|
||||
@ -209,6 +206,7 @@ void * Allocator<clear_memory_, populate>::realloc(void * buf, size_t old_size,
|
||||
}
|
||||
else
|
||||
{
|
||||
[[maybe_unused]] auto trace_free = CurrentMemoryTracker::free(old_size);
|
||||
ProfileEvents::increment(ProfileEvents::GWPAsanAllocateFailed);
|
||||
}
|
||||
}
|
||||
@ -231,13 +229,17 @@ void * Allocator<clear_memory_, populate>::realloc(void * buf, size_t old_size,
|
||||
if (alignment <= MALLOC_MIN_ALIGNMENT)
|
||||
{
|
||||
/// Resize malloc'd memory region with no special alignment requirement.
|
||||
auto trace_free = CurrentMemoryTracker::free(old_size);
|
||||
/// Realloc can do 2 possible things:
|
||||
/// - expand existing memory region
|
||||
/// - allocate new memory block and free the old one
|
||||
/// Because we don't know which option will be picked we need to make sure there is enough
|
||||
/// memory for all options
|
||||
auto trace_alloc = CurrentMemoryTracker::alloc(new_size);
|
||||
trace_free.onFree(buf, old_size);
|
||||
|
||||
void * new_buf = ::realloc(buf, new_size);
|
||||
if (nullptr == new_buf)
|
||||
{
|
||||
[[maybe_unused]] auto trace_free = CurrentMemoryTracker::free(old_size);
|
||||
throw DB::ErrnoException(
|
||||
DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY,
|
||||
"Allocator: Cannot realloc from {} to {}",
|
||||
@ -246,6 +248,8 @@ void * Allocator<clear_memory_, populate>::realloc(void * buf, size_t old_size,
|
||||
}
|
||||
|
||||
buf = new_buf;
|
||||
auto trace_free = CurrentMemoryTracker::free(old_size);
|
||||
trace_free.onFree(buf, old_size);
|
||||
trace_alloc.onAlloc(buf, new_size);
|
||||
|
||||
if constexpr (clear_memory)
|
||||
|
||||
187
src/Common/CollectionOfDerived.h
Normal file
187
src/Common/CollectionOfDerived.h
Normal file
@ -0,0 +1,187 @@
|
||||
#pragma once
|
||||
|
||||
#include <base/defines.h>
|
||||
|
||||
#include <Common/Exception.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
#include <typeindex>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
/* This is a collections of objects derived from ItemBase.
|
||||
* Collection contains no more than one instance for each derived type.
|
||||
* The derived type is used to access the instance.
|
||||
*/
|
||||
|
||||
template<class ItemBase>
|
||||
class CollectionOfDerivedItems
|
||||
{
|
||||
public:
|
||||
using Self = CollectionOfDerivedItems<ItemBase>;
|
||||
using ItemPtr = std::shared_ptr<ItemBase>;
|
||||
|
||||
private:
|
||||
struct Rec
|
||||
{
|
||||
std::type_index type_idx;
|
||||
ItemPtr ptr;
|
||||
|
||||
bool operator<(const Rec & other) const
|
||||
{
|
||||
return type_idx < other.type_idx;
|
||||
}
|
||||
|
||||
bool operator<(const std::type_index & value) const
|
||||
{
|
||||
return type_idx < value;
|
||||
}
|
||||
|
||||
bool operator==(const Rec & other) const
|
||||
{
|
||||
return type_idx == other.type_idx;
|
||||
}
|
||||
};
|
||||
using Records = std::vector<Rec>;
|
||||
|
||||
public:
|
||||
void swap(Self & other) noexcept
|
||||
{
|
||||
records.swap(other.records);
|
||||
}
|
||||
|
||||
void clear()
|
||||
{
|
||||
records.clear();
|
||||
}
|
||||
|
||||
bool empty() const
|
||||
{
|
||||
return records.empty();
|
||||
}
|
||||
|
||||
size_t size() const
|
||||
{
|
||||
return records.size();
|
||||
}
|
||||
|
||||
Self clone() const
|
||||
{
|
||||
Self result;
|
||||
result.records.reserve(records.size());
|
||||
for (const auto & rec : records)
|
||||
result.records.emplace_back(rec.type_idx, rec.ptr->clone());
|
||||
return result;
|
||||
}
|
||||
|
||||
// append items for other inscnace only if there is no such item in current instance
|
||||
void appendIfUniq(Self && other)
|
||||
{
|
||||
auto middle_idx = records.size();
|
||||
std::move(other.records.begin(), other.records.end(), std::back_inserter(records));
|
||||
// merge is stable
|
||||
std::inplace_merge(records.begin(), records.begin() + middle_idx, records.end());
|
||||
// remove duplicates
|
||||
records.erase(std::unique(records.begin(), records.end()), records.end());
|
||||
|
||||
assert(std::is_sorted(records.begin(), records.end()));
|
||||
assert(isUniqTypes());
|
||||
}
|
||||
|
||||
template <class T>
|
||||
void add(std::shared_ptr<T> info)
|
||||
{
|
||||
static_assert(std::is_base_of_v<ItemBase, T>, "Template parameter must inherit items base class");
|
||||
return addImpl(std::type_index(typeid(T)), std::move(info));
|
||||
}
|
||||
|
||||
template <class T>
|
||||
std::shared_ptr<T> get() const
|
||||
{
|
||||
static_assert(std::is_base_of_v<ItemBase, T>, "Template parameter must inherit items base class");
|
||||
auto it = getImpl(std::type_index(typeid(T)));
|
||||
if (it == records.cend())
|
||||
return nullptr;
|
||||
auto cast = std::dynamic_pointer_cast<T>(it->ptr);
|
||||
chassert(cast);
|
||||
return cast;
|
||||
}
|
||||
|
||||
template <class T>
|
||||
std::shared_ptr<T> extract()
|
||||
{
|
||||
static_assert(std::is_base_of_v<ItemBase, T>, "Template parameter must inherit items base class");
|
||||
auto it = getImpl(std::type_index(typeid(T)));
|
||||
if (it == records.cend())
|
||||
return nullptr;
|
||||
auto cast = std::dynamic_pointer_cast<T>(it->ptr);
|
||||
chassert(cast);
|
||||
|
||||
records.erase(it);
|
||||
return cast;
|
||||
}
|
||||
|
||||
std::string debug() const
|
||||
{
|
||||
std::string result;
|
||||
|
||||
for (auto & rec : records)
|
||||
{
|
||||
result.append(rec.type_idx.name());
|
||||
result.append(" ");
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private:
|
||||
bool isUniqTypes() const
|
||||
{
|
||||
auto uniq_it = std::adjacent_find(records.begin(), records.end());
|
||||
return uniq_it == records.end();
|
||||
}
|
||||
|
||||
void addImpl(std::type_index type_idx, ItemPtr item)
|
||||
{
|
||||
auto it = std::lower_bound(records.begin(), records.end(), type_idx);
|
||||
|
||||
if (it == records.end())
|
||||
{
|
||||
records.emplace_back(type_idx, item);
|
||||
return;
|
||||
}
|
||||
|
||||
if (it->type_idx == type_idx)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "inserted items must be unique by their type, type {} is inserted twice", type_idx.name());
|
||||
|
||||
|
||||
records.emplace(it, type_idx, item);
|
||||
}
|
||||
|
||||
Records::const_iterator getImpl(std::type_index type_idx) const
|
||||
{
|
||||
auto it = std::lower_bound(records.cbegin(), records.cend(), type_idx);
|
||||
|
||||
if (it == records.cend())
|
||||
return records.cend();
|
||||
|
||||
if (it->type_idx != type_idx)
|
||||
return records.cend();
|
||||
|
||||
return it;
|
||||
}
|
||||
|
||||
Records records;
|
||||
};
|
||||
|
||||
}
|
||||
@ -39,16 +39,16 @@ protected:
|
||||
|
||||
public:
|
||||
/// For compatibility with SQL, it's possible to specify that certain function name is case insensitive.
|
||||
enum CaseSensitiveness
|
||||
enum Case
|
||||
{
|
||||
CaseSensitive,
|
||||
CaseInsensitive
|
||||
Sensitive,
|
||||
Insensitive
|
||||
};
|
||||
|
||||
/** Register additional name for value
|
||||
* real_name have to be already registered.
|
||||
*/
|
||||
void registerAlias(const String & alias_name, const String & real_name, CaseSensitiveness case_sensitiveness = CaseSensitive)
|
||||
void registerAlias(const String & alias_name, const String & real_name, Case case_sensitiveness = Sensitive)
|
||||
{
|
||||
const auto & creator_map = getMap();
|
||||
const auto & case_insensitive_creator_map = getCaseInsensitiveMap();
|
||||
@ -66,12 +66,12 @@ public:
|
||||
}
|
||||
|
||||
/// We need sure the real_name exactly exists when call the function directly.
|
||||
void registerAliasUnchecked(const String & alias_name, const String & real_name, CaseSensitiveness case_sensitiveness = CaseSensitive)
|
||||
void registerAliasUnchecked(const String & alias_name, const String & real_name, Case case_sensitiveness = Sensitive)
|
||||
{
|
||||
String alias_name_lowercase = Poco::toLower(alias_name);
|
||||
const String factory_name = getFactoryName();
|
||||
|
||||
if (case_sensitiveness == CaseInsensitive)
|
||||
if (case_sensitiveness == Insensitive)
|
||||
{
|
||||
if (!case_insensitive_aliases.emplace(alias_name_lowercase, real_name).second)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "{}: case insensitive alias name '{}' is not unique", factory_name, alias_name);
|
||||
|
||||
@ -116,32 +116,32 @@ public:
|
||||
return elements;
|
||||
}
|
||||
|
||||
bool exists(const std::string & path) const override
|
||||
bool exists(const std::string & file_name) const override
|
||||
{
|
||||
return fs::exists(getPath(path));
|
||||
return fs::exists(getPath(file_name));
|
||||
}
|
||||
|
||||
std::string read(const std::string & path) const override
|
||||
std::string read(const std::string & file_name) const override
|
||||
{
|
||||
ReadBufferFromFile in(getPath(path));
|
||||
ReadBufferFromFile in(getPath(file_name));
|
||||
std::string data;
|
||||
readStringUntilEOF(data, in);
|
||||
return data;
|
||||
}
|
||||
|
||||
void write(const std::string & path, const std::string & data, bool replace) override
|
||||
void write(const std::string & file_name, const std::string & data, bool replace) override
|
||||
{
|
||||
if (!replace && fs::exists(path))
|
||||
if (!replace && fs::exists(file_name))
|
||||
{
|
||||
throw Exception(
|
||||
ErrorCodes::NAMED_COLLECTION_ALREADY_EXISTS,
|
||||
"Metadata file {} for named collection already exists",
|
||||
path);
|
||||
file_name);
|
||||
}
|
||||
|
||||
fs::create_directories(root_path);
|
||||
|
||||
auto tmp_path = getPath(path + ".tmp");
|
||||
auto tmp_path = getPath(file_name + ".tmp");
|
||||
WriteBufferFromFile out(tmp_path, data.size(), O_WRONLY | O_CREAT | O_EXCL);
|
||||
writeString(data, out);
|
||||
|
||||
@ -150,28 +150,32 @@ public:
|
||||
out.sync();
|
||||
out.close();
|
||||
|
||||
fs::rename(tmp_path, getPath(path));
|
||||
fs::rename(tmp_path, getPath(file_name));
|
||||
}
|
||||
|
||||
void remove(const std::string & path) override
|
||||
void remove(const std::string & file_name) override
|
||||
{
|
||||
if (!removeIfExists(getPath(path)))
|
||||
if (!removeIfExists(file_name))
|
||||
{
|
||||
throw Exception(
|
||||
ErrorCodes::NAMED_COLLECTION_DOESNT_EXIST,
|
||||
"Cannot remove `{}`, because it doesn't exist", path);
|
||||
"Cannot remove `{}`, because it doesn't exist", file_name);
|
||||
}
|
||||
}
|
||||
|
||||
bool removeIfExists(const std::string & path) override
|
||||
bool removeIfExists(const std::string & file_name) override
|
||||
{
|
||||
return fs::remove(getPath(path));
|
||||
return fs::remove(getPath(file_name));
|
||||
}
|
||||
|
||||
private:
|
||||
std::string getPath(const std::string & path) const
|
||||
std::string getPath(const std::string & file_name) const
|
||||
{
|
||||
return fs::path(root_path) / path;
|
||||
const auto file_name_as_path = fs::path(file_name);
|
||||
if (file_name_as_path.is_absolute())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Filename {} cannot be an absolute path", file_name);
|
||||
|
||||
return fs::path(root_path) / file_name_as_path;
|
||||
}
|
||||
|
||||
/// Delete .tmp files. They could be left undeleted in case of
|
||||
@ -264,49 +268,49 @@ public:
|
||||
return children;
|
||||
}
|
||||
|
||||
bool exists(const std::string & path) const override
|
||||
bool exists(const std::string & file_name) const override
|
||||
{
|
||||
return getClient()->exists(getPath(path));
|
||||
return getClient()->exists(getPath(file_name));
|
||||
}
|
||||
|
||||
std::string read(const std::string & path) const override
|
||||
std::string read(const std::string & file_name) const override
|
||||
{
|
||||
return getClient()->get(getPath(path));
|
||||
return getClient()->get(getPath(file_name));
|
||||
}
|
||||
|
||||
void write(const std::string & path, const std::string & data, bool replace) override
|
||||
void write(const std::string & file_name, const std::string & data, bool replace) override
|
||||
{
|
||||
if (replace)
|
||||
{
|
||||
getClient()->createOrUpdate(getPath(path), data, zkutil::CreateMode::Persistent);
|
||||
getClient()->createOrUpdate(getPath(file_name), data, zkutil::CreateMode::Persistent);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto code = getClient()->tryCreate(getPath(path), data, zkutil::CreateMode::Persistent);
|
||||
auto code = getClient()->tryCreate(getPath(file_name), data, zkutil::CreateMode::Persistent);
|
||||
|
||||
if (code == Coordination::Error::ZNODEEXISTS)
|
||||
{
|
||||
throw Exception(
|
||||
ErrorCodes::NAMED_COLLECTION_ALREADY_EXISTS,
|
||||
"Metadata file {} for named collection already exists",
|
||||
path);
|
||||
file_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void remove(const std::string & path) override
|
||||
void remove(const std::string & file_name) override
|
||||
{
|
||||
getClient()->remove(getPath(path));
|
||||
getClient()->remove(getPath(file_name));
|
||||
}
|
||||
|
||||
bool removeIfExists(const std::string & path) override
|
||||
bool removeIfExists(const std::string & file_name) override
|
||||
{
|
||||
auto code = getClient()->tryRemove(getPath(path));
|
||||
auto code = getClient()->tryRemove(getPath(file_name));
|
||||
if (code == Coordination::Error::ZOK)
|
||||
return true;
|
||||
if (code == Coordination::Error::ZNONODE)
|
||||
return false;
|
||||
throw Coordination::Exception::fromPath(code, getPath(path));
|
||||
throw Coordination::Exception::fromPath(code, getPath(file_name));
|
||||
}
|
||||
|
||||
private:
|
||||
@ -320,9 +324,13 @@ private:
|
||||
return zookeeper_client;
|
||||
}
|
||||
|
||||
std::string getPath(const std::string & path) const
|
||||
std::string getPath(const std::string & file_name) const
|
||||
{
|
||||
return fs::path(root_path) / path;
|
||||
const auto file_name_as_path = fs::path(file_name);
|
||||
if (file_name_as_path.is_absolute())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Filename {} cannot be an absolute path", file_name);
|
||||
|
||||
return fs::path(root_path) / file_name_as_path;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -72,11 +72,6 @@ public:
|
||||
/// How much seconds passed since query execution start.
|
||||
double elapsedSeconds() const { return getElapsedNanoseconds() / 1e9; }
|
||||
|
||||
void updateThreadEventData(HostToTimesMap & new_hosts_data);
|
||||
|
||||
private:
|
||||
double getCPUUsage();
|
||||
|
||||
struct MemoryUsage
|
||||
{
|
||||
UInt64 total = 0;
|
||||
@ -86,6 +81,11 @@ private:
|
||||
|
||||
MemoryUsage getMemoryUsage() const;
|
||||
|
||||
void updateThreadEventData(HostToTimesMap & new_hosts_data);
|
||||
|
||||
private:
|
||||
double getCPUUsage();
|
||||
|
||||
UInt64 getElapsedNanoseconds() const;
|
||||
|
||||
/// This flag controls whether to show the progress bar. We start showing it after
|
||||
|
||||
635
src/Common/SignalHandlers.cpp
Normal file
635
src/Common/SignalHandlers.cpp
Normal file
@ -0,0 +1,635 @@
|
||||
#include <Common/SignalHandlers.h>
|
||||
#include <Common/config_version.h>
|
||||
#include <Common/getHashOfLoadedBinary.h>
|
||||
#include <Common/CurrentThread.h>
|
||||
#include <Daemon/BaseDaemon.h>
|
||||
#include <Daemon/SentryWriter.h>
|
||||
#include <base/sleep.h>
|
||||
#include <base/getThreadId.h>
|
||||
#include <IO/WriteBufferFromFileDescriptor.h>
|
||||
#include <IO/ReadBufferFromFileDescriptor.h>
|
||||
#include <IO/WriteBufferFromFileDescriptorDiscardOnFailure.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Core/Settings.h>
|
||||
|
||||
#pragma clang diagnostic ignored "-Wreserved-identifier"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_SET_SIGNAL_HANDLER;
|
||||
extern const int CANNOT_SEND_SIGNAL;
|
||||
}
|
||||
}
|
||||
|
||||
using namespace DB;
|
||||
|
||||
|
||||
void call_default_signal_handler(int sig)
|
||||
{
|
||||
if (SIG_ERR == signal(sig, SIG_DFL))
|
||||
throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
|
||||
|
||||
if (0 != raise(sig))
|
||||
throw ErrnoException(ErrorCodes::CANNOT_SEND_SIGNAL, "Cannot send signal");
|
||||
}
|
||||
|
||||
|
||||
void writeSignalIDtoSignalPipe(int sig)
|
||||
{
|
||||
auto saved_errno = errno; /// We must restore previous value of errno in signal handler.
|
||||
|
||||
char buf[signal_pipe_buf_size];
|
||||
auto & signal_pipe = HandledSignals::instance().signal_pipe;
|
||||
WriteBufferFromFileDescriptor out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
|
||||
writeBinary(sig, out);
|
||||
out.next();
|
||||
|
||||
errno = saved_errno;
|
||||
}
|
||||
|
||||
void closeLogsSignalHandler(int sig, siginfo_t *, void *)
|
||||
{
|
||||
DENY_ALLOCATIONS_IN_SCOPE;
|
||||
writeSignalIDtoSignalPipe(sig);
|
||||
}
|
||||
|
||||
void terminateRequestedSignalHandler(int sig, siginfo_t *, void *)
|
||||
{
|
||||
DENY_ALLOCATIONS_IN_SCOPE;
|
||||
writeSignalIDtoSignalPipe(sig);
|
||||
}
|
||||
|
||||
|
||||
void signalHandler(int sig, siginfo_t * info, void * context)
|
||||
{
|
||||
if (asynchronous_stack_unwinding && sig == SIGSEGV)
|
||||
siglongjmp(asynchronous_stack_unwinding_signal_jump_buffer, 1);
|
||||
|
||||
DENY_ALLOCATIONS_IN_SCOPE;
|
||||
auto saved_errno = errno; /// We must restore previous value of errno in signal handler.
|
||||
|
||||
char buf[signal_pipe_buf_size];
|
||||
auto & signal_pipe = HandledSignals::instance().signal_pipe;
|
||||
WriteBufferFromFileDescriptorDiscardOnFailure out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
|
||||
|
||||
const ucontext_t * signal_context = reinterpret_cast<ucontext_t *>(context);
|
||||
const StackTrace stack_trace(*signal_context);
|
||||
|
||||
#if USE_GWP_ASAN
|
||||
if (const auto fault_address = reinterpret_cast<uintptr_t>(info->si_addr);
|
||||
GWPAsan::isGWPAsanError(fault_address))
|
||||
GWPAsan::printReport(fault_address);
|
||||
#endif
|
||||
|
||||
writeBinary(sig, out);
|
||||
writePODBinary(*info, out);
|
||||
writePODBinary(signal_context, out);
|
||||
writePODBinary(stack_trace, out);
|
||||
writeVectorBinary(Exception::enable_job_stack_trace ? Exception::thread_frame_pointers : std::vector<StackTrace::FramePointers>{}, out);
|
||||
writeBinary(static_cast<UInt32>(getThreadId()), out);
|
||||
writePODBinary(current_thread, out);
|
||||
|
||||
out.next();
|
||||
|
||||
if (sig != SIGTSTP) /// This signal is used for debugging.
|
||||
{
|
||||
/// The time that is usually enough for separate thread to print info into log.
|
||||
/// Under MSan full stack unwinding with DWARF info about inline functions takes 101 seconds in one case.
|
||||
for (size_t i = 0; i < 300; ++i)
|
||||
{
|
||||
/// We will synchronize with the thread printing the messages with an atomic variable to finish earlier.
|
||||
if (HandledSignals::instance().fatal_error_printed.test())
|
||||
break;
|
||||
|
||||
/// This coarse method of synchronization is perfectly ok for fatal signals.
|
||||
sleepForSeconds(1);
|
||||
}
|
||||
|
||||
/// Wait for all logs flush operations
|
||||
sleepForSeconds(3);
|
||||
call_default_signal_handler(sig);
|
||||
}
|
||||
|
||||
errno = saved_errno;
|
||||
}
|
||||
|
||||
|
||||
[[noreturn]] void terminate_handler()
|
||||
{
|
||||
static thread_local bool terminating = false;
|
||||
if (terminating)
|
||||
abort();
|
||||
|
||||
terminating = true;
|
||||
|
||||
std::string log_message;
|
||||
|
||||
if (std::current_exception())
|
||||
log_message = "Terminate called for uncaught exception:\n" + getCurrentExceptionMessage(true);
|
||||
else
|
||||
log_message = "Terminate called without an active exception";
|
||||
|
||||
/// POSIX.1 says that write(2)s of less than PIPE_BUF bytes must be atomic - man 7 pipe
|
||||
/// And the buffer should not be too small because our exception messages can be large.
|
||||
static constexpr size_t buf_size = PIPE_BUF;
|
||||
|
||||
if (log_message.size() > buf_size - 16)
|
||||
log_message.resize(buf_size - 16);
|
||||
|
||||
char buf[buf_size];
|
||||
auto & signal_pipe = HandledSignals::instance().signal_pipe;
|
||||
WriteBufferFromFileDescriptor out(signal_pipe.fds_rw[1], buf_size, buf);
|
||||
|
||||
writeBinary(static_cast<int>(SignalListener::StdTerminate), out);
|
||||
writeBinary(static_cast<UInt32>(getThreadId()), out);
|
||||
writeBinary(log_message, out);
|
||||
out.next();
|
||||
|
||||
abort();
|
||||
}
|
||||
|
||||
#if defined(SANITIZER)
|
||||
template <typename T>
|
||||
struct ValueHolder
|
||||
{
|
||||
ValueHolder(T value_) : value(value_)
|
||||
{}
|
||||
|
||||
T value;
|
||||
};
|
||||
|
||||
extern "C" void __sanitizer_set_death_callback(void (*)());
|
||||
|
||||
/// Sanitizers may not expect some function calls from death callback.
|
||||
/// Let's try to disable instrumentation to avoid possible issues.
|
||||
/// However, this callback may call other functions that are still instrumented.
|
||||
/// We can try [[clang::always_inline]] attribute for statements in future (available in clang-15)
|
||||
/// See https://github.com/google/sanitizers/issues/1543 and https://github.com/google/sanitizers/issues/1549.
|
||||
static DISABLE_SANITIZER_INSTRUMENTATION void sanitizerDeathCallback()
|
||||
{
|
||||
DENY_ALLOCATIONS_IN_SCOPE;
|
||||
/// Also need to send data via pipe. Otherwise it may lead to deadlocks or failures in printing diagnostic info.
|
||||
|
||||
char buf[signal_pipe_buf_size];
|
||||
auto & signal_pipe = HandledSignals::instance().signal_pipe;
|
||||
WriteBufferFromFileDescriptorDiscardOnFailure out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
|
||||
|
||||
const StackTrace stack_trace;
|
||||
|
||||
writeBinary(SignalListener::SanitizerTrap, out);
|
||||
writePODBinary(stack_trace, out);
|
||||
/// We create a dummy struct with a constructor so DISABLE_SANITIZER_INSTRUMENTATION is not applied to it
|
||||
/// otherwise, Memory sanitizer can't know that values initiialized inside this function are actually initialized
|
||||
/// because instrumentations are disabled leading to false positives later on
|
||||
ValueHolder<UInt32> thread_id{static_cast<UInt32>(getThreadId())};
|
||||
writeBinary(thread_id.value, out);
|
||||
writePODBinary(current_thread, out);
|
||||
|
||||
out.next();
|
||||
|
||||
/// The time that is usually enough for separate thread to print info into log.
|
||||
sleepForSeconds(20);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
void HandledSignals::addSignalHandler(const std::vector<int> & signals, signal_function handler, bool register_signal)
|
||||
{
|
||||
struct sigaction sa;
|
||||
memset(&sa, 0, sizeof(sa));
|
||||
sa.sa_sigaction = handler;
|
||||
sa.sa_flags = SA_SIGINFO;
|
||||
|
||||
#if defined(OS_DARWIN)
|
||||
sigemptyset(&sa.sa_mask);
|
||||
for (auto signal : signals)
|
||||
sigaddset(&sa.sa_mask, signal);
|
||||
#else
|
||||
if (sigemptyset(&sa.sa_mask))
|
||||
throw Poco::Exception("Cannot set signal handler.");
|
||||
|
||||
for (auto signal : signals)
|
||||
if (sigaddset(&sa.sa_mask, signal))
|
||||
throw Poco::Exception("Cannot set signal handler.");
|
||||
#endif
|
||||
|
||||
for (auto signal : signals)
|
||||
if (sigaction(signal, &sa, nullptr))
|
||||
throw Poco::Exception("Cannot set signal handler.");
|
||||
|
||||
if (register_signal)
|
||||
std::copy(signals.begin(), signals.end(), std::back_inserter(handled_signals));
|
||||
}
|
||||
|
||||
void blockSignals(const std::vector<int> & signals)
|
||||
{
|
||||
sigset_t sig_set;
|
||||
|
||||
#if defined(OS_DARWIN)
|
||||
sigemptyset(&sig_set);
|
||||
for (auto signal : signals)
|
||||
sigaddset(&sig_set, signal);
|
||||
#else
|
||||
if (sigemptyset(&sig_set))
|
||||
throw Poco::Exception("Cannot block signal.");
|
||||
|
||||
for (auto signal : signals)
|
||||
if (sigaddset(&sig_set, signal))
|
||||
throw Poco::Exception("Cannot block signal.");
|
||||
#endif
|
||||
|
||||
if (pthread_sigmask(SIG_BLOCK, &sig_set, nullptr))
|
||||
throw Poco::Exception("Cannot block signal.");
|
||||
}
|
||||
|
||||
|
||||
void SignalListener::run()
|
||||
{
|
||||
static_assert(PIPE_BUF >= 512);
|
||||
static_assert(signal_pipe_buf_size <= PIPE_BUF, "Only write of PIPE_BUF to pipe is atomic and the minimal known PIPE_BUF across supported platforms is 512");
|
||||
char buf[signal_pipe_buf_size];
|
||||
auto & signal_pipe = HandledSignals::instance().signal_pipe;
|
||||
ReadBufferFromFileDescriptor in(signal_pipe.fds_rw[0], signal_pipe_buf_size, buf);
|
||||
|
||||
while (!in.eof())
|
||||
{
|
||||
int sig = 0;
|
||||
readBinary(sig, in);
|
||||
// We may log some specific signals afterwards, with different log
|
||||
// levels and more info, but for completeness we log all signals
|
||||
// here at trace level.
|
||||
// Don't use strsignal here, because it's not thread-safe.
|
||||
LOG_TRACE(log, "Received signal {}", sig);
|
||||
|
||||
if (sig == StopThread)
|
||||
{
|
||||
LOG_INFO(log, "Stop SignalListener thread");
|
||||
break;
|
||||
}
|
||||
else if (sig == SIGHUP)
|
||||
{
|
||||
LOG_DEBUG(log, "Received signal to close logs.");
|
||||
BaseDaemon::instance().closeLogs(BaseDaemon::instance().logger());
|
||||
LOG_INFO(log, "Opened new log file after received signal.");
|
||||
}
|
||||
else if (sig == StdTerminate)
|
||||
{
|
||||
UInt32 thread_num;
|
||||
std::string message;
|
||||
|
||||
readBinary(thread_num, in);
|
||||
readBinary(message, in);
|
||||
|
||||
onTerminate(message, thread_num);
|
||||
}
|
||||
else if (sig == SIGINT ||
|
||||
sig == SIGQUIT ||
|
||||
sig == SIGTERM)
|
||||
{
|
||||
if (daemon)
|
||||
daemon->handleSignal(sig);
|
||||
}
|
||||
else
|
||||
{
|
||||
siginfo_t info{};
|
||||
ucontext_t * context{};
|
||||
StackTrace stack_trace(NoCapture{});
|
||||
std::vector<StackTrace::FramePointers> thread_frame_pointers;
|
||||
UInt32 thread_num{};
|
||||
ThreadStatus * thread_ptr{};
|
||||
|
||||
if (sig != SanitizerTrap)
|
||||
{
|
||||
readPODBinary(info, in);
|
||||
readPODBinary(context, in);
|
||||
}
|
||||
|
||||
readPODBinary(stack_trace, in);
|
||||
if (sig != SanitizerTrap)
|
||||
readVectorBinary(thread_frame_pointers, in);
|
||||
readBinary(thread_num, in);
|
||||
readPODBinary(thread_ptr, in);
|
||||
|
||||
/// This allows to receive more signals if failure happens inside onFault function.
|
||||
/// Example: segfault while symbolizing stack trace.
|
||||
try
|
||||
{
|
||||
std::thread([=, this] { onFault(sig, info, context, stack_trace, thread_frame_pointers, thread_num, thread_ptr); }).detach();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
/// Likely cannot allocate thread
|
||||
onFault(sig, info, context, stack_trace, thread_frame_pointers, thread_num, thread_ptr);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void SignalListener::onTerminate(std::string_view message, UInt32 thread_num) const
|
||||
{
|
||||
size_t pos = message.find('\n');
|
||||
|
||||
LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) {}",
|
||||
VERSION_STRING, VERSION_OFFICIAL, daemon ? daemon->build_id : "", daemon ? daemon->git_hash : "", thread_num, message.substr(0, pos));
|
||||
|
||||
/// Print trace from std::terminate exception line-by-line to make it easy for grep.
|
||||
while (pos != std::string_view::npos)
|
||||
{
|
||||
++pos;
|
||||
size_t next_pos = message.find('\n', pos);
|
||||
size_t size = next_pos;
|
||||
if (next_pos != std::string_view::npos)
|
||||
size = next_pos - pos;
|
||||
|
||||
LOG_FATAL(log, fmt::runtime(message.substr(pos, size)));
|
||||
pos = next_pos;
|
||||
}
|
||||
}
|
||||
|
||||
void SignalListener::onFault(
|
||||
int sig,
|
||||
const siginfo_t & info,
|
||||
ucontext_t * context,
|
||||
const StackTrace & stack_trace,
|
||||
const std::vector<StackTrace::FramePointers> & thread_frame_pointers,
|
||||
UInt32 thread_num,
|
||||
DB::ThreadStatus * thread_ptr) const
|
||||
try
|
||||
{
|
||||
ThreadStatus thread_status;
|
||||
|
||||
/// First log those fields that are safe to access and that should not cause new fault.
|
||||
/// That way we will have some duplicated info in the log but we don't loose important info
|
||||
/// in case of double fault.
|
||||
|
||||
LOG_FATAL(log, "########## Short fault info ############");
|
||||
LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) Received signal {}",
|
||||
VERSION_STRING, VERSION_OFFICIAL, daemon ? daemon->build_id : "", daemon ? daemon->git_hash : "",
|
||||
thread_num, sig);
|
||||
|
||||
std::string signal_description = "Unknown signal";
|
||||
|
||||
/// Some of these are not really signals, but our own indications on failure reason.
|
||||
if (sig == StdTerminate)
|
||||
signal_description = "std::terminate";
|
||||
else if (sig == SanitizerTrap)
|
||||
signal_description = "sanitizer trap";
|
||||
else if (sig >= 0)
|
||||
signal_description = strsignal(sig); // NOLINT(concurrency-mt-unsafe) // it is not thread-safe but ok in this context
|
||||
|
||||
LOG_FATAL(log, "Signal description: {}", signal_description);
|
||||
|
||||
String error_message;
|
||||
|
||||
if (sig != SanitizerTrap)
|
||||
error_message = signalToErrorMessage(sig, info, *context);
|
||||
else
|
||||
error_message = "Sanitizer trap.";
|
||||
|
||||
LOG_FATAL(log, fmt::runtime(error_message));
|
||||
|
||||
String bare_stacktrace_str;
|
||||
if (stack_trace.getSize())
|
||||
{
|
||||
/// Write bare stack trace (addresses) just in case if we will fail to print symbolized stack trace.
|
||||
/// NOTE: This still require memory allocations and mutex lock inside logger.
|
||||
/// BTW we can also print it to stderr using write syscalls.
|
||||
|
||||
WriteBufferFromOwnString bare_stacktrace;
|
||||
writeString("Stack trace:", bare_stacktrace);
|
||||
for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i)
|
||||
{
|
||||
writeChar(' ', bare_stacktrace);
|
||||
writePointerHex(stack_trace.getFramePointers()[i], bare_stacktrace);
|
||||
}
|
||||
|
||||
LOG_FATAL(log, fmt::runtime(bare_stacktrace.str()));
|
||||
bare_stacktrace_str = bare_stacktrace.str();
|
||||
}
|
||||
|
||||
/// Now try to access potentially unsafe data in thread_ptr.
|
||||
|
||||
String query_id;
|
||||
String query;
|
||||
|
||||
/// Send logs from this thread to client if possible.
|
||||
/// It will allow client to see failure messages directly.
|
||||
if (thread_ptr)
|
||||
{
|
||||
query_id = thread_ptr->getQueryId();
|
||||
query = thread_ptr->getQueryForLog();
|
||||
|
||||
if (auto logs_queue = thread_ptr->getInternalTextLogsQueue())
|
||||
{
|
||||
CurrentThread::attachInternalTextLogsQueue(logs_queue, LogsLevel::trace);
|
||||
}
|
||||
}
|
||||
|
||||
LOG_FATAL(log, "########################################");
|
||||
|
||||
if (query_id.empty())
|
||||
{
|
||||
LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) (no query) Received signal {} ({})",
|
||||
VERSION_STRING, VERSION_OFFICIAL, daemon ? daemon->build_id : "", daemon ? daemon->git_hash : "",
|
||||
thread_num, signal_description, sig);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) (query_id: {}) (query: {}) Received signal {} ({})",
|
||||
VERSION_STRING, VERSION_OFFICIAL, daemon ? daemon->build_id : "", daemon ? daemon->git_hash : "",
|
||||
thread_num, query_id, query, signal_description, sig);
|
||||
}
|
||||
|
||||
LOG_FATAL(log, fmt::runtime(error_message));
|
||||
|
||||
if (!bare_stacktrace_str.empty())
|
||||
{
|
||||
LOG_FATAL(log, fmt::runtime(bare_stacktrace_str));
|
||||
}
|
||||
|
||||
/// Write symbolized stack trace line by line for better grep-ability.
|
||||
stack_trace.toStringEveryLine([&](std::string_view s) { LOG_FATAL(log, fmt::runtime(s)); });
|
||||
|
||||
/// In case it's a scheduled job write all previous jobs origins call stacks
|
||||
std::for_each(thread_frame_pointers.rbegin(), thread_frame_pointers.rend(),
|
||||
[this](const StackTrace::FramePointers & frame_pointers)
|
||||
{
|
||||
if (size_t size = std::ranges::find(frame_pointers, nullptr) - frame_pointers.begin())
|
||||
{
|
||||
LOG_FATAL(log, "========================================");
|
||||
WriteBufferFromOwnString bare_stacktrace;
|
||||
writeString("Job's origin stack trace:", bare_stacktrace);
|
||||
std::for_each_n(frame_pointers.begin(), size,
|
||||
[&bare_stacktrace](const void * ptr)
|
||||
{
|
||||
writeChar(' ', bare_stacktrace);
|
||||
writePointerHex(ptr, bare_stacktrace);
|
||||
}
|
||||
);
|
||||
|
||||
LOG_FATAL(log, fmt::runtime(bare_stacktrace.str()));
|
||||
|
||||
StackTrace::toStringEveryLine(const_cast<void **>(frame_pointers.data()), 0, size, [this](std::string_view s) { LOG_FATAL(log, fmt::runtime(s)); });
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
#if defined(OS_LINUX)
|
||||
/// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace.
|
||||
/// Please keep the below log messages in-sync with the ones in programs/server/Server.cpp
|
||||
|
||||
if (daemon && daemon->stored_binary_hash.empty())
|
||||
{
|
||||
LOG_FATAL(log, "Integrity check of the executable skipped because the reference checksum could not be read.");
|
||||
}
|
||||
else if (daemon)
|
||||
{
|
||||
String calculated_binary_hash = getHashOfLoadedBinaryHex();
|
||||
if (calculated_binary_hash == daemon->stored_binary_hash)
|
||||
{
|
||||
LOG_FATAL(log, "Integrity check of the executable successfully passed (checksum: {})", calculated_binary_hash);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_FATAL(
|
||||
log,
|
||||
"Calculated checksum of the executable ({0}) does not correspond"
|
||||
" to the reference checksum stored in the executable ({1})."
|
||||
" This may indicate one of the following:"
|
||||
" - the executable was changed just after startup;"
|
||||
" - the executable was corrupted on disk due to faulty hardware;"
|
||||
" - the loaded executable was corrupted in memory due to faulty hardware;"
|
||||
" - the file was intentionally modified;"
|
||||
" - a logical error in the code.",
|
||||
calculated_binary_hash,
|
||||
daemon->stored_binary_hash);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/// Write crash to system.crash_log table if available.
|
||||
if (collectCrashLog)
|
||||
collectCrashLog(sig, thread_num, query_id, stack_trace);
|
||||
|
||||
Context::getGlobalContextInstance()->handleCrash();
|
||||
|
||||
/// Send crash report to developers (if configured)
|
||||
if (sig != SanitizerTrap)
|
||||
{
|
||||
if (daemon)
|
||||
{
|
||||
if (auto * sentry = SentryWriter::getInstance())
|
||||
sentry->onSignal(sig, error_message, stack_trace.getFramePointers(), stack_trace.getOffset(), stack_trace.getSize());
|
||||
}
|
||||
|
||||
/// Advice the user to send it manually.
|
||||
if (std::string_view(VERSION_OFFICIAL).contains("official build"))
|
||||
{
|
||||
const auto & date_lut = DateLUT::instance();
|
||||
|
||||
/// Approximate support period, upper bound.
|
||||
if (time(nullptr) - date_lut.makeDate(2000 + VERSION_MAJOR, VERSION_MINOR, 1) < (365 + 30) * 86400)
|
||||
{
|
||||
LOG_FATAL(log, "Report this error to https://github.com/ClickHouse/ClickHouse/issues");
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_FATAL(log, "ClickHouse version {} is old and should be upgraded to the latest version.", VERSION_STRING);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_FATAL(log, "This ClickHouse version is not official and should be upgraded to the official build.");
|
||||
}
|
||||
}
|
||||
|
||||
/// List changed settings.
|
||||
if (!query_id.empty())
|
||||
{
|
||||
ContextPtr query_context = thread_ptr->getQueryContext();
|
||||
if (query_context)
|
||||
{
|
||||
String changed_settings = query_context->getSettingsRef().toString();
|
||||
|
||||
if (changed_settings.empty())
|
||||
LOG_FATAL(log, "No settings were changed");
|
||||
else
|
||||
LOG_FATAL(log, "Changed settings: {}", changed_settings);
|
||||
}
|
||||
}
|
||||
|
||||
/// When everything is done, we will try to send these error messages to the client.
|
||||
if (thread_ptr)
|
||||
thread_ptr->onFatalError();
|
||||
|
||||
HandledSignals::instance().fatal_error_printed.test_and_set();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
/// onFault is called from the std::thread, and it should catch all exceptions; otherwise, you can get unrelated fatal errors.
|
||||
PreformattedMessage message = getCurrentExceptionMessageAndPattern(true);
|
||||
LOG_FATAL(log, message);
|
||||
}
|
||||
|
||||
HandledSignals::HandledSignals()
|
||||
{
|
||||
signal_pipe.setNonBlockingWrite();
|
||||
signal_pipe.tryIncreaseSize(1 << 20);
|
||||
}
|
||||
|
||||
void HandledSignals::reset()
|
||||
{
|
||||
/// Reset signals to SIG_DFL to avoid trying to write to the signal_pipe that will be closed after.
|
||||
for (int sig : handled_signals)
|
||||
{
|
||||
if (SIG_ERR == signal(sig, SIG_DFL))
|
||||
{
|
||||
try
|
||||
{
|
||||
throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
|
||||
}
|
||||
catch (ErrnoException &)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
signal_pipe.close();
|
||||
}
|
||||
|
||||
HandledSignals::~HandledSignals()
|
||||
{
|
||||
reset();
|
||||
};
|
||||
|
||||
HandledSignals & HandledSignals::instance()
|
||||
{
|
||||
static HandledSignals res;
|
||||
return res;
|
||||
}
|
||||
|
||||
void HandledSignals::setupTerminateHandler()
|
||||
{
|
||||
std::set_terminate(terminate_handler);
|
||||
}
|
||||
|
||||
void HandledSignals::setupCommonDeadlySignalHandlers()
|
||||
{
|
||||
/// SIGTSTP is added for debugging purposes. To output a stack trace of any running thread at anytime.
|
||||
addSignalHandler({SIGABRT, SIGSEGV, SIGILL, SIGBUS, SIGSYS, SIGFPE, SIGPIPE, SIGTSTP, SIGTRAP}, signalHandler, true);
|
||||
|
||||
#if defined(SANITIZER)
|
||||
__sanitizer_set_death_callback(sanitizerDeathCallback);
|
||||
#endif
|
||||
}
|
||||
|
||||
void HandledSignals::setupCommonTerminateRequestSignalHandlers()
|
||||
{
|
||||
addSignalHandler({SIGINT, SIGQUIT, SIGTERM}, terminateRequestedSignalHandler, true);
|
||||
}
|
||||
110
src/Common/SignalHandlers.h
Normal file
110
src/Common/SignalHandlers.h
Normal file
@ -0,0 +1,110 @@
|
||||
#pragma once
|
||||
#include <csignal>
|
||||
|
||||
#include <base/defines.h>
|
||||
#include <Common/PipeFDs.h>
|
||||
#include <Common/StackTrace.h>
|
||||
#include <Common/ThreadStatus.h>
|
||||
#include <Core/Types.h>
|
||||
#include <Poco/Runnable.h>
|
||||
|
||||
class BaseDaemon;
|
||||
|
||||
/** Reset signal handler to the default and send signal to itself.
|
||||
* It's called from user signal handler to write core dump.
|
||||
*/
|
||||
void call_default_signal_handler(int sig);
|
||||
|
||||
const size_t signal_pipe_buf_size =
|
||||
sizeof(int)
|
||||
+ sizeof(siginfo_t)
|
||||
+ sizeof(ucontext_t*)
|
||||
+ sizeof(StackTrace)
|
||||
+ sizeof(UInt64)
|
||||
+ sizeof(UInt32)
|
||||
+ sizeof(void*);
|
||||
|
||||
using signal_function = void(int, siginfo_t*, void*);
|
||||
|
||||
void writeSignalIDtoSignalPipe(int sig);
|
||||
|
||||
/** Signal handler for HUP */
|
||||
void closeLogsSignalHandler(int sig, siginfo_t *, void *);
|
||||
|
||||
void terminateRequestedSignalHandler(int sig, siginfo_t *, void *);
|
||||
|
||||
|
||||
/** Handler for "fault" or diagnostic signals. Send data about fault to separate thread to write into log.
|
||||
*/
|
||||
void signalHandler(int sig, siginfo_t * info, void * context);
|
||||
|
||||
|
||||
/** To use with std::set_terminate.
|
||||
* Collects slightly more info than __gnu_cxx::__verbose_terminate_handler,
|
||||
* and send it to pipe. Other thread will read this info from pipe and asynchronously write it to log.
|
||||
* Look at libstdc++-v3/libsupc++/vterminate.cc for example.
|
||||
*/
|
||||
[[noreturn]] void terminate_handler();
|
||||
|
||||
/// Avoid link time dependency on DB/Interpreters - will use this function only when linked.
|
||||
__attribute__((__weak__)) void collectCrashLog(
|
||||
Int32 signal, UInt64 thread_id, const String & query_id, const StackTrace & stack_trace);
|
||||
|
||||
|
||||
void blockSignals(const std::vector<int> & signals);
|
||||
|
||||
|
||||
/** The thread that read info about signal or std::terminate from pipe.
|
||||
* On HUP, close log files (for new files to be opened later).
|
||||
* On information about std::terminate, write it to log.
|
||||
* On other signals, write info to log.
|
||||
*/
|
||||
class SignalListener : public Poco::Runnable
|
||||
{
|
||||
public:
|
||||
static constexpr int StdTerminate = -1;
|
||||
static constexpr int StopThread = -2;
|
||||
static constexpr int SanitizerTrap = -3;
|
||||
|
||||
explicit SignalListener(BaseDaemon * daemon_, LoggerPtr log_)
|
||||
: daemon(daemon_), log(log_)
|
||||
{
|
||||
}
|
||||
|
||||
void run() override;
|
||||
|
||||
private:
|
||||
BaseDaemon * daemon;
|
||||
LoggerPtr log;
|
||||
|
||||
void onTerminate(std::string_view message, UInt32 thread_num) const;
|
||||
|
||||
void onFault(
|
||||
int sig,
|
||||
const siginfo_t & info,
|
||||
ucontext_t * context,
|
||||
const StackTrace & stack_trace,
|
||||
const std::vector<StackTrace::FramePointers> & thread_frame_pointers,
|
||||
UInt32 thread_num,
|
||||
DB::ThreadStatus * thread_ptr) const;
|
||||
};
|
||||
|
||||
struct HandledSignals
|
||||
{
|
||||
std::vector<int> handled_signals;
|
||||
DB::PipeFDs signal_pipe;
|
||||
std::atomic_flag fatal_error_printed;
|
||||
|
||||
HandledSignals();
|
||||
~HandledSignals();
|
||||
|
||||
void setupTerminateHandler();
|
||||
void setupCommonDeadlySignalHandlers();
|
||||
void setupCommonTerminateRequestSignalHandlers();
|
||||
|
||||
void addSignalHandler(const std::vector<int> & signals, signal_function handler, bool register_signal);
|
||||
|
||||
void reset();
|
||||
|
||||
static HandledSignals & instance();
|
||||
};
|
||||
@ -36,7 +36,7 @@ class IColumn;
|
||||
M(Dialect, dialect, Dialect::clickhouse, "Which dialect will be used to parse query", 0)\
|
||||
M(UInt64, min_compress_block_size, 65536, "The actual size of the block to compress, if the uncompressed data less than max_compress_block_size is no less than this value and no less than the volume of data for one mark.", 0) \
|
||||
M(UInt64, max_compress_block_size, 1048576, "The maximum size of blocks of uncompressed data before compressing for writing to a table.", 0) \
|
||||
M(UInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size for reading", 0) \
|
||||
M(UInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size in rows for reading", 0) \
|
||||
M(UInt64, max_insert_block_size, DEFAULT_INSERT_BLOCK_SIZE, "The maximum block size for insertion, if we control the creation of blocks for insertion.", 0) \
|
||||
M(UInt64, min_insert_block_size_rows, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough.", 0) \
|
||||
M(UInt64, min_insert_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough.", 0) \
|
||||
@ -152,7 +152,7 @@ class IColumn;
|
||||
M(UInt64, max_local_write_bandwidth, 0, "The maximum speed of local writes in bytes per second.", 0) \
|
||||
M(Bool, stream_like_engine_allow_direct_select, false, "Allow direct SELECT query for Kafka, RabbitMQ, FileLog, Redis Streams, and NATS engines. In case there are attached materialized views, SELECT query is not allowed even if this setting is enabled.", 0) \
|
||||
M(String, stream_like_engine_insert_queue, "", "When stream like engine reads from multiple queues, user will need to select one queue to insert into when writing. Used by Redis Streams and NATS.", 0) \
|
||||
\
|
||||
M(Bool, dictionary_validate_primary_key_type, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64.", 0) \
|
||||
M(Bool, distributed_insert_skip_read_only_replicas, false, "If true, INSERT into Distributed will skip read-only replicas.", 0) \
|
||||
M(Bool, distributed_foreground_insert, false, "If setting is enabled, insert query into distributed waits until data are sent to all nodes in a cluster. \n\nEnables or disables synchronous data insertion into a `Distributed` table.\n\nBy default, when inserting data into a Distributed table, the ClickHouse server sends data to cluster nodes in the background. When `distributed_foreground_insert` = 1, the data is processed synchronously, and the `INSERT` operation succeeds only after all the data is saved on all shards (at least one replica for each shard if `internal_replication` is true).", 0) ALIAS(insert_distributed_sync) \
|
||||
M(UInt64, distributed_background_insert_timeout, 0, "Timeout for insert query into distributed. Setting is used only with insert_distributed_sync enabled. Zero value means no timeout.", 0) ALIAS(insert_distributed_timeout) \
|
||||
@ -610,9 +610,8 @@ class IColumn;
|
||||
M(Bool, optimize_time_filter_with_preimage, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')", 0) \
|
||||
M(Bool, normalize_function_names, true, "Normalize function names to their canonical names", 0) \
|
||||
M(Bool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there are constants there", 0) \
|
||||
M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views if the block is not a duplicate for the table. Use true to always deduplicate in dependent tables.", 0) \
|
||||
M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views. Use true to always deduplicate in dependent tables.", 0) \
|
||||
M(Bool, throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert, true, "Throw exception on INSERT query when the setting `deduplicate_blocks_in_dependent_materialized_views` is enabled along with `async_insert`. It guarantees correctness, because these features can't work together.", 0) \
|
||||
M(Bool, update_insert_deduplication_token_in_dependent_materialized_views, false, "Should update insert deduplication token with table identifier during insert in dependent materialized views.", 0) \
|
||||
M(Bool, materialized_views_ignore_errors, false, "Allows to ignore errors for MATERIALIZED VIEW, and deliver original block to the table regardless of MVs", 0) \
|
||||
M(Bool, ignore_materialized_views_with_dropped_target_table, false, "Ignore MVs with dropped target table during pushing to views", 0) \
|
||||
M(Bool, use_compact_format_in_distributed_parts_names, true, "Changes format of directories names for distributed table insert parts.", 0) \
|
||||
@ -978,6 +977,7 @@ class IColumn;
|
||||
|
||||
#define OBSOLETE_SETTINGS(M, ALIAS) \
|
||||
/** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
|
||||
MAKE_OBSOLETE(M, Bool, update_insert_deduplication_token_in_dependent_materialized_views, 0) \
|
||||
MAKE_OBSOLETE(M, UInt64, max_memory_usage_for_all_queries, 0) \
|
||||
MAKE_OBSOLETE(M, UInt64, multiple_joins_rewriter_version, 0) \
|
||||
MAKE_OBSOLETE(M, Bool, enable_debug_queries, false) \
|
||||
|
||||
@ -67,6 +67,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
|
||||
{"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
|
||||
{"input_format_json_ignore_key_case", false, false, "Ignore json key case while read json field from string."},
|
||||
{"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
|
||||
{"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
|
||||
{"collect_hash_table_stats_during_joins", false, true, "New setting."},
|
||||
{"max_size_to_preallocate_for_joins", 0, 100'000'000, "New setting."},
|
||||
{"input_format_orc_read_use_writer_time_zone", false, false, "Whether use the writer's time zone in ORC stripe for ORC row reader, the default ORC row reader's time zone is GMT."},
|
||||
|
||||
@ -20,7 +20,6 @@
|
||||
#endif
|
||||
#include <cerrno>
|
||||
#include <cstring>
|
||||
#include <csignal>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <algorithm>
|
||||
@ -37,6 +36,7 @@
|
||||
#include <Poco/Pipe.h>
|
||||
|
||||
#include <Common/ErrorHandlers.h>
|
||||
#include <Common/SignalHandlers.h>
|
||||
#include <base/argsToConfig.h>
|
||||
#include <base/getThreadId.h>
|
||||
#include <base/coverage.h>
|
||||
@ -54,7 +54,6 @@
|
||||
#include <Common/Config/ConfigProcessor.h>
|
||||
#include <Common/SymbolIndex.h>
|
||||
#include <Common/getExecutablePath.h>
|
||||
#include <Common/getHashOfLoadedBinary.h>
|
||||
#include <Common/Elf.h>
|
||||
#include <Common/setThreadName.h>
|
||||
#include <Common/logger_useful.h>
|
||||
@ -79,8 +78,6 @@ namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_SET_SIGNAL_HANDLER;
|
||||
extern const int CANNOT_SEND_SIGNAL;
|
||||
extern const int SYSTEM_ERROR;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
@ -88,114 +85,6 @@ namespace DB
|
||||
|
||||
using namespace DB;
|
||||
|
||||
PipeFDs signal_pipe;
|
||||
|
||||
|
||||
/** Reset signal handler to the default and send signal to itself.
|
||||
* It's called from user signal handler to write core dump.
|
||||
*/
|
||||
static void call_default_signal_handler(int sig)
|
||||
{
|
||||
if (SIG_ERR == signal(sig, SIG_DFL))
|
||||
throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
|
||||
|
||||
if (0 != raise(sig))
|
||||
throw ErrnoException(ErrorCodes::CANNOT_SEND_SIGNAL, "Cannot send signal");
|
||||
}
|
||||
|
||||
static const size_t signal_pipe_buf_size =
|
||||
sizeof(int)
|
||||
+ sizeof(siginfo_t)
|
||||
+ sizeof(ucontext_t*)
|
||||
+ sizeof(StackTrace)
|
||||
+ sizeof(UInt64)
|
||||
+ sizeof(UInt32)
|
||||
+ sizeof(void*);
|
||||
|
||||
using signal_function = void(int, siginfo_t*, void*);
|
||||
|
||||
static void writeSignalIDtoSignalPipe(int sig)
|
||||
{
|
||||
auto saved_errno = errno; /// We must restore previous value of errno in signal handler.
|
||||
|
||||
char buf[signal_pipe_buf_size];
|
||||
WriteBufferFromFileDescriptor out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
|
||||
writeBinary(sig, out);
|
||||
out.next();
|
||||
|
||||
errno = saved_errno;
|
||||
}
|
||||
|
||||
/** Signal handler for HUP */
|
||||
static void closeLogsSignalHandler(int sig, siginfo_t *, void *)
|
||||
{
|
||||
DENY_ALLOCATIONS_IN_SCOPE;
|
||||
writeSignalIDtoSignalPipe(sig);
|
||||
}
|
||||
|
||||
static void terminateRequestedSignalHandler(int sig, siginfo_t *, void *)
|
||||
{
|
||||
DENY_ALLOCATIONS_IN_SCOPE;
|
||||
writeSignalIDtoSignalPipe(sig);
|
||||
}
|
||||
|
||||
|
||||
static std::atomic_flag fatal_error_printed;
|
||||
|
||||
/** Handler for "fault" or diagnostic signals. Send data about fault to separate thread to write into log.
|
||||
*/
|
||||
static void signalHandler(int sig, siginfo_t * info, void * context)
|
||||
{
|
||||
if (asynchronous_stack_unwinding && sig == SIGSEGV)
|
||||
siglongjmp(asynchronous_stack_unwinding_signal_jump_buffer, 1);
|
||||
|
||||
DENY_ALLOCATIONS_IN_SCOPE;
|
||||
auto saved_errno = errno; /// We must restore previous value of errno in signal handler.
|
||||
|
||||
char buf[signal_pipe_buf_size];
|
||||
WriteBufferFromFileDescriptorDiscardOnFailure out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
|
||||
|
||||
const ucontext_t * signal_context = reinterpret_cast<ucontext_t *>(context);
|
||||
const StackTrace stack_trace(*signal_context);
|
||||
|
||||
#if USE_GWP_ASAN
|
||||
if (const auto fault_address = reinterpret_cast<uintptr_t>(info->si_addr);
|
||||
GWPAsan::isGWPAsanError(fault_address))
|
||||
GWPAsan::printReport(fault_address);
|
||||
#endif
|
||||
|
||||
writeBinary(sig, out);
|
||||
writePODBinary(*info, out);
|
||||
writePODBinary(signal_context, out);
|
||||
writePODBinary(stack_trace, out);
|
||||
writeVectorBinary(Exception::enable_job_stack_trace ? Exception::thread_frame_pointers : std::vector<StackTrace::FramePointers>{}, out);
|
||||
writeBinary(static_cast<UInt32>(getThreadId()), out);
|
||||
writePODBinary(current_thread, out);
|
||||
|
||||
out.next();
|
||||
|
||||
if (sig != SIGTSTP) /// This signal is used for debugging.
|
||||
{
|
||||
/// The time that is usually enough for separate thread to print info into log.
|
||||
/// Under MSan full stack unwinding with DWARF info about inline functions takes 101 seconds in one case.
|
||||
for (size_t i = 0; i < 300; ++i)
|
||||
{
|
||||
/// We will synchronize with the thread printing the messages with an atomic variable to finish earlier.
|
||||
if (fatal_error_printed.test())
|
||||
break;
|
||||
|
||||
/// This coarse method of synchronization is perfectly ok for fatal signals.
|
||||
sleepForSeconds(1);
|
||||
}
|
||||
|
||||
/// Wait for all logs flush operations
|
||||
sleepForSeconds(3);
|
||||
call_default_signal_handler(sig);
|
||||
}
|
||||
|
||||
errno = saved_errno;
|
||||
}
|
||||
|
||||
|
||||
static bool getenvBool(const char * name)
|
||||
{
|
||||
@ -207,445 +96,6 @@ static bool getenvBool(const char * name)
|
||||
}
|
||||
|
||||
|
||||
/// Avoid link time dependency on DB/Interpreters - will use this function only when linked.
|
||||
__attribute__((__weak__)) void collectCrashLog(
|
||||
Int32 signal, UInt64 thread_id, const String & query_id, const StackTrace & stack_trace);
|
||||
|
||||
|
||||
/** The thread that read info about signal or std::terminate from pipe.
|
||||
* On HUP, close log files (for new files to be opened later).
|
||||
* On information about std::terminate, write it to log.
|
||||
* On other signals, write info to log.
|
||||
*/
|
||||
class SignalListener : public Poco::Runnable
|
||||
{
|
||||
public:
|
||||
static constexpr int StdTerminate = -1;
|
||||
static constexpr int StopThread = -2;
|
||||
static constexpr int SanitizerTrap = -3;
|
||||
|
||||
explicit SignalListener(BaseDaemon & daemon_)
|
||||
: log(getLogger("BaseDaemon"))
|
||||
, daemon(daemon_)
|
||||
{
|
||||
}
|
||||
|
||||
void run() override
|
||||
{
|
||||
static_assert(PIPE_BUF >= 512);
|
||||
static_assert(signal_pipe_buf_size <= PIPE_BUF, "Only write of PIPE_BUF to pipe is atomic and the minimal known PIPE_BUF across supported platforms is 512");
|
||||
char buf[signal_pipe_buf_size];
|
||||
ReadBufferFromFileDescriptor in(signal_pipe.fds_rw[0], signal_pipe_buf_size, buf);
|
||||
|
||||
while (!in.eof())
|
||||
{
|
||||
int sig = 0;
|
||||
readBinary(sig, in);
|
||||
// We may log some specific signals afterwards, with different log
|
||||
// levels and more info, but for completeness we log all signals
|
||||
// here at trace level.
|
||||
// Don't use strsignal here, because it's not thread-safe.
|
||||
LOG_TRACE(log, "Received signal {}", sig);
|
||||
|
||||
if (sig == StopThread)
|
||||
{
|
||||
LOG_INFO(log, "Stop SignalListener thread");
|
||||
break;
|
||||
}
|
||||
else if (sig == SIGHUP)
|
||||
{
|
||||
LOG_DEBUG(log, "Received signal to close logs.");
|
||||
BaseDaemon::instance().closeLogs(BaseDaemon::instance().logger());
|
||||
LOG_INFO(log, "Opened new log file after received signal.");
|
||||
}
|
||||
else if (sig == StdTerminate)
|
||||
{
|
||||
UInt32 thread_num;
|
||||
std::string message;
|
||||
|
||||
readBinary(thread_num, in);
|
||||
readBinary(message, in);
|
||||
|
||||
onTerminate(message, thread_num);
|
||||
}
|
||||
else if (sig == SIGINT ||
|
||||
sig == SIGQUIT ||
|
||||
sig == SIGTERM)
|
||||
{
|
||||
daemon.handleSignal(sig);
|
||||
}
|
||||
else
|
||||
{
|
||||
siginfo_t info{};
|
||||
ucontext_t * context{};
|
||||
StackTrace stack_trace(NoCapture{});
|
||||
std::vector<StackTrace::FramePointers> thread_frame_pointers;
|
||||
UInt32 thread_num{};
|
||||
ThreadStatus * thread_ptr{};
|
||||
|
||||
if (sig != SanitizerTrap)
|
||||
{
|
||||
readPODBinary(info, in);
|
||||
readPODBinary(context, in);
|
||||
}
|
||||
|
||||
readPODBinary(stack_trace, in);
|
||||
|
||||
if (sig != SanitizerTrap)
|
||||
readVectorBinary(thread_frame_pointers, in);
|
||||
|
||||
readBinary(thread_num, in);
|
||||
readPODBinary(thread_ptr, in);
|
||||
|
||||
/// This allows to receive more signals if failure happens inside onFault function.
|
||||
/// Example: segfault while symbolizing stack trace.
|
||||
try
|
||||
{
|
||||
std::thread([=, this] { onFault(sig, info, context, stack_trace, thread_frame_pointers, thread_num, thread_ptr); }).detach();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
/// Likely cannot allocate thread
|
||||
onFault(sig, info, context, stack_trace, thread_frame_pointers, thread_num, thread_ptr);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
LoggerPtr log;
|
||||
BaseDaemon & daemon;
|
||||
|
||||
void onTerminate(std::string_view message, UInt32 thread_num) const
|
||||
{
|
||||
size_t pos = message.find('\n');
|
||||
|
||||
LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) {}",
|
||||
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id, daemon.git_hash, thread_num, message.substr(0, pos));
|
||||
|
||||
/// Print trace from std::terminate exception line-by-line to make it easy for grep.
|
||||
while (pos != std::string_view::npos)
|
||||
{
|
||||
++pos;
|
||||
size_t next_pos = message.find('\n', pos);
|
||||
size_t size = next_pos;
|
||||
if (next_pos != std::string_view::npos)
|
||||
size = next_pos - pos;
|
||||
|
||||
LOG_FATAL(log, fmt::runtime(message.substr(pos, size)));
|
||||
pos = next_pos;
|
||||
}
|
||||
}
|
||||
|
||||
void onFault(
|
||||
int sig,
|
||||
const siginfo_t & info,
|
||||
ucontext_t * context,
|
||||
const StackTrace & stack_trace,
|
||||
const std::vector<StackTrace::FramePointers> & thread_frame_pointers,
|
||||
UInt32 thread_num,
|
||||
ThreadStatus * thread_ptr) const
|
||||
try
|
||||
{
|
||||
ThreadStatus thread_status;
|
||||
|
||||
/// First log those fields that are safe to access and that should not cause new fault.
|
||||
/// That way we will have some duplicated info in the log but we don't loose important info
|
||||
/// in case of double fault.
|
||||
|
||||
LOG_FATAL(log, "########## Short fault info ############");
|
||||
LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) Received signal {}",
|
||||
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id, daemon.git_hash,
|
||||
thread_num, sig);
|
||||
|
||||
std::string signal_description = "Unknown signal";
|
||||
|
||||
/// Some of these are not really signals, but our own indications on failure reason.
|
||||
if (sig == StdTerminate)
|
||||
signal_description = "std::terminate";
|
||||
else if (sig == SanitizerTrap)
|
||||
signal_description = "sanitizer trap";
|
||||
else if (sig >= 0)
|
||||
signal_description = strsignal(sig); // NOLINT(concurrency-mt-unsafe) // it is not thread-safe but ok in this context
|
||||
|
||||
LOG_FATAL(log, "Signal description: {}", signal_description);
|
||||
|
||||
String error_message;
|
||||
|
||||
if (sig != SanitizerTrap)
|
||||
error_message = signalToErrorMessage(sig, info, *context);
|
||||
else
|
||||
error_message = "Sanitizer trap.";
|
||||
|
||||
LOG_FATAL(log, fmt::runtime(error_message));
|
||||
|
||||
String bare_stacktrace_str;
|
||||
if (stack_trace.getSize())
|
||||
{
|
||||
/// Write bare stack trace (addresses) just in case if we will fail to print symbolized stack trace.
|
||||
/// NOTE: This still require memory allocations and mutex lock inside logger.
|
||||
/// BTW we can also print it to stderr using write syscalls.
|
||||
|
||||
WriteBufferFromOwnString bare_stacktrace;
|
||||
writeString("Stack trace:", bare_stacktrace);
|
||||
for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i)
|
||||
{
|
||||
writeChar(' ', bare_stacktrace);
|
||||
writePointerHex(stack_trace.getFramePointers()[i], bare_stacktrace);
|
||||
}
|
||||
|
||||
LOG_FATAL(log, fmt::runtime(bare_stacktrace.str()));
|
||||
bare_stacktrace_str = bare_stacktrace.str();
|
||||
}
|
||||
|
||||
/// Now try to access potentially unsafe data in thread_ptr.
|
||||
|
||||
String query_id;
|
||||
String query;
|
||||
|
||||
/// Send logs from this thread to client if possible.
|
||||
/// It will allow client to see failure messages directly.
|
||||
if (thread_ptr)
|
||||
{
|
||||
query_id = thread_ptr->getQueryId();
|
||||
query = thread_ptr->getQueryForLog();
|
||||
|
||||
if (auto logs_queue = thread_ptr->getInternalTextLogsQueue())
|
||||
{
|
||||
CurrentThread::attachInternalTextLogsQueue(logs_queue, LogsLevel::trace);
|
||||
}
|
||||
}
|
||||
|
||||
LOG_FATAL(log, "########################################");
|
||||
|
||||
if (query_id.empty())
|
||||
{
|
||||
LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) (no query) Received signal {} ({})",
|
||||
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id, daemon.git_hash,
|
||||
thread_num, signal_description, sig);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) (query_id: {}) (query: {}) Received signal {} ({})",
|
||||
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id, daemon.git_hash,
|
||||
thread_num, query_id, query, signal_description, sig);
|
||||
}
|
||||
|
||||
LOG_FATAL(log, fmt::runtime(error_message));
|
||||
|
||||
if (!bare_stacktrace_str.empty())
|
||||
{
|
||||
LOG_FATAL(log, fmt::runtime(bare_stacktrace_str));
|
||||
}
|
||||
|
||||
/// Write symbolized stack trace line by line for better grep-ability.
|
||||
stack_trace.toStringEveryLine([&](std::string_view s) { LOG_FATAL(log, fmt::runtime(s)); });
|
||||
|
||||
/// In case it's a scheduled job write all previous jobs origins call stacks
|
||||
std::for_each(thread_frame_pointers.rbegin(), thread_frame_pointers.rend(),
|
||||
[this](const StackTrace::FramePointers & frame_pointers)
|
||||
{
|
||||
if (size_t size = std::ranges::find(frame_pointers, nullptr) - frame_pointers.begin())
|
||||
{
|
||||
LOG_FATAL(log, "========================================");
|
||||
WriteBufferFromOwnString bare_stacktrace;
|
||||
writeString("Job's origin stack trace:", bare_stacktrace);
|
||||
std::for_each_n(frame_pointers.begin(), size,
|
||||
[&bare_stacktrace](const void * ptr)
|
||||
{
|
||||
writeChar(' ', bare_stacktrace);
|
||||
writePointerHex(ptr, bare_stacktrace);
|
||||
}
|
||||
);
|
||||
|
||||
LOG_FATAL(log, fmt::runtime(bare_stacktrace.str()));
|
||||
|
||||
StackTrace::toStringEveryLine(const_cast<void **>(frame_pointers.data()), 0, size, [this](std::string_view s) { LOG_FATAL(log, fmt::runtime(s)); });
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
#if defined(OS_LINUX)
|
||||
/// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace.
|
||||
/// Please keep the below log messages in-sync with the ones in programs/server/Server.cpp
|
||||
|
||||
if (daemon.stored_binary_hash.empty())
|
||||
{
|
||||
LOG_FATAL(log, "Integrity check of the executable skipped because the reference checksum could not be read.");
|
||||
}
|
||||
else
|
||||
{
|
||||
String calculated_binary_hash = getHashOfLoadedBinaryHex();
|
||||
if (calculated_binary_hash == daemon.stored_binary_hash)
|
||||
{
|
||||
LOG_FATAL(log, "Integrity check of the executable successfully passed (checksum: {})", calculated_binary_hash);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_FATAL(
|
||||
log,
|
||||
"Calculated checksum of the executable ({0}) does not correspond"
|
||||
" to the reference checksum stored in the executable ({1})."
|
||||
" This may indicate one of the following:"
|
||||
" - the executable was changed just after startup;"
|
||||
" - the executable was corrupted on disk due to faulty hardware;"
|
||||
" - the loaded executable was corrupted in memory due to faulty hardware;"
|
||||
" - the file was intentionally modified;"
|
||||
" - a logical error in the code.",
|
||||
calculated_binary_hash,
|
||||
daemon.stored_binary_hash);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/// Write crash to system.crash_log table if available.
|
||||
if (collectCrashLog)
|
||||
collectCrashLog(sig, thread_num, query_id, stack_trace);
|
||||
|
||||
Context::getGlobalContextInstance()->handleCrash();
|
||||
|
||||
/// Send crash report to developers (if configured)
|
||||
if (sig != SanitizerTrap)
|
||||
{
|
||||
if (auto * sentry = SentryWriter::getInstance())
|
||||
sentry->onSignal(sig, error_message, stack_trace.getFramePointers(), stack_trace.getOffset(), stack_trace.getSize());
|
||||
|
||||
/// Advice the user to send it manually.
|
||||
if (std::string_view(VERSION_OFFICIAL).contains("official build"))
|
||||
{
|
||||
const auto & date_lut = DateLUT::instance();
|
||||
|
||||
/// Approximate support period, upper bound.
|
||||
if (time(nullptr) - date_lut.makeDate(2000 + VERSION_MAJOR, VERSION_MINOR, 1) < (365 + 30) * 86400)
|
||||
{
|
||||
LOG_FATAL(log, "Report this error to https://github.com/ClickHouse/ClickHouse/issues");
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_FATAL(log, "ClickHouse version {} is old and should be upgraded to the latest version.", VERSION_STRING);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_FATAL(log, "This ClickHouse version is not official and should be upgraded to the official build.");
|
||||
}
|
||||
}
|
||||
|
||||
/// List changed settings.
|
||||
if (!query_id.empty())
|
||||
{
|
||||
ContextPtr query_context = thread_ptr->getQueryContext();
|
||||
if (query_context)
|
||||
{
|
||||
String changed_settings = query_context->getSettingsRef().toString();
|
||||
|
||||
if (changed_settings.empty())
|
||||
LOG_FATAL(log, "No settings were changed");
|
||||
else
|
||||
LOG_FATAL(log, "Changed settings: {}", changed_settings);
|
||||
}
|
||||
}
|
||||
|
||||
/// When everything is done, we will try to send these error messages to the client.
|
||||
if (thread_ptr)
|
||||
thread_ptr->onFatalError();
|
||||
|
||||
fatal_error_printed.test_and_set();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
/// onFault is called from the std::thread, and it should catch all exceptions; otherwise, you can get unrelated fatal errors.
|
||||
PreformattedMessage message = getCurrentExceptionMessageAndPattern(true);
|
||||
LOG_FATAL(getLogger(__PRETTY_FUNCTION__), message);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
#if defined(SANITIZER)
|
||||
|
||||
template <typename T>
|
||||
struct ValueHolder
|
||||
{
|
||||
ValueHolder(T value_) : value(value_)
|
||||
{}
|
||||
|
||||
T value;
|
||||
};
|
||||
|
||||
extern "C" void __sanitizer_set_death_callback(void (*)());
|
||||
|
||||
/// Sanitizers may not expect some function calls from death callback.
|
||||
/// Let's try to disable instrumentation to avoid possible issues.
|
||||
/// However, this callback may call other functions that are still instrumented.
|
||||
/// We can try [[clang::always_inline]] attribute for statements in future (available in clang-15)
|
||||
/// See https://github.com/google/sanitizers/issues/1543 and https://github.com/google/sanitizers/issues/1549.
|
||||
static DISABLE_SANITIZER_INSTRUMENTATION void sanitizerDeathCallback()
|
||||
{
|
||||
DENY_ALLOCATIONS_IN_SCOPE;
|
||||
/// Also need to send data via pipe. Otherwise it may lead to deadlocks or failures in printing diagnostic info.
|
||||
|
||||
char buf[signal_pipe_buf_size];
|
||||
WriteBufferFromFileDescriptorDiscardOnFailure out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
|
||||
|
||||
const StackTrace stack_trace;
|
||||
|
||||
writeBinary(SignalListener::SanitizerTrap, out);
|
||||
writePODBinary(stack_trace, out);
|
||||
/// We create a dummy struct with a constructor so DISABLE_SANITIZER_INSTRUMENTATION is not applied to it
|
||||
/// otherwise, Memory sanitizer can't know that values initiialized inside this function are actually initialized
|
||||
/// because instrumentations are disabled leading to false positives later on
|
||||
ValueHolder<UInt32> thread_id{static_cast<UInt32>(getThreadId())};
|
||||
writeBinary(thread_id.value, out);
|
||||
writePODBinary(current_thread, out);
|
||||
|
||||
out.next();
|
||||
|
||||
/// The time that is usually enough for separate thread to print info into log.
|
||||
sleepForSeconds(20);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
/** To use with std::set_terminate.
|
||||
* Collects slightly more info than __gnu_cxx::__verbose_terminate_handler,
|
||||
* and send it to pipe. Other thread will read this info from pipe and asynchronously write it to log.
|
||||
* Look at libstdc++-v3/libsupc++/vterminate.cc for example.
|
||||
*/
|
||||
[[noreturn]] static void terminate_handler()
|
||||
{
|
||||
static thread_local bool terminating = false;
|
||||
if (terminating)
|
||||
abort();
|
||||
|
||||
terminating = true;
|
||||
|
||||
std::string log_message;
|
||||
|
||||
if (std::current_exception())
|
||||
log_message = "Terminate called for uncaught exception:\n" + getCurrentExceptionMessage(true);
|
||||
else
|
||||
log_message = "Terminate called without an active exception";
|
||||
|
||||
/// POSIX.1 says that write(2)s of less than PIPE_BUF bytes must be atomic - man 7 pipe
|
||||
/// And the buffer should not be too small because our exception messages can be large.
|
||||
static constexpr size_t buf_size = PIPE_BUF;
|
||||
|
||||
if (log_message.size() > buf_size - 16)
|
||||
log_message.resize(buf_size - 16);
|
||||
|
||||
char buf[buf_size];
|
||||
WriteBufferFromFileDescriptor out(signal_pipe.fds_rw[1], buf_size, buf);
|
||||
|
||||
writeBinary(static_cast<int>(SignalListener::StdTerminate), out);
|
||||
writeBinary(static_cast<UInt32>(getThreadId()), out);
|
||||
writeBinary(log_message, out);
|
||||
out.next();
|
||||
|
||||
abort();
|
||||
}
|
||||
|
||||
|
||||
static std::string createDirectory(const std::string & file)
|
||||
{
|
||||
fs::path path = fs::path(file).parent_path();
|
||||
@ -698,22 +148,7 @@ BaseDaemon::~BaseDaemon()
|
||||
{
|
||||
writeSignalIDtoSignalPipe(SignalListener::StopThread);
|
||||
signal_listener_thread.join();
|
||||
/// Reset signals to SIG_DFL to avoid trying to write to the signal_pipe that will be closed after.
|
||||
for (int sig : handled_signals)
|
||||
if (SIG_ERR == signal(sig, SIG_DFL))
|
||||
{
|
||||
try
|
||||
{
|
||||
throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
|
||||
}
|
||||
catch (ErrnoException &)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
signal_pipe.close();
|
||||
|
||||
HandledSignals::instance().reset();
|
||||
SentryWriter::resetInstance();
|
||||
}
|
||||
|
||||
@ -752,6 +187,8 @@ void BaseDaemon::closeFDs()
|
||||
#else
|
||||
fs::path proc_path{"/proc/self/fd"};
|
||||
#endif
|
||||
|
||||
const auto & signal_pipe = HandledSignals::instance().signal_pipe;
|
||||
if (fs::is_directory(proc_path)) /// Hooray, proc exists
|
||||
{
|
||||
/// in /proc/self/fd directory filenames are numeric file descriptors.
|
||||
@ -972,56 +409,6 @@ void BaseDaemon::initialize(Application & self)
|
||||
}
|
||||
|
||||
|
||||
static void addSignalHandler(const std::vector<int> & signals, signal_function handler, std::vector<int> * out_handled_signals)
|
||||
{
|
||||
struct sigaction sa;
|
||||
memset(&sa, 0, sizeof(sa));
|
||||
sa.sa_sigaction = handler;
|
||||
sa.sa_flags = SA_SIGINFO;
|
||||
|
||||
#if defined(OS_DARWIN)
|
||||
sigemptyset(&sa.sa_mask);
|
||||
for (auto signal : signals)
|
||||
sigaddset(&sa.sa_mask, signal);
|
||||
#else
|
||||
if (sigemptyset(&sa.sa_mask))
|
||||
throw Poco::Exception("Cannot set signal handler.");
|
||||
|
||||
for (auto signal : signals)
|
||||
if (sigaddset(&sa.sa_mask, signal))
|
||||
throw Poco::Exception("Cannot set signal handler.");
|
||||
#endif
|
||||
|
||||
for (auto signal : signals)
|
||||
if (sigaction(signal, &sa, nullptr))
|
||||
throw Poco::Exception("Cannot set signal handler.");
|
||||
|
||||
if (out_handled_signals)
|
||||
std::copy(signals.begin(), signals.end(), std::back_inserter(*out_handled_signals));
|
||||
}
|
||||
|
||||
|
||||
static void blockSignals(const std::vector<int> & signals)
|
||||
{
|
||||
sigset_t sig_set;
|
||||
|
||||
#if defined(OS_DARWIN)
|
||||
sigemptyset(&sig_set);
|
||||
for (auto signal : signals)
|
||||
sigaddset(&sig_set, signal);
|
||||
#else
|
||||
if (sigemptyset(&sig_set))
|
||||
throw Poco::Exception("Cannot block signal.");
|
||||
|
||||
for (auto signal : signals)
|
||||
if (sigaddset(&sig_set, signal))
|
||||
throw Poco::Exception("Cannot block signal.");
|
||||
#endif
|
||||
|
||||
if (pthread_sigmask(SIG_BLOCK, &sig_set, nullptr))
|
||||
throw Poco::Exception("Cannot block signal.");
|
||||
}
|
||||
|
||||
extern const char * GIT_HASH;
|
||||
|
||||
void BaseDaemon::initializeTerminationAndSignalProcessing()
|
||||
@ -1045,29 +432,21 @@ void BaseDaemon::initializeTerminationAndSignalProcessing()
|
||||
};
|
||||
}
|
||||
}
|
||||
std::set_terminate(terminate_handler);
|
||||
|
||||
/// We want to avoid SIGPIPE when working with sockets and pipes, and just handle return value/errno instead.
|
||||
blockSignals({SIGPIPE});
|
||||
|
||||
/// Setup signal handlers.
|
||||
/// SIGTSTP is added for debugging purposes. To output a stack trace of any running thread at anytime.
|
||||
addSignalHandler({SIGABRT, SIGSEGV, SIGILL, SIGBUS, SIGSYS, SIGFPE, SIGPIPE, SIGTSTP, SIGTRAP}, signalHandler, &handled_signals);
|
||||
addSignalHandler({SIGHUP}, closeLogsSignalHandler, &handled_signals);
|
||||
addSignalHandler({SIGINT, SIGQUIT, SIGTERM}, terminateRequestedSignalHandler, &handled_signals);
|
||||
|
||||
#if defined(SANITIZER)
|
||||
__sanitizer_set_death_callback(sanitizerDeathCallback);
|
||||
#endif
|
||||
HandledSignals::instance().setupTerminateHandler();
|
||||
HandledSignals::instance().setupCommonDeadlySignalHandlers();
|
||||
HandledSignals::instance().setupCommonTerminateRequestSignalHandlers();
|
||||
HandledSignals::instance().addSignalHandler({SIGHUP}, closeLogsSignalHandler, true);
|
||||
|
||||
/// Set up Poco ErrorHandler for Poco Threads.
|
||||
static KillingErrorHandler killing_error_handler;
|
||||
Poco::ErrorHandler::set(&killing_error_handler);
|
||||
|
||||
signal_pipe.setNonBlockingWrite();
|
||||
signal_pipe.tryIncreaseSize(1 << 20);
|
||||
|
||||
signal_listener = std::make_unique<SignalListener>(*this);
|
||||
signal_listener = std::make_unique<SignalListener>(this, getLogger("BaseDaemon"));
|
||||
signal_listener_thread.start(*signal_listener);
|
||||
|
||||
#if defined(__ELF__) && !defined(OS_FREEBSD)
|
||||
@ -1273,7 +652,7 @@ void BaseDaemon::setupWatchdog()
|
||||
/// Forward signals to the child process.
|
||||
if (forward_signals)
|
||||
{
|
||||
addSignalHandler(
|
||||
HandledSignals::instance().addSignalHandler(
|
||||
{SIGHUP, SIGINT, SIGQUIT, SIGTERM},
|
||||
[](int sig, siginfo_t *, void *)
|
||||
{
|
||||
@ -1289,7 +668,7 @@ void BaseDaemon::setupWatchdog()
|
||||
(void)res;
|
||||
}
|
||||
},
|
||||
nullptr);
|
||||
false);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -1306,6 +685,10 @@ void BaseDaemon::setupWatchdog()
|
||||
int status = 0;
|
||||
do
|
||||
{
|
||||
// Close log files to prevent keeping descriptors of unlinked rotated files.
|
||||
// On next log write files will be reopened.
|
||||
closeLogs(logger());
|
||||
|
||||
if (-1 != waitpid(pid, &status, WUNTRACED | WCONTINUED) || errno == ECHILD)
|
||||
{
|
||||
if (WIFSTOPPED(status))
|
||||
|
||||
@ -168,8 +168,6 @@ protected:
|
||||
String git_hash;
|
||||
String stored_binary_hash;
|
||||
|
||||
std::vector<int> handled_signals;
|
||||
|
||||
bool should_setup_watchdog = false;
|
||||
char * argv0 = nullptr;
|
||||
};
|
||||
|
||||
@ -17,7 +17,7 @@ SerializationPtr DataTypeDate::doGetDefaultSerialization() const
|
||||
|
||||
void registerDataTypeDate(DataTypeFactory & factory)
|
||||
{
|
||||
factory.registerSimpleDataType("Date", [] { return DataTypePtr(std::make_shared<DataTypeDate>()); }, DataTypeFactory::CaseInsensitive);
|
||||
factory.registerSimpleDataType("Date", [] { return DataTypePtr(std::make_shared<DataTypeDate>()); }, DataTypeFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -24,7 +24,7 @@ Field DataTypeDate32::getDefault() const
|
||||
void registerDataTypeDate32(DataTypeFactory & factory)
|
||||
{
|
||||
factory.registerSimpleDataType(
|
||||
"Date32", [] { return DataTypePtr(std::make_shared<DataTypeDate32>()); }, DataTypeFactory::CaseInsensitive);
|
||||
"Date32", [] { return DataTypePtr(std::make_shared<DataTypeDate32>()); }, DataTypeFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -15,8 +15,8 @@ void registerDataTypeDomainBool(DataTypeFactory & factory)
|
||||
std::make_unique<DataTypeCustomFixedName>("Bool"), std::make_unique<SerializationBool>(type->getDefaultSerialization())));
|
||||
});
|
||||
|
||||
factory.registerAlias("bool", "Bool", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("boolean", "Bool", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("bool", "Bool", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("boolean", "Bool", DataTypeFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -318,7 +318,7 @@ void registerDataTypeEnum(DataTypeFactory & factory)
|
||||
factory.registerDataType("Enum", create);
|
||||
|
||||
/// MySQL
|
||||
factory.registerAlias("ENUM", "Enum", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("ENUM", "Enum", DataTypeFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -175,7 +175,7 @@ DataTypePtr DataTypeFactory::getCustom(DataTypeCustomDescPtr customization) cons
|
||||
}
|
||||
|
||||
|
||||
void DataTypeFactory::registerDataType(const String & family_name, Value creator, CaseSensitiveness case_sensitiveness)
|
||||
void DataTypeFactory::registerDataType(const String & family_name, Value creator, Case case_sensitiveness)
|
||||
{
|
||||
if (creator == nullptr)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "DataTypeFactory: the data type family {} has been provided a null constructor", family_name);
|
||||
@ -189,12 +189,12 @@ void DataTypeFactory::registerDataType(const String & family_name, Value creator
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "DataTypeFactory: the data type family name '{}' is not unique",
|
||||
family_name);
|
||||
|
||||
if (case_sensitiveness == CaseInsensitive
|
||||
if (case_sensitiveness == Case::Insensitive
|
||||
&& !case_insensitive_data_types.emplace(family_name_lowercase, creator).second)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "DataTypeFactory: the case insensitive data type family name '{}' is not unique", family_name);
|
||||
}
|
||||
|
||||
void DataTypeFactory::registerSimpleDataType(const String & name, SimpleCreator creator, CaseSensitiveness case_sensitiveness)
|
||||
void DataTypeFactory::registerSimpleDataType(const String & name, SimpleCreator creator, Case case_sensitiveness)
|
||||
{
|
||||
if (creator == nullptr)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "DataTypeFactory: the data type {} has been provided a null constructor",
|
||||
@ -208,7 +208,7 @@ void DataTypeFactory::registerSimpleDataType(const String & name, SimpleCreator
|
||||
}, case_sensitiveness);
|
||||
}
|
||||
|
||||
void DataTypeFactory::registerDataTypeCustom(const String & family_name, CreatorWithCustom creator, CaseSensitiveness case_sensitiveness)
|
||||
void DataTypeFactory::registerDataTypeCustom(const String & family_name, CreatorWithCustom creator, Case case_sensitiveness)
|
||||
{
|
||||
registerDataType(family_name, [creator](const ASTPtr & ast)
|
||||
{
|
||||
@ -219,7 +219,7 @@ void DataTypeFactory::registerDataTypeCustom(const String & family_name, Creator
|
||||
}, case_sensitiveness);
|
||||
}
|
||||
|
||||
void DataTypeFactory::registerSimpleDataTypeCustom(const String & name, SimpleCreatorWithCustom creator, CaseSensitiveness case_sensitiveness)
|
||||
void DataTypeFactory::registerSimpleDataTypeCustom(const String & name, SimpleCreatorWithCustom creator, Case case_sensitiveness)
|
||||
{
|
||||
registerDataTypeCustom(name, [name, creator](const ASTPtr & ast)
|
||||
{
|
||||
|
||||
@ -41,16 +41,16 @@ public:
|
||||
DataTypePtr tryGet(const ASTPtr & ast) const;
|
||||
|
||||
/// Register a type family by its name.
|
||||
void registerDataType(const String & family_name, Value creator, CaseSensitiveness case_sensitiveness = CaseSensitive);
|
||||
void registerDataType(const String & family_name, Value creator, Case case_sensitiveness = Case::Sensitive);
|
||||
|
||||
/// Register a simple data type, that have no parameters.
|
||||
void registerSimpleDataType(const String & name, SimpleCreator creator, CaseSensitiveness case_sensitiveness = CaseSensitive);
|
||||
void registerSimpleDataType(const String & name, SimpleCreator creator, Case case_sensitiveness = Case::Sensitive);
|
||||
|
||||
/// Register a customized type family
|
||||
void registerDataTypeCustom(const String & family_name, CreatorWithCustom creator, CaseSensitiveness case_sensitiveness = CaseSensitive);
|
||||
void registerDataTypeCustom(const String & family_name, CreatorWithCustom creator, Case case_sensitiveness = Case::Sensitive);
|
||||
|
||||
/// Register a simple customized data type
|
||||
void registerSimpleDataTypeCustom(const String & name, SimpleCreatorWithCustom creator, CaseSensitiveness case_sensitiveness = CaseSensitive);
|
||||
void registerSimpleDataTypeCustom(const String & name, SimpleCreatorWithCustom creator, Case case_sensitiveness = Case::Sensitive);
|
||||
|
||||
private:
|
||||
template <bool nullptr_on_error>
|
||||
|
||||
@ -64,7 +64,7 @@ void registerDataTypeFixedString(DataTypeFactory & factory)
|
||||
factory.registerDataType("FixedString", create);
|
||||
|
||||
/// Compatibility alias.
|
||||
factory.registerAlias("BINARY", "FixedString", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("BINARY", "FixedString", DataTypeFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -9,9 +9,9 @@ namespace DB
|
||||
void registerDataTypeIPv4andIPv6(DataTypeFactory & factory)
|
||||
{
|
||||
factory.registerSimpleDataType("IPv4", [] { return DataTypePtr(std::make_shared<DataTypeIPv4>()); });
|
||||
factory.registerAlias("INET4", "IPv4", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("INET4", "IPv4", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerSimpleDataType("IPv6", [] { return DataTypePtr(std::make_shared<DataTypeIPv6>()); });
|
||||
factory.registerAlias("INET6", "IPv6", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("INET6", "IPv6", DataTypeFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -76,7 +76,7 @@ void registerDataTypeObject(DataTypeFactory & factory)
|
||||
factory.registerDataType("Object", create);
|
||||
factory.registerSimpleDataType("JSON",
|
||||
[] { return std::make_shared<DataTypeObject>("JSON", false); },
|
||||
DataTypeFactory::CaseInsensitive);
|
||||
DataTypeFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -62,38 +62,38 @@ void registerDataTypeString(DataTypeFactory & factory)
|
||||
|
||||
/// These synonims are added for compatibility.
|
||||
|
||||
factory.registerAlias("CHAR", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("NCHAR", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("CHARACTER", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("VARCHAR", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("NVARCHAR", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("VARCHAR2", "String", DataTypeFactory::CaseInsensitive); /// Oracle
|
||||
factory.registerAlias("TEXT", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("TINYTEXT", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("MEDIUMTEXT", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("LONGTEXT", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("BLOB", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("CLOB", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("TINYBLOB", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("MEDIUMBLOB", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("LONGBLOB", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("BYTEA", "String", DataTypeFactory::CaseInsensitive); /// PostgreSQL
|
||||
factory.registerAlias("CHAR", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("NCHAR", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("CHARACTER", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("VARCHAR", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("NVARCHAR", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("VARCHAR2", "String", DataTypeFactory::Case::Insensitive); /// Oracle
|
||||
factory.registerAlias("TEXT", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("TINYTEXT", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("MEDIUMTEXT", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("LONGTEXT", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("BLOB", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("CLOB", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("TINYBLOB", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("MEDIUMBLOB", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("LONGBLOB", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("BYTEA", "String", DataTypeFactory::Case::Insensitive); /// PostgreSQL
|
||||
|
||||
factory.registerAlias("CHARACTER LARGE OBJECT", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("CHARACTER VARYING", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("CHAR LARGE OBJECT", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("CHAR VARYING", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("NATIONAL CHAR", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("NATIONAL CHARACTER", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("NATIONAL CHARACTER LARGE OBJECT", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("NATIONAL CHARACTER VARYING", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("NATIONAL CHAR VARYING", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("NCHAR VARYING", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("NCHAR LARGE OBJECT", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("BINARY LARGE OBJECT", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("BINARY VARYING", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("VARBINARY", "String", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("GEOMETRY", "String", DataTypeFactory::CaseInsensitive); //mysql
|
||||
factory.registerAlias("CHARACTER LARGE OBJECT", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("CHARACTER VARYING", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("CHAR LARGE OBJECT", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("CHAR VARYING", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("NATIONAL CHAR", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("NATIONAL CHARACTER", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("NATIONAL CHARACTER LARGE OBJECT", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("NATIONAL CHARACTER VARYING", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("NATIONAL CHAR VARYING", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("NCHAR VARYING", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("NCHAR LARGE OBJECT", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("BINARY LARGE OBJECT", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("BINARY VARYING", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("VARBINARY", "String", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("GEOMETRY", "String", DataTypeFactory::Case::Insensitive); //mysql
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@ -364,15 +364,15 @@ template class DataTypeDecimal<Decimal256>;
|
||||
|
||||
void registerDataTypeDecimal(DataTypeFactory & factory)
|
||||
{
|
||||
factory.registerDataType("Decimal32", createExact<Decimal32>, DataTypeFactory::CaseInsensitive);
|
||||
factory.registerDataType("Decimal64", createExact<Decimal64>, DataTypeFactory::CaseInsensitive);
|
||||
factory.registerDataType("Decimal128", createExact<Decimal128>, DataTypeFactory::CaseInsensitive);
|
||||
factory.registerDataType("Decimal256", createExact<Decimal256>, DataTypeFactory::CaseInsensitive);
|
||||
factory.registerDataType("Decimal32", createExact<Decimal32>, DataTypeFactory::Case::Insensitive);
|
||||
factory.registerDataType("Decimal64", createExact<Decimal64>, DataTypeFactory::Case::Insensitive);
|
||||
factory.registerDataType("Decimal128", createExact<Decimal128>, DataTypeFactory::Case::Insensitive);
|
||||
factory.registerDataType("Decimal256", createExact<Decimal256>, DataTypeFactory::Case::Insensitive);
|
||||
|
||||
factory.registerDataType("Decimal", create, DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("DEC", "Decimal", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("NUMERIC", "Decimal", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("FIXED", "Decimal", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerDataType("Decimal", create, DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("DEC", "Decimal", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("NUMERIC", "Decimal", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("FIXED", "Decimal", DataTypeFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -65,41 +65,41 @@ void registerDataTypeNumbers(DataTypeFactory & factory)
|
||||
|
||||
/// These synonyms are added for compatibility.
|
||||
|
||||
factory.registerAlias("TINYINT", "Int8", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("INT1", "Int8", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("BYTE", "Int8", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("TINYINT SIGNED", "Int8", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("INT1 SIGNED", "Int8", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("SMALLINT", "Int16", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("SMALLINT SIGNED", "Int16", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("INT", "Int32", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("INTEGER", "Int32", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("MEDIUMINT", "Int32", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("MEDIUMINT SIGNED", "Int32", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("INT SIGNED", "Int32", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("INTEGER SIGNED", "Int32", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("BIGINT", "Int64", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("SIGNED", "Int64", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("BIGINT SIGNED", "Int64", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("TIME", "Int64", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("TINYINT", "Int8", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("INT1", "Int8", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("BYTE", "Int8", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("TINYINT SIGNED", "Int8", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("INT1 SIGNED", "Int8", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("SMALLINT", "Int16", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("SMALLINT SIGNED", "Int16", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("INT", "Int32", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("INTEGER", "Int32", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("MEDIUMINT", "Int32", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("MEDIUMINT SIGNED", "Int32", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("INT SIGNED", "Int32", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("INTEGER SIGNED", "Int32", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("BIGINT", "Int64", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("SIGNED", "Int64", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("BIGINT SIGNED", "Int64", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("TIME", "Int64", DataTypeFactory::Case::Insensitive);
|
||||
|
||||
factory.registerAlias("TINYINT UNSIGNED", "UInt8", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("INT1 UNSIGNED", "UInt8", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("SMALLINT UNSIGNED", "UInt16", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("YEAR", "UInt16", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("MEDIUMINT UNSIGNED", "UInt32", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("INT UNSIGNED", "UInt32", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("INTEGER UNSIGNED", "UInt32", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("UNSIGNED", "UInt64", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("BIGINT UNSIGNED", "UInt64", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("BIT", "UInt64", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("SET", "UInt64", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("TINYINT UNSIGNED", "UInt8", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("INT1 UNSIGNED", "UInt8", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("SMALLINT UNSIGNED", "UInt16", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("YEAR", "UInt16", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("MEDIUMINT UNSIGNED", "UInt32", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("INT UNSIGNED", "UInt32", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("INTEGER UNSIGNED", "UInt32", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("UNSIGNED", "UInt64", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("BIGINT UNSIGNED", "UInt64", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("BIT", "UInt64", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("SET", "UInt64", DataTypeFactory::Case::Insensitive);
|
||||
|
||||
factory.registerAlias("FLOAT", "Float32", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("REAL", "Float32", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("SINGLE", "Float32", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("DOUBLE", "Float64", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("DOUBLE PRECISION", "Float64", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("FLOAT", "Float32", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("REAL", "Float32", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("SINGLE", "Float32", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("DOUBLE", "Float64", DataTypeFactory::Case::Insensitive);
|
||||
factory.registerAlias("DOUBLE PRECISION", "Float64", DataTypeFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
/// Explicit template instantiations.
|
||||
|
||||
@ -11,6 +11,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
struct SerializationVariantElement::DeserializeBinaryBulkStateVariantElement : public ISerialization::DeserializeBinaryBulkState
|
||||
@ -188,13 +189,6 @@ void SerializationVariantElement::deserializeBinaryBulkWithMultipleStreams(
|
||||
assert_cast<ColumnLowCardinality &>(*variant_element_state->variant->assumeMutable()).nestedRemoveNullable();
|
||||
}
|
||||
|
||||
/// If nothing to deserialize, just insert defaults.
|
||||
if (variant_limit == 0)
|
||||
{
|
||||
mutable_column->insertManyDefaults(num_new_discriminators);
|
||||
return;
|
||||
}
|
||||
|
||||
addVariantToPath(settings.path);
|
||||
nested_serialization->deserializeBinaryBulkWithMultipleStreams(variant_element_state->variant, *variant_limit, settings, variant_element_state->variant_element_state, cache);
|
||||
removeVariantFromPath(settings.path);
|
||||
@ -209,6 +203,17 @@ void SerializationVariantElement::deserializeBinaryBulkWithMultipleStreams(
|
||||
return;
|
||||
}
|
||||
|
||||
/// If there was nothing to deserialize or nothing was actually deserialized when variant_limit > 0, just insert defaults.
|
||||
/// The second case means that we don't have a stream for such sub-column. It may happen during ALTER MODIFY column with Variant extension.
|
||||
if (variant_limit == 0 || variant_element_state->variant->empty())
|
||||
{
|
||||
mutable_column->insertManyDefaults(num_new_discriminators);
|
||||
return;
|
||||
}
|
||||
|
||||
if (variant_element_state->variant->size() < *variant_limit)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of deserialized variant column less than the limit: {} < {}", variant_element_state->variant->size(), *variant_limit);
|
||||
|
||||
size_t variant_offset = variant_element_state->variant->size() - *variant_limit;
|
||||
|
||||
/// If we have only our discriminator in range, insert the whole range to result column.
|
||||
|
||||
@ -108,11 +108,11 @@ static DataTypePtr create64(const ASTPtr & arguments)
|
||||
|
||||
void registerDataTypeDateTime(DataTypeFactory & factory)
|
||||
{
|
||||
factory.registerDataType("DateTime", create, DataTypeFactory::CaseInsensitive);
|
||||
factory.registerDataType("DateTime32", create32, DataTypeFactory::CaseInsensitive);
|
||||
factory.registerDataType("DateTime64", create64, DataTypeFactory::CaseInsensitive);
|
||||
factory.registerDataType("DateTime", create, DataTypeFactory::Case::Insensitive);
|
||||
factory.registerDataType("DateTime32", create32, DataTypeFactory::Case::Insensitive);
|
||||
factory.registerDataType("DateTime64", create64, DataTypeFactory::Case::Insensitive);
|
||||
|
||||
factory.registerAlias("TIMESTAMP", "DateTime", DataTypeFactory::CaseInsensitive);
|
||||
factory.registerAlias("TIMESTAMP", "DateTime", DataTypeFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -39,8 +39,10 @@ namespace ErrorCodes
|
||||
class AtomicDatabaseTablesSnapshotIterator final : public DatabaseTablesSnapshotIterator
|
||||
{
|
||||
public:
|
||||
explicit AtomicDatabaseTablesSnapshotIterator(DatabaseTablesSnapshotIterator && base)
|
||||
: DatabaseTablesSnapshotIterator(std::move(base)) {}
|
||||
explicit AtomicDatabaseTablesSnapshotIterator(DatabaseTablesSnapshotIterator && base) noexcept
|
||||
: DatabaseTablesSnapshotIterator(std::move(base))
|
||||
{
|
||||
}
|
||||
UUID uuid() const override { return table()->getStorageID().uuid; }
|
||||
};
|
||||
|
||||
@ -111,12 +113,12 @@ StoragePtr DatabaseAtomic::detachTable(ContextPtr /* context */, const String &
|
||||
// it is important to call the destructors of not_in_use without
|
||||
// locked mutex to avoid potential deadlock.
|
||||
DetachedTables not_in_use;
|
||||
StoragePtr table;
|
||||
StoragePtr detached_table;
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
table = DatabaseOrdinary::detachTableUnlocked(name);
|
||||
detached_table = DatabaseOrdinary::detachTableUnlocked(name);
|
||||
table_name_to_path.erase(name);
|
||||
detached_tables.emplace(table->getStorageID().uuid, table);
|
||||
detached_tables.emplace(detached_table->getStorageID().uuid, detached_table);
|
||||
not_in_use = cleanupDetachedTables();
|
||||
}
|
||||
|
||||
@ -126,7 +128,7 @@ StoragePtr DatabaseAtomic::detachTable(ContextPtr /* context */, const String &
|
||||
LOG_DEBUG(log, "Finished removing not used detached tables");
|
||||
}
|
||||
|
||||
return table;
|
||||
return detached_table;
|
||||
}
|
||||
|
||||
void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_name, bool sync)
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
#pragma once
|
||||
|
||||
#include <Databases/DatabasesCommon.h>
|
||||
#include <Databases/DatabaseOrdinary.h>
|
||||
#include <Databases/DatabasesCommon.h>
|
||||
#include <Storages/IStorage_fwd.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -105,7 +105,7 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const String & table_name, Co
|
||||
return {};
|
||||
}
|
||||
|
||||
auto names_and_types = StorageDictionary::getNamesAndTypes(ExternalDictionariesLoader::getDictionaryStructure(*load_result.config));
|
||||
auto names_and_types = StorageDictionary::getNamesAndTypes(ExternalDictionariesLoader::getDictionaryStructure(*load_result.config), false);
|
||||
buffer << "CREATE TABLE " << backQuoteIfNeed(getDatabaseName()) << '.' << backQuoteIfNeed(table_name) << " (";
|
||||
buffer << names_and_types.toNamesAndTypesDescription();
|
||||
buffer << ") Engine = Dictionary(" << backQuoteIfNeed(table_name) << ")";
|
||||
|
||||
@ -188,6 +188,13 @@ void DatabaseLazy::attachTable(ContextPtr /* context_ */, const String & table_n
|
||||
|
||||
it->second.expiration_iterator = cache_expiration_queue.emplace(cache_expiration_queue.end(), current_time, table_name);
|
||||
|
||||
LOG_DEBUG(log, "Add info for detached table {} to snapshot.", backQuote(table_name));
|
||||
if (snapshot_detached_tables.contains(table_name))
|
||||
{
|
||||
LOG_DEBUG(log, "Clean info about detached table {} from snapshot.", backQuote(table_name));
|
||||
snapshot_detached_tables.erase(table_name);
|
||||
}
|
||||
|
||||
CurrentMetrics::add(CurrentMetrics::AttachedTable, 1);
|
||||
}
|
||||
|
||||
@ -204,6 +211,15 @@ StoragePtr DatabaseLazy::detachTable(ContextPtr /* context */, const String & ta
|
||||
if (it->second.expiration_iterator != cache_expiration_queue.end())
|
||||
cache_expiration_queue.erase(it->second.expiration_iterator);
|
||||
tables_cache.erase(it);
|
||||
LOG_DEBUG(log, "Add info for detached table {} to snapshot.", backQuote(table_name));
|
||||
snapshot_detached_tables.emplace(
|
||||
table_name,
|
||||
SnapshotDetachedTable{
|
||||
.database = res->getStorageID().database_name,
|
||||
.table = res->getStorageID().table_name,
|
||||
.uuid = res->getStorageID().uuid,
|
||||
.metadata_path = getObjectMetadataPath(table_name),
|
||||
.is_permanently = false});
|
||||
|
||||
CurrentMetrics::sub(CurrentMetrics::AttachedTable, 1);
|
||||
}
|
||||
|
||||
@ -23,6 +23,7 @@
|
||||
#include <Storages/StorageFactory.h>
|
||||
#include <TableFunctions/TableFunctionFactory.h>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Common/escapeForFileName.h>
|
||||
#include <Common/filesystemHelpers.h>
|
||||
@ -308,6 +309,16 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri
|
||||
try
|
||||
{
|
||||
FS::createFile(detached_permanently_flag);
|
||||
|
||||
std::lock_guard lock(mutex);
|
||||
if (const auto it = snapshot_detached_tables.find(table_name); it == snapshot_detached_tables.end())
|
||||
{
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table={}", table_name);
|
||||
}
|
||||
else
|
||||
{
|
||||
it->second.is_permanently = true;
|
||||
}
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
|
||||
@ -189,7 +189,7 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
|
||||
size_t prev_tables_count = metadata.parsed_tables.size();
|
||||
size_t prev_total_dictionaries = metadata.total_dictionaries;
|
||||
|
||||
auto process_metadata = [&metadata, is_startup, this](const String & file_name)
|
||||
auto process_metadata = [&metadata, is_startup, local_context, this](const String & file_name)
|
||||
{
|
||||
fs::path path(getMetadataPath());
|
||||
fs::path file_path(file_name);
|
||||
@ -197,7 +197,7 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
|
||||
|
||||
try
|
||||
{
|
||||
auto ast = parseQueryFromMetadata(log, getContext(), full_path.string(), /*throw_on_error*/ true, /*remove_empty*/ false);
|
||||
auto ast = parseQueryFromMetadata(log, local_context, full_path.string(), /*throw_on_error*/ true, /*remove_empty*/ false);
|
||||
if (ast)
|
||||
{
|
||||
FunctionNameNormalizer::visit(ast.get());
|
||||
@ -226,8 +226,23 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
|
||||
if (fs::exists(full_path.string() + detached_suffix))
|
||||
{
|
||||
const std::string table_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4));
|
||||
permanently_detached_tables.push_back(table_name);
|
||||
LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name));
|
||||
|
||||
std::lock_guard lock(mutex);
|
||||
permanently_detached_tables.push_back(table_name);
|
||||
|
||||
const auto detached_table_name = create_query->getTable();
|
||||
|
||||
snapshot_detached_tables.emplace(
|
||||
detached_table_name,
|
||||
SnapshotDetachedTable{
|
||||
.database = create_query->getDatabase(),
|
||||
.table = detached_table_name,
|
||||
.uuid = create_query->uuid,
|
||||
.metadata_path = getObjectMetadataPath(detached_table_name),
|
||||
.is_permanently = true});
|
||||
|
||||
LOG_TRACE(log, "Add permanently detached table {} to system.detached_tables", detached_table_name);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -489,6 +504,12 @@ DatabaseTablesIteratorPtr DatabaseOrdinary::getTablesIterator(ContextPtr local_c
|
||||
return DatabaseWithOwnTablesBase::getTablesIterator(local_context, filter_by_table_name, skip_not_loaded);
|
||||
}
|
||||
|
||||
DatabaseDetachedTablesSnapshotIteratorPtr DatabaseOrdinary::getDetachedTablesIterator(
|
||||
ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const
|
||||
{
|
||||
return DatabaseWithOwnTablesBase::getDetachedTablesIterator(local_context, filter_by_table_name, skip_not_loaded);
|
||||
}
|
||||
|
||||
Strings DatabaseOrdinary::getAllTableNames(ContextPtr) const
|
||||
{
|
||||
std::set<String> unique_names;
|
||||
|
||||
@ -57,6 +57,9 @@ public:
|
||||
LoadTaskPtr startupDatabaseAsync(AsyncLoader & async_loader, LoadJobSet startup_after, LoadingStrictnessLevel mode) override;
|
||||
|
||||
DatabaseTablesIteratorPtr getTablesIterator(ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
|
||||
DatabaseDetachedTablesSnapshotIteratorPtr getDetachedTablesIterator(
|
||||
ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
|
||||
|
||||
Strings getAllTableNames(ContextPtr context) const override;
|
||||
|
||||
void alterTable(
|
||||
@ -64,7 +67,11 @@ public:
|
||||
const StorageID & table_id,
|
||||
const StorageInMemoryMetadata & metadata) override;
|
||||
|
||||
Strings getNamesOfPermanentlyDetachedTables() const override { return permanently_detached_tables; }
|
||||
Strings getNamesOfPermanentlyDetachedTables() const override
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
return permanently_detached_tables;
|
||||
}
|
||||
|
||||
protected:
|
||||
virtual void commitAlterTable(
|
||||
@ -74,7 +81,7 @@ protected:
|
||||
const String & statement,
|
||||
ContextPtr query_context);
|
||||
|
||||
Strings permanently_detached_tables;
|
||||
Strings permanently_detached_tables TSA_GUARDED_BY(mutex);
|
||||
|
||||
std::unordered_map<String, LoadTaskPtr> load_table TSA_GUARDED_BY(mutex);
|
||||
std::unordered_map<String, LoadTaskPtr> startup_table TSA_GUARDED_BY(mutex);
|
||||
|
||||
@ -2,12 +2,9 @@
|
||||
|
||||
#include <Backups/BackupEntriesCollector.h>
|
||||
#include <Backups/RestorerFromBackup.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
#include <Common/escapeForFileName.h>
|
||||
#include <Interpreters/InterpreterCreateQuery.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/DatabaseCatalog.h>
|
||||
#include <Interpreters/InterpreterCreateQuery.h>
|
||||
#include <Parsers/ASTCreateQuery.h>
|
||||
#include <Parsers/ASTSelectWithUnionQuery.h>
|
||||
#include <Parsers/ParserCreateQuery.h>
|
||||
@ -16,6 +13,10 @@
|
||||
#include <Storages/StorageFactory.h>
|
||||
#include <Storages/Utils.h>
|
||||
#include <TableFunctions/TableFunctionFactory.h>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
#include <Common/escapeForFileName.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -237,6 +238,24 @@ DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(ContextPt
|
||||
return std::make_unique<DatabaseTablesSnapshotIterator>(std::move(filtered_tables), database_name);
|
||||
}
|
||||
|
||||
DatabaseDetachedTablesSnapshotIteratorPtr DatabaseWithOwnTablesBase::getDetachedTablesIterator(
|
||||
ContextPtr, const FilterByNameFunction & filter_by_table_name, bool /* skip_not_loaded */) const
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
if (!filter_by_table_name)
|
||||
return std::make_unique<DatabaseDetachedTablesSnapshotIterator>(snapshot_detached_tables);
|
||||
|
||||
SnapshotDetachedTables filtered_detached_tables;
|
||||
for (const auto & [detached_table_name, snapshot] : snapshot_detached_tables)
|
||||
if (filter_by_table_name(detached_table_name))
|
||||
{
|
||||
filtered_detached_tables.emplace(detached_table_name, snapshot);
|
||||
}
|
||||
|
||||
|
||||
return std::make_unique<DatabaseDetachedTablesSnapshotIterator>(std::move(filtered_detached_tables));
|
||||
}
|
||||
|
||||
bool DatabaseWithOwnTablesBase::empty() const
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
@ -251,27 +270,36 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(ContextPtr /* context_ */, con
|
||||
|
||||
StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_name)
|
||||
{
|
||||
StoragePtr res;
|
||||
|
||||
auto it = tables.find(table_name);
|
||||
if (it == tables.end())
|
||||
throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist",
|
||||
backQuote(database_name), backQuote(table_name));
|
||||
res = it->second;
|
||||
|
||||
auto table_storage = it->second;
|
||||
|
||||
snapshot_detached_tables.emplace(
|
||||
table_name,
|
||||
SnapshotDetachedTable{
|
||||
.database = it->second->getStorageID().getDatabaseName(),
|
||||
.table = table_name,
|
||||
.uuid = it->second->getStorageID().uuid,
|
||||
.metadata_path = getObjectMetadataPath(table_name),
|
||||
.is_permanently = false});
|
||||
|
||||
tables.erase(it);
|
||||
res->is_detached = true;
|
||||
table_storage->is_detached = true;
|
||||
|
||||
if (res->isSystemStorage() == false)
|
||||
CurrentMetrics::sub(getAttachedCounterForStorage(res), 1);
|
||||
if (table_storage->isSystemStorage() == false)
|
||||
CurrentMetrics::sub(getAttachedCounterForStorage(table_storage), 1);
|
||||
|
||||
auto table_id = res->getStorageID();
|
||||
auto table_id = table_storage->getStorageID();
|
||||
if (table_id.hasUUID())
|
||||
{
|
||||
assert(database_name == DatabaseCatalog::TEMPORARY_DATABASE || getUUID() != UUIDHelpers::Nil);
|
||||
DatabaseCatalog::instance().removeUUIDMapping(table_id.uuid);
|
||||
}
|
||||
|
||||
return res;
|
||||
return table_storage;
|
||||
}
|
||||
|
||||
void DatabaseWithOwnTablesBase::attachTable(ContextPtr /* context_ */, const String & table_name, const StoragePtr & table, const String &)
|
||||
@ -300,6 +328,8 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c
|
||||
throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {} already exists.", table_id.getFullTableName());
|
||||
}
|
||||
|
||||
snapshot_detached_tables.erase(table_name);
|
||||
|
||||
/// It is important to reset is_detached here since in case of RENAME in
|
||||
/// non-Atomic database the is_detached is set to true before RENAME.
|
||||
table->is_detached = false;
|
||||
@ -337,6 +367,7 @@ void DatabaseWithOwnTablesBase::shutdown()
|
||||
|
||||
std::lock_guard lock(mutex);
|
||||
tables.clear();
|
||||
snapshot_detached_tables.clear();
|
||||
}
|
||||
|
||||
DatabaseWithOwnTablesBase::~DatabaseWithOwnTablesBase()
|
||||
|
||||
@ -37,6 +37,9 @@ public:
|
||||
|
||||
DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
|
||||
|
||||
DatabaseDetachedTablesSnapshotIteratorPtr
|
||||
getDetachedTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
|
||||
|
||||
std::vector<std::pair<ASTPtr, StoragePtr>> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override;
|
||||
void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr<IRestoreCoordination> restore_coordination, UInt64 timeout_ms) override;
|
||||
|
||||
@ -46,12 +49,13 @@ public:
|
||||
|
||||
protected:
|
||||
Tables tables TSA_GUARDED_BY(mutex);
|
||||
SnapshotDetachedTables snapshot_detached_tables TSA_GUARDED_BY(mutex);
|
||||
LoggerPtr log;
|
||||
|
||||
DatabaseWithOwnTablesBase(const String & name_, const String & logger, ContextPtr context);
|
||||
|
||||
void attachTableUnlocked(const String & table_name, const StoragePtr & table) TSA_REQUIRES(mutex);
|
||||
StoragePtr detachTableUnlocked(const String & table_name) TSA_REQUIRES(mutex);
|
||||
StoragePtr detachTableUnlocked(const String & table_name) TSA_REQUIRES(mutex);
|
||||
StoragePtr getTableUnlocked(const String & table_name) const TSA_REQUIRES(mutex);
|
||||
StoragePtr tryGetTableNoWait(const String & table_name) const;
|
||||
};
|
||||
|
||||
@ -5,20 +5,21 @@
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
#include <Interpreters/executeQuery.h>
|
||||
#include <Parsers/IAST_fwd.h>
|
||||
#include <QueryPipeline/BlockIO.h>
|
||||
#include <Storages/IStorage_fwd.h>
|
||||
#include <base/types.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/AsyncLoader.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/PoolId.h>
|
||||
#include <Common/ThreadPool_fwd.h>
|
||||
#include <QueryPipeline/BlockIO.h>
|
||||
|
||||
#include <ctime>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <stdexcept>
|
||||
#include <vector>
|
||||
#include <map>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -110,6 +111,57 @@ public:
|
||||
|
||||
using DatabaseTablesIteratorPtr = std::unique_ptr<IDatabaseTablesIterator>;
|
||||
|
||||
struct SnapshotDetachedTable final
|
||||
{
|
||||
String database;
|
||||
String table;
|
||||
UUID uuid = UUIDHelpers::Nil;
|
||||
String metadata_path;
|
||||
bool is_permanently{};
|
||||
};
|
||||
|
||||
class DatabaseDetachedTablesSnapshotIterator
|
||||
{
|
||||
private:
|
||||
SnapshotDetachedTables snapshot;
|
||||
SnapshotDetachedTables::iterator it;
|
||||
|
||||
protected:
|
||||
DatabaseDetachedTablesSnapshotIterator(DatabaseDetachedTablesSnapshotIterator && other) noexcept
|
||||
{
|
||||
size_t idx = std::distance(other.snapshot.begin(), other.it);
|
||||
std::swap(snapshot, other.snapshot);
|
||||
other.it = other.snapshot.end();
|
||||
it = snapshot.begin();
|
||||
std::advance(it, idx);
|
||||
}
|
||||
|
||||
public:
|
||||
explicit DatabaseDetachedTablesSnapshotIterator(const SnapshotDetachedTables & tables_) : snapshot(tables_), it(snapshot.begin())
|
||||
{
|
||||
}
|
||||
|
||||
explicit DatabaseDetachedTablesSnapshotIterator(SnapshotDetachedTables && tables_) : snapshot(std::move(tables_)), it(snapshot.begin())
|
||||
{
|
||||
}
|
||||
|
||||
void next() { ++it; }
|
||||
|
||||
bool isValid() const { return it != snapshot.end(); }
|
||||
|
||||
String database() const { return it->second.database; }
|
||||
|
||||
String table() const { return it->second.table; }
|
||||
|
||||
UUID uuid() const { return it->second.uuid; }
|
||||
|
||||
String metadataPath() const { return it->second.metadata_path; }
|
||||
|
||||
bool isPermanently() const { return it->second.is_permanently; }
|
||||
};
|
||||
|
||||
using DatabaseDetachedTablesSnapshotIteratorPtr = std::unique_ptr<DatabaseDetachedTablesSnapshotIterator>;
|
||||
|
||||
|
||||
/** Database engine.
|
||||
* It is responsible for:
|
||||
@ -232,6 +284,12 @@ public:
|
||||
/// Wait for all tables to be loaded and started up. If `skip_not_loaded` is true, then not yet loaded or not yet started up (at the moment of iterator creation) tables are excluded.
|
||||
virtual DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name = {}, bool skip_not_loaded = false) const = 0; /// NOLINT
|
||||
|
||||
virtual DatabaseDetachedTablesSnapshotIteratorPtr getDetachedTablesIterator(
|
||||
ContextPtr /*context*/, const FilterByNameFunction & /*filter_by_table_name = {}*/, bool /*skip_not_loaded = false*/) const
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get detached tables for Database{}", getEngineName());
|
||||
}
|
||||
|
||||
/// Returns list of table names.
|
||||
virtual Strings getAllTableNames(ContextPtr context) const
|
||||
{
|
||||
|
||||
@ -58,15 +58,6 @@ std::optional<AttributeUnderlyingType> tryGetAttributeUnderlyingType(TypeIndex i
|
||||
|
||||
}
|
||||
|
||||
|
||||
DictionarySpecialAttribute::DictionarySpecialAttribute(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix)
|
||||
: name{config.getString(config_prefix + ".name", "")}, expression{config.getString(config_prefix + ".expression", "")}
|
||||
{
|
||||
if (name.empty() && !expression.empty())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Element {}.name is empty", config_prefix);
|
||||
}
|
||||
|
||||
|
||||
DictionaryStructure::DictionaryStructure(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix)
|
||||
{
|
||||
std::string structure_prefix = config_prefix + ".structure";
|
||||
@ -79,7 +70,8 @@ DictionaryStructure::DictionaryStructure(const Poco::Util::AbstractConfiguration
|
||||
|
||||
if (has_id)
|
||||
{
|
||||
id.emplace(config, structure_prefix + ".id");
|
||||
static constexpr auto id_default_type = "UInt64";
|
||||
id.emplace(makeDictionaryTypedSpecialAttribute(config, structure_prefix + ".id", id_default_type));
|
||||
}
|
||||
else if (has_key)
|
||||
{
|
||||
|
||||
@ -89,14 +89,6 @@ constexpr void callOnDictionaryAttributeType(AttributeUnderlyingType type, F &&
|
||||
});
|
||||
}
|
||||
|
||||
struct DictionarySpecialAttribute final
|
||||
{
|
||||
const std::string name;
|
||||
const std::string expression;
|
||||
|
||||
DictionarySpecialAttribute(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix);
|
||||
};
|
||||
|
||||
struct DictionaryTypedSpecialAttribute final
|
||||
{
|
||||
const std::string name;
|
||||
@ -108,7 +100,7 @@ struct DictionaryTypedSpecialAttribute final
|
||||
/// Name of identifier plus list of attributes
|
||||
struct DictionaryStructure final
|
||||
{
|
||||
std::optional<DictionarySpecialAttribute> id;
|
||||
std::optional<DictionaryTypedSpecialAttribute> id;
|
||||
std::optional<std::vector<DictionaryAttribute>> key;
|
||||
std::vector<DictionaryAttribute> attributes;
|
||||
std::unordered_map<std::string, size_t> attribute_name_to_index;
|
||||
|
||||
@ -382,6 +382,15 @@ void buildPrimaryKeyConfiguration(
|
||||
name_element->appendChild(name);
|
||||
|
||||
buildAttributeExpressionIfNeeded(doc, id_element, dict_attr);
|
||||
|
||||
if (!dict_attr->type)
|
||||
return;
|
||||
|
||||
AutoPtr<Element> type_element(doc->createElement("type"));
|
||||
id_element->appendChild(type_element);
|
||||
|
||||
AutoPtr<Text> type(doc->createTextNode(queryToString(dict_attr->type)));
|
||||
type_element->appendChild(type);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -150,9 +150,9 @@ using FunctionCRC64ECMA = FunctionCRC<CRC64ECMAImpl>;
|
||||
|
||||
REGISTER_FUNCTION(CRC)
|
||||
{
|
||||
factory.registerFunction<FunctionCRC32ZLib>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionCRC32IEEE>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionCRC64ECMA>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionCRC32ZLib>({}, FunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction<FunctionCRC32IEEE>({}, FunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction<FunctionCRC64ECMA>({}, FunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -137,10 +137,10 @@ FunctionOverloadResolverPtr createInternalCastOverloadResolver(CastType type, st
|
||||
|
||||
REGISTER_FUNCTION(CastOverloadResolvers)
|
||||
{
|
||||
factory.registerFunction("_CAST", [](ContextPtr context){ return CastOverloadResolverImpl::create(context, CastType::nonAccurate, true, {}); }, {}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("_CAST", [](ContextPtr context){ return CastOverloadResolverImpl::create(context, CastType::nonAccurate, true, {}); }, {}, FunctionFactory::Case::Insensitive);
|
||||
/// Note: "internal" (not affected by null preserving setting) versions of accurate cast functions are unneeded.
|
||||
|
||||
factory.registerFunction("CAST", [](ContextPtr context){ return CastOverloadResolverImpl::create(context, CastType::nonAccurate, false, {}); }, {}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("CAST", [](ContextPtr context){ return CastOverloadResolverImpl::create(context, CastType::nonAccurate, false, {}); }, {}, FunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction("accurateCast", [](ContextPtr context){ return CastOverloadResolverImpl::create(context, CastType::accurate, false, {}); }, {});
|
||||
factory.registerFunction("accurateCastOrNull", [](ContextPtr context){ return CastOverloadResolverImpl::create(context, CastType::accurateOrNull, false, {}); }, {});
|
||||
}
|
||||
|
||||
@ -116,7 +116,7 @@ private:
|
||||
|
||||
REGISTER_FUNCTION(Char)
|
||||
{
|
||||
factory.registerFunction<FunctionChar>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionChar>({}, FunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -46,7 +46,7 @@ public:
|
||||
|
||||
REGISTER_FUNCTION(FQDN)
|
||||
{
|
||||
factory.registerFunction<FunctionFQDN>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionFQDN>({}, FunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("fullHostName", "FQDN");
|
||||
}
|
||||
|
||||
|
||||
@ -31,7 +31,7 @@ void FunctionFactory::registerFunction(
|
||||
const std::string & name,
|
||||
FunctionCreator creator,
|
||||
FunctionDocumentation doc,
|
||||
CaseSensitiveness case_sensitiveness)
|
||||
Case case_sensitiveness)
|
||||
{
|
||||
if (!functions.emplace(name, FunctionFactoryData{creator, doc}).second)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "FunctionFactory: the function name '{}' is not unique", name);
|
||||
@ -41,7 +41,7 @@ void FunctionFactory::registerFunction(
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "FunctionFactory: the function name '{}' is already registered as alias",
|
||||
name);
|
||||
|
||||
if (case_sensitiveness == CaseInsensitive)
|
||||
if (case_sensitiveness == Case::Insensitive)
|
||||
{
|
||||
if (!case_insensitive_functions.emplace(function_name_lowercase, FunctionFactoryData{creator, doc}).second)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "FunctionFactory: the case insensitive function name '{}' is not unique",
|
||||
@ -54,7 +54,7 @@ void FunctionFactory::registerFunction(
|
||||
const std::string & name,
|
||||
FunctionSimpleCreator creator,
|
||||
FunctionDocumentation doc,
|
||||
CaseSensitiveness case_sensitiveness)
|
||||
Case case_sensitiveness)
|
||||
{
|
||||
registerFunction(name, [my_creator = std::move(creator)](ContextPtr context)
|
||||
{
|
||||
|
||||
@ -30,7 +30,7 @@ public:
|
||||
static FunctionFactory & instance();
|
||||
|
||||
template <typename Function>
|
||||
void registerFunction(FunctionDocumentation doc = {}, CaseSensitiveness case_sensitiveness = CaseSensitive)
|
||||
void registerFunction(FunctionDocumentation doc = {}, Case case_sensitiveness = Case::Sensitive)
|
||||
{
|
||||
registerFunction<Function>(Function::name, std::move(doc), case_sensitiveness);
|
||||
}
|
||||
@ -56,13 +56,13 @@ public:
|
||||
const std::string & name,
|
||||
FunctionCreator creator,
|
||||
FunctionDocumentation doc = {},
|
||||
CaseSensitiveness case_sensitiveness = CaseSensitive);
|
||||
Case case_sensitiveness = Case::Sensitive);
|
||||
|
||||
void registerFunction(
|
||||
const std::string & name,
|
||||
FunctionSimpleCreator creator,
|
||||
FunctionDocumentation doc = {},
|
||||
CaseSensitiveness case_sensitiveness = CaseSensitive);
|
||||
Case case_sensitiveness = Case::Sensitive);
|
||||
|
||||
FunctionDocumentation getDocumentation(const std::string & name) const;
|
||||
|
||||
@ -79,7 +79,7 @@ private:
|
||||
String getFactoryName() const override { return "FunctionFactory"; }
|
||||
|
||||
template <typename Function>
|
||||
void registerFunction(const std::string & name, FunctionDocumentation doc = {}, CaseSensitiveness case_sensitiveness = CaseSensitive)
|
||||
void registerFunction(const std::string & name, FunctionDocumentation doc = {}, Case case_sensitiveness = Case::Sensitive)
|
||||
{
|
||||
registerFunction(name, &Function::create, std::move(doc), case_sensitiveness);
|
||||
}
|
||||
|
||||
@ -445,8 +445,7 @@ The function returns a value of type String.
|
||||
{"with specified seed", "SELECT generateRandomStructure(1, 42)", "c1 UInt128"},
|
||||
},
|
||||
.categories{"Random"}
|
||||
},
|
||||
FunctionFactory::CaseSensitive);
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -728,10 +728,10 @@ public:
|
||||
|
||||
REGISTER_FUNCTION(BinaryRepr)
|
||||
{
|
||||
factory.registerFunction<EncodeToBinaryRepresentation<HexImpl>>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<DecodeFromBinaryRepresentation<UnhexImpl>>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<EncodeToBinaryRepresentation<BinImpl>>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<DecodeFromBinaryRepresentation<UnbinImpl>>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<EncodeToBinaryRepresentation<HexImpl>>({}, FunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction<DecodeFromBinaryRepresentation<UnhexImpl>>({}, FunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction<EncodeToBinaryRepresentation<BinImpl>>({}, FunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction<DecodeFromBinaryRepresentation<UnbinImpl>>({}, FunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1169,10 +1169,10 @@ REGISTER_FUNCTION(Coding)
|
||||
factory.registerFunction<FunctionIPv6StringToNum<IPStringToNumExceptionMode::Null>>();
|
||||
|
||||
/// MySQL compatibility aliases:
|
||||
factory.registerAlias("INET_ATON", FunctionIPv4StringToNum<IPStringToNumExceptionMode::Throw>::name, FunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("INET6_NTOA", FunctionIPv6NumToString::name, FunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("INET6_ATON", FunctionIPv6StringToNum<IPStringToNumExceptionMode::Throw>::name, FunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("INET_NTOA", NameFunctionIPv4NumToString::name, FunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("INET_ATON", FunctionIPv4StringToNum<IPStringToNumExceptionMode::Throw>::name, FunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("INET6_NTOA", FunctionIPv6NumToString::name, FunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("INET6_ATON", FunctionIPv6StringToNum<IPStringToNumExceptionMode::Throw>::name, FunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("INET_NTOA", NameFunctionIPv4NumToString::name, FunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -180,8 +180,7 @@ An optional second argument can be passed to specify a timezone for the timestam
|
||||
{"ulid", "SELECT ULIDStringToDateTime(generateULID())", ""},
|
||||
{"timezone", "SELECT ULIDStringToDateTime(generateULID(), 'Asia/Istanbul')", ""}},
|
||||
.categories{"ULID"}
|
||||
},
|
||||
FunctionFactory::CaseSensitive);
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -496,8 +496,8 @@ This function accepts a UUID and returns a FixedString(16) as its binary represe
|
||||
│ 612f3c40-5d3b-217e-707b-6a546a3d7b29 │ a/<@];!~p{jTj={) │ @</a];!~p{jTj={) │
|
||||
└──────────────────────────────────────┴──────────────────┴──────────────────┘
|
||||
)"}},
|
||||
.categories{"UUID"}},
|
||||
FunctionFactory::CaseSensitive);
|
||||
.categories{"UUID"}});
|
||||
|
||||
|
||||
factory.registerFunction<FunctionUUIDv7ToDateTime>(
|
||||
FunctionDocumentation{
|
||||
@ -509,8 +509,7 @@ An optional second argument can be passed to specify a timezone for the timestam
|
||||
.examples{
|
||||
{"uuid","select UUIDv7ToDateTime(generateUUIDv7())", ""},
|
||||
{"uuid","select generateUUIDv7() as uuid, UUIDv7ToDateTime(uuid), UUIDv7ToDateTime(uuid, 'America/New_York')", ""}},
|
||||
.categories{"UUID"}},
|
||||
FunctionFactory::CaseSensitive);
|
||||
.categories{"UUID"}});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -5224,7 +5224,7 @@ REGISTER_FUNCTION(Conversion)
|
||||
/// MySQL compatibility alias. Cannot be registered as alias,
|
||||
/// because we don't want it to be normalized to toDate in queries,
|
||||
/// otherwise CREATE DICTIONARY query breaks.
|
||||
factory.registerFunction("DATE", &FunctionToDate::create, {}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("DATE", &FunctionToDate::create, {}, FunctionFactory::Case::Insensitive);
|
||||
|
||||
factory.registerFunction<FunctionToDate32>();
|
||||
factory.registerFunction<FunctionToDateTime>();
|
||||
|
||||
@ -1184,7 +1184,7 @@ private:
|
||||
|
||||
if (icolumn->size() != vec_to.size())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Argument column '{}' size {} doesn't match result column size {} of function {}",
|
||||
icolumn->getName(), icolumn->size(), vec_to.size(), getName());
|
||||
icolumn->getName(), icolumn->size(), vec_to.size(), getName());
|
||||
|
||||
if constexpr (Keyed)
|
||||
if (key_cols.size() != vec_to.size() && key_cols.size() != 1)
|
||||
@ -1223,6 +1223,9 @@ private:
|
||||
else executeGeneric<first>(key_cols, icolumn, vec_to);
|
||||
}
|
||||
|
||||
/// Return a fixed random-looking magic number when input is empty.
|
||||
static constexpr auto filler = 0xe28dbde7fe22e41c;
|
||||
|
||||
void executeForArgument(const KeyColumnsType & key_cols, const IDataType * type, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, bool & is_first) const
|
||||
{
|
||||
/// Flattening of tuples.
|
||||
@ -1231,6 +1234,11 @@ private:
|
||||
const auto & tuple_columns = tuple->getColumns();
|
||||
const DataTypes & tuple_types = typeid_cast<const DataTypeTuple &>(*type).getElements();
|
||||
size_t tuple_size = tuple_columns.size();
|
||||
|
||||
if (0 == tuple_size && is_first)
|
||||
for (auto & hash : vec_to)
|
||||
hash = static_cast<ToType>(filler);
|
||||
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
executeForArgument(key_cols, tuple_types[i].get(), tuple_columns[i].get(), vec_to, is_first);
|
||||
}
|
||||
@ -1239,6 +1247,11 @@ private:
|
||||
const auto & tuple_columns = tuple_const->getColumns();
|
||||
const DataTypes & tuple_types = typeid_cast<const DataTypeTuple &>(*type).getElements();
|
||||
size_t tuple_size = tuple_columns.size();
|
||||
|
||||
if (0 == tuple_size && is_first)
|
||||
for (auto & hash : vec_to)
|
||||
hash = static_cast<ToType>(filler);
|
||||
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
{
|
||||
auto tmp = ColumnConst::create(tuple_columns[i], column->size());
|
||||
@ -1300,10 +1313,7 @@ public:
|
||||
constexpr size_t first_data_argument = Keyed;
|
||||
|
||||
if (arguments.size() <= first_data_argument)
|
||||
{
|
||||
/// Return a fixed random-looking magic number when input is empty
|
||||
vec_to.assign(input_rows_count, static_cast<ToType>(0xe28dbde7fe22e41c));
|
||||
}
|
||||
vec_to.assign(input_rows_count, static_cast<ToType>(filler));
|
||||
|
||||
KeyColumnsType key_cols{};
|
||||
if constexpr (Keyed)
|
||||
|
||||
@ -41,8 +41,7 @@ REGISTER_FUNCTION(Hashing)
|
||||
.description="Calculates value of XXH3 64-bit hash function. Refer to https://github.com/Cyan4973/xxHash for detailed documentation.",
|
||||
.examples{{"hash", "SELECT xxh3('ClickHouse')", ""}},
|
||||
.categories{"Hash"}
|
||||
},
|
||||
FunctionFactory::CaseSensitive);
|
||||
});
|
||||
|
||||
factory.registerFunction<FunctionWyHash64>();
|
||||
|
||||
|
||||
@ -29,7 +29,7 @@ REGISTER_FUNCTION(Logical)
|
||||
factory.registerFunction<FunctionAnd>();
|
||||
factory.registerFunction<FunctionOr>();
|
||||
factory.registerFunction<FunctionXor>();
|
||||
factory.registerFunction<FunctionNot>({}, FunctionFactory::CaseInsensitive); /// Operator NOT(x) can be parsed as a function.
|
||||
factory.registerFunction<FunctionNot>({}, FunctionFactory::Case::Insensitive); /// Operator NOT(x) can be parsed as a function.
|
||||
}
|
||||
|
||||
namespace ErrorCodes
|
||||
|
||||
@ -99,8 +99,8 @@ using FunctionSubDate = FunctionOpDate<SubDate>;
|
||||
|
||||
REGISTER_FUNCTION(AddInterval)
|
||||
{
|
||||
factory.registerFunction<FunctionAddDate>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionSubDate>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionAddDate>({}, FunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction<FunctionSubDate>({}, FunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -7,16 +7,16 @@ namespace DB
|
||||
|
||||
REGISTER_FUNCTION(Round)
|
||||
{
|
||||
factory.registerFunction<FunctionRound>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionRoundBankers>({}, FunctionFactory::CaseSensitive);
|
||||
factory.registerFunction<FunctionFloor>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionCeil>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionTrunc>({}, FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionRound>({}, FunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction<FunctionRoundBankers>({}, FunctionFactory::Case::Sensitive);
|
||||
factory.registerFunction<FunctionFloor>({}, FunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction<FunctionCeil>({}, FunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction<FunctionTrunc>({}, FunctionFactory::Case::Insensitive);
|
||||
factory.registerFunction<FunctionRoundDown>();
|
||||
|
||||
/// Compatibility aliases.
|
||||
factory.registerAlias("ceiling", "ceil", FunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("truncate", "trunc", FunctionFactory::CaseInsensitive);
|
||||
factory.registerAlias("ceiling", "ceil", FunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("truncate", "trunc", FunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user