thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into read-cgroup-memory-usage-async-metrics

Browse Source
This commit is contained in:
Antonio Andelic 2024-07-23 16:09:21 +02:00
commit 9d92bcff46
193 changed files with 4347 additions and 1177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
.github/actions/release/action.yml vendored
View File

@ -62,8 +62,8 @@ runs:
if: ${{ inputs.type == 'patch' }}
shell: bash
run: |
python3 ./tests/ci/create_release.py --set-progress-started --progress "update ChangeLog"
[ "$(git branch --show-current)" != "master" ] && echo "not on the master" && exit 1
git checkout master
python3 ./tests/ci/create_release.py --set-progress-started --progress "update changelog, docker version, security"
echo "List versions"
./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
echo "Update docker version"
@ -96,17 +96,13 @@ runs:
Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
### Changelog category (leave one):
- Not for changelog (changelog entry is not required)
- name: Reset changes if Dry-run
if: ${{ inputs.dry-run }}
- name: Complete previous steps and Restore git state
if: ${{ inputs.type == 'patch' }}
shell: bash
run: |
git reset --hard HEAD
- name: Checkout back to GITHUB_REF
shell: bash
run: |
git checkout "$GITHUB_REF_NAME"
# set current progress to OK
python3 ./tests/ci/create_release.py --set-progress-completed
git reset --hard HEAD
git checkout "$GITHUB_REF_NAME"
- name: Create GH Release
shell: bash
if: ${{ inputs.type == 'patch' }}
@ -146,24 +142,23 @@ runs:
if: ${{ inputs.type == 'patch' }}
shell: bash
run: |
python3 ./tests/ci/create_release.py --set-progress-started --progress "docker server release"
cd "./tests/ci"
python3 ./create_release.py --set-progress-started --progress "docker server release"
export CHECK_NAME="Docker server image"
python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
python3 ./tests/ci/create_release.py --set-progress-completed
python3 ./create_release.py --set-progress-completed
- name: Docker clickhouse/clickhouse-keeper building
if: ${{ inputs.type == 'patch' }}
shell: bash
run: |
python3 ./tests/ci/create_release.py --set-progress-started --progress "docker keeper release"
cd "./tests/ci"
python3 ./create_release.py --set-progress-started --progress "docker keeper release"
export CHECK_NAME="Docker keeper image"
python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
python3 ./tests/ci/create_release.py --set-progress-completed
- name: Set Release progress completed
python3 ./create_release.py --set-progress-completed
- name: Set current Release progress to Completed with OK
shell: bash
run: |
# If we here - set completed status, to post proper Slack OK or FAIL message in the next step
python3 ./tests/ci/create_release.py --set-progress-started --progress "completed"
python3 ./tests/ci/create_release.py --set-progress-completed
- name: Post Slack Message

14
.github/workflows/backport_branches.yml vendored
View File

@ -241,8 +241,9 @@ jobs:
runner_type: stress-tester
data: ${{ needs.RunConfig.outputs.data }}
FinishCheck:
if: ${{ !failure() && !cancelled() }}
if: ${{ !cancelled() }}
needs:
- RunConfig
- Builds_Report
- FunctionalStatelessTestAsan
- FunctionalStatefulTestDebug
@ -257,6 +258,7 @@ jobs:
with:
clear-repository: true
- name: Finish label
if: ${{ !failure() }}
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
# update mergeable check
@ -264,3 +266,13 @@ jobs:
# update overall ci report
python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
python3 merge_pr.py
- name: Check Workflow results
run: |
export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
${{ toJson(needs) }}
EOF
echo "::group::Workflow results"
python3 -m json.tool "$WORKFLOW_RESULT_FILE"
echo "::endgroup::"
python3 ./tests/ci/ci_buddy.py --check-wf-status

38
.github/workflows/master.yml vendored
View File

@ -121,34 +121,6 @@ jobs:
runner_type: style-checker-aarch64
data: ${{ needs.RunConfig.outputs.data }}
MarkReleaseReady:
if: ${{ !failure() && !cancelled() }}
needs: [RunConfig, Builds_1, Builds_2]
runs-on: [self-hosted, style-checker-aarch64]
steps:
- name: Debug
run: |
echo need with different filters
cat << 'EOF'
${{ toJSON(needs) }}
${{ toJSON(needs.*.result) }}
no failures ${{ !contains(needs.*.result, 'failure') }}
no skips ${{ !contains(needs.*.result, 'skipped') }}
no both ${{ !(contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
EOF
- name: Not ready
# fail the job to be able to restart it
if: ${{ contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure') }}
run: exit 1
- name: Check out repository code
if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
uses: ClickHouse/checkout@v1
- name: Mark Commit Release Ready
if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
python3 mark_release_ready.py
FinishCheck:
if: ${{ !cancelled() }}
needs: [RunConfig, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3]
@ -160,3 +132,13 @@ jobs:
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
- name: Check Workflow results
run: |
export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
${{ toJson(needs) }}
EOF
echo "::group::Workflow results"
python3 -m json.tool "$WORKFLOW_RESULT_FILE"
echo "::endgroup::"
python3 ./tests/ci/ci_buddy.py --check-wf-status

13
.github/workflows/merge_queue.yml vendored
View File

@ -93,7 +93,7 @@ jobs:
data: ${{ needs.RunConfig.outputs.data }}
CheckReadyForMerge:
if: ${{ !cancelled() && needs.StyleCheck.result == 'success' }}
if: ${{ !cancelled() }}
# Test_2 or Test_3 must not have jobs required for Mergeable check
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Tests_1]
runs-on: [self-hosted, style-checker-aarch64]
@ -101,6 +101,17 @@ jobs:
- name: Check out repository code
uses: ClickHouse/checkout@v1
- name: Check and set merge status
if: ${{ needs.StyleCheck.result == 'success' }}
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
- name: Check Workflow results
run: |
export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
${{ toJson(needs) }}
EOF
echo "::group::Workflow results"
python3 -m json.tool "$WORKFLOW_RESULT_FILE"
echo "::endgroup::"
python3 ./tests/ci/ci_buddy.py --check-wf-status

17
.github/workflows/nightly.yml vendored
View File

@ -44,3 +44,20 @@ jobs:
with:
data: "${{ needs.RunConfig.outputs.data }}"
set_latest: true
CheckWorkflow:
if: ${{ !cancelled() }}
needs: [RunConfig, BuildDockers]
runs-on: [self-hosted, style-checker-aarch64]
steps:
- name: Check out repository code
uses: ClickHouse/checkout@v1
- name: Check Workflow results
run: |
export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
${{ toJson(needs) }}
EOF
echo "::group::Workflow results"
python3 -m json.tool "$WORKFLOW_RESULT_FILE"
echo "::endgroup::"
python3 ./tests/ci/ci_buddy.py --check-wf-status

13
.github/workflows/pull_request.yml vendored
View File

@ -151,7 +151,7 @@ jobs:
data: ${{ needs.RunConfig.outputs.data }}
CheckReadyForMerge:
if: ${{ !cancelled() && needs.StyleCheck.result == 'success' }}
if: ${{ !cancelled() }}
# Test_2 or Test_3 must not have jobs required for Mergeable check
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1]
runs-on: [self-hosted, style-checker-aarch64]
@ -161,9 +161,20 @@ jobs:
with:
filter: tree:0
- name: Check and set merge status
if: ${{ needs.StyleCheck.result == 'success' }}
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
- name: Check Workflow results
run: |
export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
${{ toJson(needs) }}
EOF
echo "::group::Workflow results"
python3 -m json.tool "$WORKFLOW_RESULT_FILE"
echo "::endgroup::"
python3 ./tests/ci/ci_buddy.py --check-wf-status
################################# Stage Final #################################
#

14
.github/workflows/release_branches.yml vendored
View File

@ -441,8 +441,9 @@ jobs:
runner_type: stress-tester
data: ${{ needs.RunConfig.outputs.data }}
FinishCheck:
if: ${{ !failure() && !cancelled() }}
if: ${{ !cancelled() }}
needs:
- RunConfig
- DockerServerImage
- DockerKeeperImage
- Builds_Report
@ -478,9 +479,20 @@ jobs:
with:
clear-repository: true
- name: Finish label
if: ${{ !failure() }}
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
# update mergeable check
python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
# update overall ci report
python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
- name: Check Workflow results
run: |
export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
${{ toJson(needs) }}
EOF
echo "::group::Workflow results"
python3 -m json.tool "$WORKFLOW_RESULT_FILE"
echo "::endgroup::"
python3 ./tests/ci/ci_buddy.py --check-wf-status

6
.yamllint
View File

@ -14,3 +14,9 @@ rules:
comments:
min-spaces-from-content: 1
document-start: disable
colons: disable
indentation: disable
line-length: disable
trailing-spaces: disable
truthy: disable
new-line-at-end-of-file: disable

13
base/base/defines.h
View File

@ -87,10 +87,13 @@
# define ASAN_POISON_MEMORY_REGION(a, b)
#endif
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER) || defined(UNDEFINED_BEHAVIOR_SANITIZER)
#define ABORT_ON_LOGICAL_ERROR
#endif
/// We used to have only ABORT_ON_LOGICAL_ERROR macro, but most of its uses were actually in places where we didn't care about logical errors
/// but wanted to check exactly if the current build type is debug or with sanitizer. This new macro is introduced to fix those places.
#if !defined(DEBUG_OR_SANITIZER_BUILD)
# if !defined(NDEBUG) || defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER) \
|| defined(UNDEFINED_BEHAVIOR_SANITIZER)
# define DEBUG_OR_SANITIZER_BUILD
# endif
#endif
/// chassert(x) is similar to assert(x), but:
@ -101,7 +104,7 @@
/// Also it makes sense to call abort() instead of __builtin_unreachable() in debug builds,
/// because SIGABRT is easier to debug than SIGTRAP (the second one makes gdb crazy)
#if !defined(chassert)
#if defined(ABORT_ON_LOGICAL_ERROR)
# if defined(DEBUG_OR_SANITIZER_BUILD)
// clang-format off
#include <base/types.h>
namespace DB

33
docs/en/operations/settings/settings-users.md
View File

@ -22,6 +22,21 @@ Structure of the `users` section:
<!-- Or -->
<password_sha256_hex></password_sha256_hex>
<ssh_keys>
<ssh_key>
<type>ssh-ed25519</type>
<base64_key>AAAAC3NzaC1lZDI1NTE5AAAAIDNf0r6vRl24Ix3tv2IgPmNPO2ATa2krvt80DdcTatLj</base64_key>
</ssh_key>
<ssh_key>
<type>ecdsa-sha2-nistp256</type>
<base64_key>AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNxeV2uN5UY6CUbCzTA1rXfYimKQA5ivNIqxdax4bcMXz4D0nSk2l5E1TkR5mG8EBWtmExSPbcEPJ8V7lyWWbA8=</base64_key>
</ssh_key>
<ssh_key>
<type>ssh-rsa</type>
<base64_key>AAAAB3NzaC1yc2EAAAADAQABAAABgQCpgqL1SHhPVBOTFlOm0pu+cYBbADzC2jL41sPMawYCJHDyHuq7t+htaVVh2fRgpAPmSEnLEC2d4BEIKMtPK3bfR8plJqVXlLt6Q8t4b1oUlnjb3VPA9P6iGcW7CV1FBkZQEVx8ckOfJ3F+kI5VsrRlEDgiecm/C1VPl0/9M2llW/mPUMaD65cM9nlZgM/hUeBrfxOEqM11gDYxEZm1aRSbZoY4dfdm3vzvpSQ6lrCrkjn3X2aSmaCLcOWJhfBWMovNDB8uiPuw54g3ioZ++qEQMlfxVsqXDGYhXCrsArOVuW/5RbReO79BvXqdssiYShfwo+GhQ0+aLWMIW/jgBkkqx/n7uKLzCMX7b2F+aebRYFh+/QXEj7SnihdVfr9ud6NN3MWzZ1ltfIczlEcFLrLJ1Yq57wW6wXtviWh59WvTWFiPejGjeSjjJyqqB49tKdFVFuBnIU5u/bch2DXVgiAEdQwUrIp1ACoYPq22HFFAYUJrL32y7RxX3PGzuAv3LOc=</base64_key>
</ssh_key>
</ssh_keys>
<access_management>0|1</access_management>
<networks incl="networks" replace="replace">
@ -79,6 +94,24 @@ Password can be specified in plaintext or in SHA256 (hex format).
The first line of the result is the password. The second line is the corresponding double SHA1 hash.
### username/ssh-key {#user-sshkey}
This setting allows authenticating with SSH keys.
Given a SSH key (as generated by `ssh-keygen`) like
```
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDNf0r6vRl24Ix3tv2IgPmNPO2ATa2krvt80DdcTatLj john@example.com
```
The `ssh_key` element is expected to be
```
<ssh_key>
<type>ssh-ed25519</type>
<base64_key>AAAAC3NzaC1lZDI1NTE5AAAAIDNf0r6vRl24Ix3tv2IgPmNPO2ATa2krvt80DdcTatLj</base64_key>
</ssh_key>
```
Substitute `ssh-ed25519` with `ssh-rsa` or `ecdsa-sha2-nistp256` for the other supported algorithms.
### access_management {#access_management-user-setting}
This setting enables or disables using of SQL-driven [access control and account management](../../guides/sre/user-management/index.md#access-control) for the user.

5
docs/en/sql-reference/statements/select/join.md
View File

@ -297,7 +297,7 @@ Algorithm requires the special column in tables. This column:
- Must contain an ordered sequence.
- Can be one of the following types: [Int, UInt](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md), [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md), [Decimal](../../../sql-reference/data-types/decimal.md).
- Can’t be the only column in the `JOIN` clause.
- For `hash` join algorithm it can’t be the only column in the `JOIN` clause.
Syntax `ASOF JOIN ... ON`:
@ -337,7 +337,8 @@ For example, consider the following tables:
`ASOF JOIN` can take the timestamp of a user event from `table_1` and find an event in `table_2` where the timestamp is closest to the timestamp of the event from `table_1` corresponding to the closest match condition. Equal timestamp values are the closest if available. Here, the `user_id` column can be used for joining on equality and the `ev_time` column can be used for joining on the closest match. In our example, `event_1_1` can be joined with `event_2_1` and `event_1_2` can be joined with `event_2_3`, but `event_2_2` can’t be joined.
:::note
`ASOF` join is **not** supported in the [Join](../../../engines/table-engines/special/join.md) table engine.
`ASOF JOIN` is supported only by `hash` and `full_sorting_merge` join algorithms.
It's **not** supported in the [Join](../../../engines/table-engines/special/join.md) table engine.
:::
## PASTE JOIN Usage

46
src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp
View File

@ -68,6 +68,41 @@ QueryTreeNodePtr findEqualsFunction(const QueryTreeNodes & nodes)
return nullptr;
}
/// Checks if the node is combination of isNull and notEquals functions of two the same arguments
bool matchIsNullOfTwoArgs(const QueryTreeNodes & nodes, QueryTreeNodePtr & lhs, QueryTreeNodePtr & rhs)
{
QueryTreeNodePtrWithHashSet all_arguments;
for (const auto & node : nodes)
{
const auto * func_node = node->as<FunctionNode>();
if (!func_node)
return false;
const auto & arguments = func_node->getArguments().getNodes();
if (func_node->getFunctionName() == "isNull" && arguments.size() == 1)
all_arguments.insert(QueryTreeNodePtrWithHash(arguments[0]));
else if (func_node->getFunctionName() == "notEquals" && arguments.size() == 2)
{
if (arguments[0]->isEqual(*arguments[1]))
return false;
all_arguments.insert(QueryTreeNodePtrWithHash(arguments[0]));
all_arguments.insert(QueryTreeNodePtrWithHash(arguments[1]));
}
else
return false;
if (all_arguments.size() > 2)
return false;
}
if (all_arguments.size() != 2)
return false;
lhs = all_arguments.begin()->node;
rhs = std::next(all_arguments.begin())->node;
return true;
}
bool isBooleanConstant(const QueryTreeNodePtr & node, bool expected_value)
{
const auto * constant_node = node->as<ConstantNode>();
@ -213,11 +248,14 @@ private:
else if (func_name == "and")
{
const auto & and_arguments = argument_function->getArguments().getNodes();
bool all_are_is_null = and_arguments.size() == 2 && isNodeFunction(and_arguments[0], "isNull") && isNodeFunction(and_arguments[1], "isNull");
if (all_are_is_null)
QueryTreeNodePtr is_null_lhs_arg;
QueryTreeNodePtr is_null_rhs_arg;
if (matchIsNullOfTwoArgs(and_arguments, is_null_lhs_arg, is_null_rhs_arg))
{
is_null_argument_to_indices[getFunctionArgument(and_arguments.front(), 0)].push_back(or_operands.size() - 1);
is_null_argument_to_indices[getFunctionArgument(and_arguments.back(), 0)].push_back(or_operands.size() - 1);
is_null_argument_to_indices[is_null_lhs_arg].push_back(or_operands.size() - 1);
is_null_argument_to_indices[is_null_rhs_arg].push_back(or_operands.size() - 1);
continue;
}
/// Expression `a = b AND (a IS NOT NULL) AND true AND (b IS NOT NULL)` we can be replaced with `a = b`

6
src/Analyzer/QueryTreePassManager.cpp
View File

@ -62,7 +62,7 @@ namespace ErrorCodes
namespace
{
#if defined(ABORT_ON_LOGICAL_ERROR)
#if defined(DEBUG_OR_SANITIZER_BUILD)
/** This visitor checks if Query Tree structure is valid after each pass
* in debug build.
@ -183,7 +183,7 @@ void QueryTreePassManager::run(QueryTreeNodePtr query_tree_node)
for (size_t i = 0; i < passes_size; ++i)
{
passes[i]->run(query_tree_node, current_context);
#if defined(ABORT_ON_LOGICAL_ERROR)
#if defined(DEBUG_OR_SANITIZER_BUILD)
ValidationChecker(passes[i]->getName()).visit(query_tree_node);
#endif
}
@ -208,7 +208,7 @@ void QueryTreePassManager::run(QueryTreeNodePtr query_tree_node, size_t up_to_pa
for (size_t i = 0; i < up_to_pass_index; ++i)
{
passes[i]->run(query_tree_node, current_context);
#if defined(ABORT_ON_LOGICAL_ERROR)
#if defined(DEBUG_OR_SANITIZER_BUILD)
ValidationChecker(passes[i]->getName()).visit(query_tree_node);
#endif
}

2
src/Backups/BackupUtils.cpp
View File

@ -105,7 +105,7 @@ bool compareRestoredTableDef(const IAST & restored_table_create_query, const IAS
auto new_query = query.clone();
adjustCreateQueryForBackup(new_query, global_context);
ASTCreateQuery & create = typeid_cast<ASTCreateQuery &>(*new_query);
create.setUUID({});
create.resetUUIDs();
create.if_not_exists = false;
return new_query;
};

6
src/Backups/RestoreCoordinationLocal.cpp
View File

@ -1,4 +1,5 @@
#include <Backups/RestoreCoordinationLocal.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/formatAST.h>
#include <Common/logger_useful.h>
@ -67,7 +68,7 @@ void RestoreCoordinationLocal::generateUUIDForTable(ASTCreateQuery & create_quer
auto it = create_query_uuids.find(query_str);
if (it != create_query_uuids.end())
{
create_query.setUUID(it->second);
it->second.copyToQuery(create_query);
return true;
}
return false;
@ -79,7 +80,8 @@ void RestoreCoordinationLocal::generateUUIDForTable(ASTCreateQuery & create_quer
return;
}
auto new_uuids = create_query.generateRandomUUID(/* always_generate_new_uuid= */ true);
CreateQueryUUIDs new_uuids{create_query, /* generate_random= */ true, /* force_random= */ true};
new_uuids.copyToQuery(create_query);
{
std::lock_guard lock{mutex};

9
src/Backups/RestoreCoordinationLocal.h
View File

@ -1,16 +1,17 @@
#pragma once
#include <Backups/IRestoreCoordination.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/CreateQueryUUIDs.h>
#include <Common/Logger.h>
#include <mutex>
#include <set>
#include <unordered_set>
namespace Poco { class Logger; }
namespace DB
{
class ASTCreateQuery;
/// Implementation of the IRestoreCoordination interface performing coordination in memory.
class RestoreCoordinationLocal : public IRestoreCoordination
@ -55,7 +56,7 @@ private:
std::set<std::pair<String /* database_zk_path */, String /* table_name */>> acquired_tables_in_replicated_databases;
std::unordered_set<String /* table_zk_path */> acquired_data_in_replicated_tables;
std::unordered_map<String, ASTCreateQuery::UUIDs> create_query_uuids;
std::unordered_map<String, CreateQueryUUIDs> create_query_uuids;
std::unordered_set<String /* root_zk_path */> acquired_data_in_keeper_map_tables;
mutable std::mutex mutex;

9
src/Backups/RestoreCoordinationRemote.cpp
View File

@ -3,6 +3,7 @@
#include <Backups/RestoreCoordinationRemote.h>
#include <Backups/BackupCoordinationStageSync.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/CreateQueryUUIDs.h>
#include <Parsers/formatAST.h>
#include <Functions/UserDefined/UserDefinedSQLObjectType.h>
#include <Common/ZooKeeper/KeeperException.h>
@ -269,7 +270,8 @@ bool RestoreCoordinationRemote::acquireInsertingDataForKeeperMap(const String &
void RestoreCoordinationRemote::generateUUIDForTable(ASTCreateQuery & create_query)
{
String query_str = serializeAST(create_query);
String new_uuids_str = create_query.generateRandomUUID(/* always_generate_new_uuid= */ true).toString();
CreateQueryUUIDs new_uuids{create_query, /* generate_random= */ true, /* force_random= */ true};
String new_uuids_str = new_uuids.toString();
auto holder = with_retries.createRetriesControlHolder("generateUUIDForTable");
holder.retries_ctl.retryLoop(
@ -281,11 +283,14 @@ void RestoreCoordinationRemote::generateUUIDForTable(ASTCreateQuery & create_que
Coordination::Error res = zk->tryCreate(path, new_uuids_str, zkutil::CreateMode::Persistent);
if (res == Coordination::Error::ZOK)
{
new_uuids.copyToQuery(create_query);
return;
}
if (res == Coordination::Error::ZNODEEXISTS)
{
create_query.setUUID(ASTCreateQuery::UUIDs::fromString(zk->get(path)));
CreateQueryUUIDs::fromString(zk->get(path)).copyToQuery(create_query);
return;
}

4
src/Columns/ColumnAggregateFunction.cpp
View File

@ -267,7 +267,7 @@ bool ColumnAggregateFunction::structureEquals(const IColumn & to) const
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnAggregateFunction::insertRangeFrom(const IColumn & from, size_t start, size_t length)
#else
void ColumnAggregateFunction::doInsertRangeFrom(const IColumn & from, size_t start, size_t length)
@ -465,7 +465,7 @@ void ColumnAggregateFunction::insertFromWithOwnership(const IColumn & from, size
insertMergeFrom(from, n);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnAggregateFunction::insertFrom(const IColumn & from, size_t n)
#else
void ColumnAggregateFunction::doInsertFrom(const IColumn & from, size_t n)

6
src/Columns/ColumnAggregateFunction.h
View File

@ -145,7 +145,7 @@ public:
void insertData(const char * pos, size_t length) override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & from, size_t n) override;
#else
using IColumn::insertFrom;
@ -189,7 +189,7 @@ public:
void protect() override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn & from, size_t start, size_t length) override;
#else
void doInsertRangeFrom(const IColumn & from, size_t start, size_t length) override;
@ -212,7 +212,7 @@ public:
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t, size_t, const IColumn &, int) const override
#else
int doCompareAt(size_t, size_t, const IColumn &, int) const override

6
src/Columns/ColumnArray.cpp
View File

@ -336,7 +336,7 @@ bool ColumnArray::tryInsert(const Field & x)
return true;
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnArray::insertFrom(const IColumn & src_, size_t n)
#else
void ColumnArray::doInsertFrom(const IColumn & src_, size_t n)
@ -395,7 +395,7 @@ int ColumnArray::compareAtImpl(size_t n, size_t m, const IColumn & rhs_, int nan
: 1);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int ColumnArray::compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const
#else
int ColumnArray::doCompareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const
@ -542,7 +542,7 @@ void ColumnArray::getExtremes(Field & min, Field & max) const
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnArray::insertRangeFrom(const IColumn & src, size_t start, size_t length)
#else
void ColumnArray::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)

6
src/Columns/ColumnArray.h
View File

@ -84,14 +84,14 @@ public:
void updateHashWithValue(size_t n, SipHash & hash) const override;
WeakHash32 getWeakHash32() const override;
void updateHashFast(SipHash & hash) const override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
#else
void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
#endif
void insert(const Field & x) override;
bool tryInsert(const Field & x) override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & src_, size_t n) override;
#else
void doInsertFrom(const IColumn & src_, size_t n) override;
@ -103,7 +103,7 @@ public:
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
ColumnPtr index(const IColumn & indexes, size_t limit) const override;
template <typename Type> ColumnPtr indexImpl(const PaddedPODArray<Type> & indexes, size_t limit) const;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
#else
int doCompareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;

4
src/Columns/ColumnCompressed.h
View File

@ -86,7 +86,7 @@ public:
bool isDefaultAt(size_t) const override { throwMustBeDecompressed(); }
void insert(const Field &) override { throwMustBeDecompressed(); }
bool tryInsert(const Field &) override { throwMustBeDecompressed(); }
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn &, size_t, size_t) override { throwMustBeDecompressed(); }
#else
void doInsertRangeFrom(const IColumn &, size_t, size_t) override { throwMustBeDecompressed(); }
@ -105,7 +105,7 @@ public:
void expand(const Filter &, bool) override { throwMustBeDecompressed(); }
ColumnPtr permute(const Permutation &, size_t) const override { throwMustBeDecompressed(); }
ColumnPtr index(const IColumn &, size_t) const override { throwMustBeDecompressed(); }
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t, size_t, const IColumn &, int) const override { throwMustBeDecompressed(); }
#else
int doCompareAt(size_t, size_t, const IColumn &, int) const override { throwMustBeDecompressed(); }

8
src/Columns/ColumnConst.h
View File

@ -123,7 +123,7 @@ public:
return data->isNullAt(0);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn &, size_t /*start*/, size_t length) override
#else
void doInsertRangeFrom(const IColumn &, size_t /*start*/, size_t length) override
@ -151,7 +151,7 @@ public:
++s;
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn &, size_t) override
#else
void doInsertFrom(const IColumn &, size_t) override
@ -160,7 +160,7 @@ public:
++s;
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertManyFrom(const IColumn & /*src*/, size_t /* position */, size_t length) override { s += length; }
#else
void doInsertManyFrom(const IColumn & /*src*/, size_t /* position */, size_t length) override { s += length; }
@ -237,7 +237,7 @@ public:
return data->allocatedBytes() + sizeof(s);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t, size_t, const IColumn & rhs, int nan_direction_hint) const override
#else
int doCompareAt(size_t, size_t, const IColumn & rhs, int nan_direction_hint) const override

4
src/Columns/ColumnDecimal.cpp
View File

@ -31,7 +31,7 @@ namespace ErrorCodes
}
template <is_decimal T>
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int ColumnDecimal<T>::compareAt(size_t n, size_t m, const IColumn & rhs_, int) const
#else
int ColumnDecimal<T>::doCompareAt(size_t n, size_t m, const IColumn & rhs_, int) const
@ -333,7 +333,7 @@ void ColumnDecimal<T>::insertData(const char * src, size_t /*length*/)
}
template <is_decimal T>
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnDecimal<T>::insertRangeFrom(const IColumn & src, size_t start, size_t length)
#else
void ColumnDecimal<T>::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)

8
src/Columns/ColumnDecimal.h
View File

@ -55,13 +55,13 @@ public:
void reserve(size_t n) override { data.reserve_exact(n); }
void shrinkToFit() override { data.shrink_to_fit(); }
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & src, size_t n) override { data.push_back(static_cast<const Self &>(src).getData()[n]); }
#else
void doInsertFrom(const IColumn & src, size_t n) override { data.push_back(static_cast<const Self &>(src).getData()[n]); }
#endif
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertManyFrom(const IColumn & src, size_t position, size_t length) override
#else
void doInsertManyFrom(const IColumn & src, size_t position, size_t length) override
@ -76,7 +76,7 @@ public:
void insertManyDefaults(size_t length) override { data.resize_fill(data.size() + length); }
void insert(const Field & x) override { data.push_back(x.get<T>()); }
bool tryInsert(const Field & x) override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
#else
void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
@ -104,7 +104,7 @@ public:
void updateHashWithValue(size_t n, SipHash & hash) const override;
WeakHash32 getWeakHash32() const override;
void updateHashFast(SipHash & hash) const override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
#else
int doCompareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;

8
src/Columns/ColumnDynamic.cpp
View File

@ -215,7 +215,7 @@ bool ColumnDynamic::tryInsert(const DB::Field & x)
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnDynamic::insertFrom(const DB::IColumn & src_, size_t n)
#else
void ColumnDynamic::doInsertFrom(const DB::IColumn & src_, size_t n)
@ -269,7 +269,7 @@ void ColumnDynamic::doInsertFrom(const DB::IColumn & src_, size_t n)
variant_col.insertIntoVariantFrom(string_variant_discr, *tmp_string_column, 0);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnDynamic::insertRangeFrom(const DB::IColumn & src_, size_t start, size_t length)
#else
void ColumnDynamic::doInsertRangeFrom(const DB::IColumn & src_, size_t start, size_t length)
@ -439,7 +439,7 @@ void ColumnDynamic::doInsertRangeFrom(const DB::IColumn & src_, size_t start, si
}
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnDynamic::insertManyFrom(const DB::IColumn & src_, size_t position, size_t length)
#else
void ColumnDynamic::doInsertManyFrom(const DB::IColumn & src_, size_t position, size_t length)
@ -603,7 +603,7 @@ void ColumnDynamic::updateHashWithValue(size_t n, SipHash & hash) const
variant_col.getVariantByGlobalDiscriminator(discr).updateHashWithValue(variant_col.offsetAt(n), hash);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int ColumnDynamic::compareAt(size_t n, size_t m, const DB::IColumn & rhs, int nan_direction_hint) const
#else
int ColumnDynamic::doCompareAt(size_t n, size_t m, const DB::IColumn & rhs, int nan_direction_hint) const

4
src/Columns/ColumnDynamic.h
View File

@ -144,7 +144,7 @@ public:
void insert(const Field & x) override;
bool tryInsert(const Field & x) override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & src_, size_t n) override;
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
@ -221,7 +221,7 @@ public:
return scattered_columns;
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
#else
int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;

6
src/Columns/ColumnFixedString.cpp
View File

@ -74,7 +74,7 @@ bool ColumnFixedString::tryInsert(const Field & x)
return true;
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnFixedString::insertFrom(const IColumn & src_, size_t index)
#else
void ColumnFixedString::doInsertFrom(const IColumn & src_, size_t index)
@ -90,7 +90,7 @@ void ColumnFixedString::doInsertFrom(const IColumn & src_, size_t index)
memcpySmallAllowReadWriteOverflow15(chars.data() + old_size, &src.chars[n * index], n);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnFixedString::insertManyFrom(const IColumn & src, size_t position, size_t length)
#else
void ColumnFixedString::doInsertManyFrom(const IColumn & src, size_t position, size_t length)
@ -225,7 +225,7 @@ size_t ColumnFixedString::estimateCardinalityInPermutedRange(const Permutation &
return elements.size();
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnFixedString::insertRangeFrom(const IColumn & src, size_t start, size_t length)
#else
void ColumnFixedString::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)

8
src/Columns/ColumnFixedString.h
View File

@ -98,13 +98,13 @@ public:
bool tryInsert(const Field & x) override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & src_, size_t index) override;
#else
void doInsertFrom(const IColumn & src_, size_t index) override;
#endif
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
#else
void doInsertManyFrom(const IColumn & src, size_t position, size_t length) override;
@ -137,7 +137,7 @@ public:
void updateHashFast(SipHash & hash) const override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t p1, size_t p2, const IColumn & rhs_, int /*nan_direction_hint*/) const override
#else
int doCompareAt(size_t p1, size_t p2, const IColumn & rhs_, int /*nan_direction_hint*/) const override
@ -156,7 +156,7 @@ public:
size_t estimateCardinalityInPermutedRange(const Permutation & permutation, const EqualRange & equal_range) const override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
#else
void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;

4
src/Columns/ColumnFunction.cpp
View File

@ -72,7 +72,7 @@ ColumnPtr ColumnFunction::cut(size_t start, size_t length) const
return ColumnFunction::create(length, function, capture, is_short_circuit_argument, is_function_compiled);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnFunction::insertFrom(const IColumn & src, size_t n)
#else
void ColumnFunction::doInsertFrom(const IColumn & src, size_t n)
@ -93,7 +93,7 @@ void ColumnFunction::doInsertFrom(const IColumn & src, size_t n)
++elements_size;
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnFunction::insertRangeFrom(const IColumn & src, size_t start, size_t length)
#else
void ColumnFunction::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)

6
src/Columns/ColumnFunction.h
View File

@ -95,12 +95,12 @@ public:
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot insert into {}", getName());
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & src, size_t n) override;
#else
void doInsertFrom(const IColumn & src, size_t n) override;
#endif
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn &, size_t start, size_t length) override;
#else
void doInsertRangeFrom(const IColumn &, size_t start, size_t length) override;
@ -146,7 +146,7 @@ public:
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "popBack is not implemented for {}", getName());
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t, size_t, const IColumn &, int) const override
#else
int doCompareAt(size_t, size_t, const IColumn &, int) const override

6
src/Columns/ColumnLowCardinality.cpp
View File

@ -158,7 +158,7 @@ void ColumnLowCardinality::insertDefault()
idx.insertPosition(getDictionary().getDefaultValueIndex());
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnLowCardinality::insertFrom(const IColumn & src, size_t n)
#else
void ColumnLowCardinality::doInsertFrom(const IColumn & src, size_t n)
@ -190,7 +190,7 @@ void ColumnLowCardinality::insertFromFullColumn(const IColumn & src, size_t n)
idx.insertPosition(getDictionary().uniqueInsertFrom(src, n));
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnLowCardinality::insertRangeFrom(const IColumn & src, size_t start, size_t length)
#else
void ColumnLowCardinality::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
@ -362,7 +362,7 @@ int ColumnLowCardinality::compareAtImpl(size_t n, size_t m, const IColumn & rhs,
return getDictionary().compareAt(n_index, m_index, low_cardinality_column.getDictionary(), nan_direction_hint);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int ColumnLowCardinality::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
#else
int ColumnLowCardinality::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const

6
src/Columns/ColumnLowCardinality.h
View File

@ -78,14 +78,14 @@ public:
bool tryInsert(const Field & x) override;
void insertDefault() override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & src, size_t n) override;
#else
void doInsertFrom(const IColumn & src, size_t n) override;
#endif
void insertFromFullColumn(const IColumn & src, size_t n);
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
#else
void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
@ -135,7 +135,7 @@ public:
return ColumnLowCardinality::create(dictionary.getColumnUniquePtr(), getIndexes().index(indexes_, limit));
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
#else
int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;

8
src/Columns/ColumnMap.cpp
View File

@ -153,7 +153,7 @@ void ColumnMap::updateHashFast(SipHash & hash) const
nested->updateHashFast(hash);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnMap::insertFrom(const IColumn & src, size_t n)
#else
void ColumnMap::doInsertFrom(const IColumn & src, size_t n)
@ -162,7 +162,7 @@ void ColumnMap::doInsertFrom(const IColumn & src, size_t n)
nested->insertFrom(assert_cast<const ColumnMap &>(src).getNestedColumn(), n);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnMap::insertManyFrom(const IColumn & src, size_t position, size_t length)
#else
void ColumnMap::doInsertManyFrom(const IColumn & src, size_t position, size_t length)
@ -171,7 +171,7 @@ void ColumnMap::doInsertManyFrom(const IColumn & src, size_t position, size_t le
assert_cast<ColumnArray &>(*nested).insertManyFrom(assert_cast<const ColumnMap &>(src).getNestedColumn(), position, length);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnMap::insertRangeFrom(const IColumn & src, size_t start, size_t length)
#else
void ColumnMap::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
@ -222,7 +222,7 @@ MutableColumns ColumnMap::scatter(ColumnIndex num_columns, const Selector & sele
return res;
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int ColumnMap::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
#else
int ColumnMap::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const

4
src/Columns/ColumnMap.h
View File

@ -67,7 +67,7 @@ public:
WeakHash32 getWeakHash32() const override;
void updateHashFast(SipHash & hash) const override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & src_, size_t n) override;
void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
@ -83,7 +83,7 @@ public:
ColumnPtr index(const IColumn & indexes, size_t limit) const override;
ColumnPtr replicate(const Offsets & offsets) const override;
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
#else
int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;

8
src/Columns/ColumnNullable.cpp
View File

@ -217,7 +217,7 @@ const char * ColumnNullable::skipSerializedInArena(const char * pos) const
return pos;
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnNullable::insertRangeFrom(const IColumn & src, size_t start, size_t length)
#else
void ColumnNullable::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
@ -258,7 +258,7 @@ bool ColumnNullable::tryInsert(const Field & x)
return true;
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnNullable::insertFrom(const IColumn & src, size_t n)
#else
void ColumnNullable::doInsertFrom(const IColumn & src, size_t n)
@ -270,7 +270,7 @@ void ColumnNullable::doInsertFrom(const IColumn & src, size_t n)
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnNullable::insertManyFrom(const IColumn & src, size_t position, size_t length)
#else
void ColumnNullable::doInsertManyFrom(const IColumn & src, size_t position, size_t length)
@ -410,7 +410,7 @@ int ColumnNullable::compareAtImpl(size_t n, size_t m, const IColumn & rhs_, int
return getNestedColumn().compareAt(n, m, nested_rhs, null_direction_hint);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int ColumnNullable::compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const
#else
int ColumnNullable::doCompareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const

6
src/Columns/ColumnNullable.h
View File

@ -69,7 +69,7 @@ public:
char * serializeValueIntoMemory(size_t n, char * memory) const override;
const char * deserializeAndInsertFromArena(const char * pos) override;
const char * skipSerializedInArena(const char * pos) const override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
#else
void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
@ -77,7 +77,7 @@ public:
void insert(const Field & x) override;
bool tryInsert(const Field & x) override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & src, size_t n) override;
void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
#else
@ -100,7 +100,7 @@ public:
void expand(const Filter & mask, bool inverted) override;
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
ColumnPtr index(const IColumn & indexes, size_t limit) const override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override;
#else
int doCompareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override;

4
src/Columns/ColumnObject.cpp
View File

@ -763,7 +763,7 @@ void ColumnObject::get(size_t n, Field & res) const
}
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnObject::insertFrom(const IColumn & src, size_t n)
#else
void ColumnObject::doInsertFrom(const IColumn & src, size_t n)
@ -772,7 +772,7 @@ void ColumnObject::doInsertFrom(const IColumn & src, size_t n)
insert(src[n]);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnObject::insertRangeFrom(const IColumn & src, size_t start, size_t length)
#else
void ColumnObject::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)

4
src/Columns/ColumnObject.h
View File

@ -211,7 +211,7 @@ public:
bool tryInsert(const Field & field) override;
void insertDefault() override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & src, size_t n) override;
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
#else
@ -236,7 +236,7 @@ public:
/// Order of rows in ColumnObject is undefined.
void getPermutation(PermutationSortDirection, PermutationSortStability, size_t, int, Permutation & res) const override;
void updatePermutation(PermutationSortDirection, PermutationSortStability, size_t, int, Permutation &, EqualRanges &) const override {}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t, size_t, const IColumn &, int) const override { return 0; }
#else
int doCompareAt(size_t, size_t, const IColumn &, int) const override { return 0; }

6
src/Columns/ColumnSparse.cpp
View File

@ -174,7 +174,7 @@ const char * ColumnSparse::skipSerializedInArena(const char * pos) const
return values->skipSerializedInArena(pos);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnSparse::insertRangeFrom(const IColumn & src, size_t start, size_t length)
#else
void ColumnSparse::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
@ -252,7 +252,7 @@ bool ColumnSparse::tryInsert(const Field & x)
return true;
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnSparse::insertFrom(const IColumn & src, size_t n)
#else
void ColumnSparse::doInsertFrom(const IColumn & src, size_t n)
@ -454,7 +454,7 @@ ColumnPtr ColumnSparse::indexImpl(const PaddedPODArray<Type> & indexes, size_t l
return ColumnSparse::create(std::move(res_values), std::move(res_offsets), limit);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int ColumnSparse::compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const
#else
int ColumnSparse::doCompareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const

6
src/Columns/ColumnSparse.h
View File

@ -81,14 +81,14 @@ public:
char * serializeValueIntoMemory(size_t n, char * memory) const override;
const char * deserializeAndInsertFromArena(const char * pos) override;
const char * skipSerializedInArena(const char *) const override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
#else
void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
#endif
void insert(const Field & x) override;
bool tryInsert(const Field & x) override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & src, size_t n) override;
#else
void doInsertFrom(const IColumn & src, size_t n) override;
@ -106,7 +106,7 @@ public:
template <typename Type>
ColumnPtr indexImpl(const PaddedPODArray<Type> & indexes, size_t limit) const;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override;
#else
int doCompareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override;

4
src/Columns/ColumnString.cpp
View File

@ -39,7 +39,7 @@ ColumnString::ColumnString(const ColumnString & src)
last_offset, chars.size());
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnString::insertManyFrom(const IColumn & src, size_t position, size_t length)
#else
void ColumnString::doInsertManyFrom(const IColumn & src, size_t position, size_t length)
@ -132,7 +132,7 @@ WeakHash32 ColumnString::getWeakHash32() const
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnString::insertRangeFrom(const IColumn & src, size_t start, size_t length)
#else
void ColumnString::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)

8
src/Columns/ColumnString.h
View File

@ -142,7 +142,7 @@ public:
return true;
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & src_, size_t n) override
#else
void doInsertFrom(const IColumn & src_, size_t n) override
@ -169,7 +169,7 @@ public:
}
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
#else
void doInsertManyFrom(const IColumn & src, size_t position, size_t length) override;
@ -220,7 +220,7 @@ public:
hash.update(reinterpret_cast<const char *>(chars.data()), chars.size() * sizeof(chars[0]));
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
#else
void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
@ -250,7 +250,7 @@ public:
offsets.push_back(offsets.back() + 1);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t n, size_t m, const IColumn & rhs_, int /*nan_direction_hint*/) const override
#else
int doCompareAt(size_t n, size_t m, const IColumn & rhs_, int /*nan_direction_hint*/) const override

8
src/Columns/ColumnTuple.cpp
View File

@ -206,7 +206,7 @@ bool ColumnTuple::tryInsert(const Field & x)
return true;
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnTuple::insertFrom(const IColumn & src_, size_t n)
#else
void ColumnTuple::doInsertFrom(const IColumn & src_, size_t n)
@ -223,7 +223,7 @@ void ColumnTuple::doInsertFrom(const IColumn & src_, size_t n)
columns[i]->insertFrom(*src.columns[i], n);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnTuple::insertManyFrom(const IColumn & src, size_t position, size_t length)
#else
void ColumnTuple::doInsertManyFrom(const IColumn & src, size_t position, size_t length)
@ -327,7 +327,7 @@ void ColumnTuple::updateHashFast(SipHash & hash) const
column->updateHashFast(hash);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnTuple::insertRangeFrom(const IColumn & src, size_t start, size_t length)
#else
void ColumnTuple::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
@ -483,7 +483,7 @@ int ColumnTuple::compareAtImpl(size_t n, size_t m, const IColumn & rhs, int nan_
return 0;
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int ColumnTuple::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
#else
int ColumnTuple::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const

6
src/Columns/ColumnTuple.h
View File

@ -66,7 +66,7 @@ public:
void insert(const Field & x) override;
bool tryInsert(const Field & x) override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & src_, size_t n) override;
void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
#else
@ -83,7 +83,7 @@ public:
void updateHashWithValue(size_t n, SipHash & hash) const override;
WeakHash32 getWeakHash32() const override;
void updateHashFast(SipHash & hash) const override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
#else
void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
@ -94,7 +94,7 @@ public:
ColumnPtr index(const IColumn & indexes, size_t limit) const override;
ColumnPtr replicate(const Offsets & offsets) const override;
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
#else
int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;

4
src/Columns/ColumnUnique.h
View File

@ -90,7 +90,7 @@ public:
return getNestedColumn()->updateHashWithValue(n, hash_func);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
#else
int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
@ -492,7 +492,7 @@ const char * ColumnUnique<ColumnType>::skipSerializedInArena(const char *) const
}
template <typename ColumnType>
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int ColumnUnique<ColumnType>::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
#else
int ColumnUnique<ColumnType>::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const

8
src/Columns/ColumnVariant.cpp
View File

@ -595,7 +595,7 @@ void ColumnVariant::insertManyFromImpl(const DB::IColumn & src_, size_t position
}
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnVariant::insertFrom(const IColumn & src_, size_t n)
#else
void ColumnVariant::doInsertFrom(const IColumn & src_, size_t n)
@ -604,7 +604,7 @@ void ColumnVariant::doInsertFrom(const IColumn & src_, size_t n)
insertFromImpl(src_, n, nullptr);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnVariant::insertRangeFrom(const IColumn & src_, size_t start, size_t length)
#else
void ColumnVariant::doInsertRangeFrom(const IColumn & src_, size_t start, size_t length)
@ -613,7 +613,7 @@ void ColumnVariant::doInsertRangeFrom(const IColumn & src_, size_t start, size_t
insertRangeFromImpl(src_, start, length, nullptr);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnVariant::insertManyFrom(const DB::IColumn & src_, size_t position, size_t length)
#else
void ColumnVariant::doInsertManyFrom(const DB::IColumn & src_, size_t position, size_t length)
@ -1175,7 +1175,7 @@ bool ColumnVariant::hasEqualValues() const
return local_discriminators->hasEqualValues() && variants[localDiscriminatorAt(0)]->hasEqualValues();
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int ColumnVariant::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
#else
int ColumnVariant::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const

4
src/Columns/ColumnVariant.h
View File

@ -180,7 +180,7 @@ public:
void insert(const Field & x) override;
bool tryInsert(const Field & x) override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & src_, size_t n) override;
void insertRangeFrom(const IColumn & src_, size_t start, size_t length) override;
void insertManyFrom(const IColumn & src_, size_t position, size_t length) override;
@ -223,7 +223,7 @@ public:
ColumnPtr indexImpl(const PaddedPODArray<Type> & indexes, size_t limit) const;
ColumnPtr replicate(const Offsets & replicate_offsets) const override;
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
#else
int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;

2
src/Columns/ColumnVector.cpp
View File

@ -502,7 +502,7 @@ bool ColumnVector<T>::tryInsert(const DB::Field & x)
}
template <typename T>
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void ColumnVector<T>::insertRangeFrom(const IColumn & src, size_t start, size_t length)
#else
void ColumnVector<T>::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)

8
src/Columns/ColumnVector.h
View File

@ -64,7 +64,7 @@ public:
return data.size();
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn & src, size_t n) override
#else
void doInsertFrom(const IColumn & src, size_t n) override
@ -73,7 +73,7 @@ public:
data.push_back(assert_cast<const Self &>(src).getData()[n]);
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertManyFrom(const IColumn & src, size_t position, size_t length) override
#else
void doInsertManyFrom(const IColumn & src, size_t position, size_t length) override
@ -150,7 +150,7 @@ public:
}
/// This method implemented in header because it could be possibly devirtualized.
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override
#else
int doCompareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override
@ -240,7 +240,7 @@ public:
bool tryInsert(const DB::Field & x) override;
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
#else
void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;

2
src/Columns/IColumn.cpp
View File

@ -46,7 +46,7 @@ String IColumn::dumpStructure() const
return res.str();
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void IColumn::insertFrom(const IColumn & src, size_t n)
#else
void IColumn::doInsertFrom(const IColumn & src, size_t n)

10
src/Columns/IColumn.h
View File

@ -179,7 +179,7 @@ public:
/// Appends n-th element from other column with the same type.
/// Is used in merge-sort and merges. It could be implemented in inherited classes more optimally than default implementation.
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
virtual void insertFrom(const IColumn & src, size_t n);
#else
void insertFrom(const IColumn & src, size_t n)
@ -191,7 +191,7 @@ public:
/// Appends range of elements from other column with the same type.
/// Could be used to concatenate columns.
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
virtual void insertRangeFrom(const IColumn & src, size_t start, size_t length) = 0;
#else
void insertRangeFrom(const IColumn & src, size_t start, size_t length)
@ -202,7 +202,7 @@ public:
#endif
/// Appends one element from other column with the same type multiple times.
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
virtual void insertManyFrom(const IColumn & src, size_t position, size_t length)
{
for (size_t i = 0; i < length; ++i)
@ -345,7 +345,7 @@ public:
*
* For non Nullable and non floating point types, nan_direction_hint is ignored.
*/
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
[[nodiscard]] virtual int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const = 0;
#else
[[nodiscard]] int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
@ -667,7 +667,7 @@ protected:
Sort full_sort,
PartialSort partial_sort) const;
#if defined(ABORT_ON_LOGICAL_ERROR)
#if defined(DEBUG_OR_SANITIZER_BUILD)
virtual void doInsertFrom(const IColumn & src, size_t n);
virtual void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) = 0;

6
src/Columns/IColumnDummy.h
View File

@ -27,7 +27,7 @@ public:
size_t byteSize() const override { return 0; }
size_t byteSizeAt(size_t) const override { return 0; }
size_t allocatedBytes() const override { return 0; }
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
int compareAt(size_t, size_t, const IColumn &, int) const override { return 0; }
#else
int doCompareAt(size_t, size_t, const IColumn &, int) const override { return 0; }
@ -73,7 +73,7 @@ public:
{
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertFrom(const IColumn &, size_t) override
#else
void doInsertFrom(const IColumn &, size_t) override
@ -82,7 +82,7 @@ public:
++s;
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn & /*src*/, size_t /*start*/, size_t length) override
#else
void doInsertRangeFrom(const IColumn & /*src*/, size_t /*start*/, size_t length) override

2
src/Columns/IColumnUnique.h
View File

@ -86,7 +86,7 @@ public:
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method tryInsert is not supported for ColumnUnique.");
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
void insertRangeFrom(const IColumn &, size_t, size_t) override
#else
void doInsertRangeFrom(const IColumn &, size_t, size_t) override

2
src/Columns/benchmarks/benchmark_column_insert_many_from.cpp
View File

@ -52,7 +52,7 @@ static ColumnPtr mockColumn(const DataTypePtr & type, size_t rows)
}
#if !defined(ABORT_ON_LOGICAL_ERROR)
#if !defined(DEBUG_OR_SANITIZER_BUILD)
static NO_INLINE void insertManyFrom(IColumn & dst, const IColumn & src)
#else
static NO_INLINE void doInsertManyFrom(IColumn & dst, const IColumn & src)

2
src/Common/Config/AbstractConfigurationComparison.cpp
View File

@ -38,7 +38,7 @@ namespace
std::erase_if(left_subkeys, [&](const String & key) { return ignore_keys->contains(key); });
std::erase_if(right_subkeys, [&](const String & key) { return ignore_keys->contains(key); });
#if defined(ABORT_ON_LOGICAL_ERROR)
#if defined(DEBUG_OR_SANITIZER_BUILD)
/// Compound `ignore_keys` are not yet implemented.
for (const auto & ignore_key : *ignore_keys)
chassert(ignore_key.find('.') == std::string_view::npos);

8
src/Common/CurrentMetrics.cpp
View File

@ -234,10 +234,10 @@
M(PartsCommitted, "Deprecated. See PartsActive.") \
M(PartsPreActive, "The part is in data_parts, but not used for SELECTs.") \
M(PartsActive, "Active data part, used by current and upcoming SELECTs.") \
M(AttachedDatabase, "Active database, used by current and upcoming SELECTs.") \
M(AttachedTable, "Active table, used by current and upcoming SELECTs.") \
M(AttachedView, "Active view, used by current and upcoming SELECTs.") \
M(AttachedDictionary, "Active dictionary, used by current and upcoming SELECTs.") \
M(AttachedDatabase, "Active databases.") \
M(AttachedTable, "Active tables.") \
M(AttachedView, "Active views.") \
M(AttachedDictionary, "Active dictionaries.") \
M(PartsOutdated, "Not active data part, but could be used by only current SELECTs, could be deleted after SELECTs finishes.") \
M(PartsDeleting, "Not active data part with identity refcounter, it is deleting right now by a cleaner.") \
M(PartsDeleteOnDestroy, "Part was moved to another disk and should be deleted in own destructor.") \

4
src/Common/Exception.cpp
View File

@ -64,7 +64,7 @@ void handle_error_code(const std::string & msg, int code, bool remote, const Exc
{
// In debug builds and builds with sanitizers, treat LOGICAL_ERROR as an assertion failure.
// Log the message before we fail.
#ifdef ABORT_ON_LOGICAL_ERROR
#ifdef DEBUG_OR_SANITIZER_BUILD
if (code == ErrorCodes::LOGICAL_ERROR)
{
abortOnFailedAssertion(msg, trace.data(), 0, trace.size());
@ -443,7 +443,7 @@ PreformattedMessage getCurrentExceptionMessageAndPattern(bool with_stacktrace, b
}
catch (...) {} // NOLINT(bugprone-empty-catch)
#ifdef ABORT_ON_LOGICAL_ERROR
#ifdef DEBUG_OR_SANITIZER_BUILD
try
{
throw;

2
src/Common/MemoryTracker.cpp
View File

@ -184,7 +184,7 @@ void MemoryTracker::debugLogBigAllocationWithoutCheck(Int64 size [[maybe_unused]
{
/// Big allocations through allocNoThrow (without checking memory limits) may easily lead to OOM (and it's hard to debug).
/// Let's find them.
#ifdef ABORT_ON_LOGICAL_ERROR
#ifdef DEBUG_OR_SANITIZER_BUILD
if (size < 0)
return;

2
src/Common/PageCache.cpp
View File

@ -424,7 +424,7 @@ static void logUnexpectedSyscallError(std::string name)
{
std::string message = fmt::format("{} failed: {}", name, errnoToString());
LOG_WARNING(&Poco::Logger::get("PageCache"), "{}", message);
#if defined(ABORT_ON_LOGICAL_ERROR)
#if defined(DEBUG_OR_SANITIZER_BUILD)
volatile bool true_ = true;
if (true_) // suppress warning about missing [[noreturn]]
abortOnFailedAssertion(message);

2
src/Common/assert_cast.h
View File

@ -25,7 +25,7 @@ namespace DB
template <typename To, typename From>
inline To assert_cast(From && from)
{
#ifdef ABORT_ON_LOGICAL_ERROR
#ifdef DEBUG_OR_SANITIZER_BUILD
try
{
if constexpr (std::is_pointer_v<To>)

7
src/Common/getRandomASCIIString.cpp
View File

@ -6,12 +6,17 @@ namespace DB
{
String getRandomASCIIString(size_t length)
{
return getRandomASCIIString(length, thread_local_rng);
}
String getRandomASCIIString(size_t length, pcg64 & rng)
{
std::uniform_int_distribution<int> distribution('a', 'z');
String res;
res.resize(length);
for (auto & c : res)
c = distribution(thread_local_rng);
c = distribution(rng);
return res;
}

3
src/Common/getRandomASCIIString.h
View File

@ -2,11 +2,14 @@
#include <Core/Types.h>
#include <pcg_random.hpp>
namespace DB
{
/// Slow random string. Useful for random names and things like this. Not for generating data.
String getRandomASCIIString(size_t length);
String getRandomASCIIString(size_t length, pcg64 & rng);
}

2
src/Common/tests/gtest_rw_lock.cpp
View File

@ -166,7 +166,7 @@ TEST(Common, RWLockRecursive)
auto lock2 = fifo_lock->getLock(RWLockImpl::Read, "q2");
#ifndef ABORT_ON_LOGICAL_ERROR
#ifndef DEBUG_OR_SANITIZER_BUILD
/// It throws LOGICAL_ERROR
EXPECT_ANY_THROW({fifo_lock->getLock(RWLockImpl::Write, "q2");});
#endif

19
src/Databases/DDLDependencyVisitor.cpp
View File

@ -80,13 +80,20 @@ namespace
/// CREATE TABLE or CREATE DICTIONARY or CREATE VIEW or CREATE TEMPORARY TABLE or CREATE DATABASE query.
void visitCreateQuery(const ASTCreateQuery & create)
{
QualifiedTableName to_table{create.to_table_id.database_name, create.to_table_id.table_name};
if (!to_table.table.empty())
if (create.targets)
{
/// TO target_table (for materialized views)
if (to_table.database.empty())
to_table.database = current_database;
dependencies.emplace(to_table);
for (const auto & target : create.targets->targets)
{
const auto & table_id = target.table_id;
if (!table_id.table_name.empty())
{
/// TO target_table (for materialized views)
QualifiedTableName target_name{table_id.database_name, table_id.table_name};
if (target_name.database.empty())
target_name.database = current_database;
dependencies.emplace(target_name);
}
}
}
QualifiedTableName as_table{create.as_database, create.as_table};

17
src/Databases/DDLRenamingVisitor.cpp
View File

@ -86,12 +86,19 @@ namespace
create.as_table = as_table_new.table;
}
QualifiedTableName to_table{create.to_table_id.database_name, create.to_table_id.table_name};
if (!to_table.table.empty() && !to_table.database.empty())
if (create.targets)
{
auto to_table_new = data.renaming_map.getNewTableName(to_table);
if (to_table_new != to_table)
create.to_table_id = StorageID{to_table_new.database, to_table_new.table};
for (auto & target : create.targets->targets)
{
auto & table_id = target.table_id;
if (!table_id.database_name.empty() && !table_id.table_name.empty())
{
QualifiedTableName target_name{table_id.database_name, table_id.table_name};
auto new_target_name = data.renaming_map.getNewTableName(target_name);
if (new_target_name != target_name)
table_id = StorageID{new_target_name.database, new_target_name.table};
}
}
}
}

4
src/Databases/DatabaseLazy.cpp
View File

@ -195,7 +195,7 @@ void DatabaseLazy::attachTable(ContextPtr /* context_ */, const String & table_n
snapshot_detached_tables.erase(table_name);
}
CurrentMetrics::add(CurrentMetrics::AttachedTable, 1);
CurrentMetrics::add(CurrentMetrics::AttachedTable);
}
StoragePtr DatabaseLazy::detachTable(ContextPtr /* context */, const String & table_name)
@ -221,7 +221,7 @@ StoragePtr DatabaseLazy::detachTable(ContextPtr /* context */, const String & ta
.metadata_path = getObjectMetadataPath(table_name),
.is_permanently = false});
CurrentMetrics::sub(CurrentMetrics::AttachedTable, 1);
CurrentMetrics::sub(CurrentMetrics::AttachedTable);
}
return res;
}

160
src/Databases/DatabaseReplicated.cpp
View File

@ -729,81 +729,14 @@ void DatabaseReplicated::checkQueryValid(const ASTPtr & query, ContextPtr query_
if (auto * create = query->as<ASTCreateQuery>())
{
bool replicated_table = create->storage && create->storage->engine &&
(startsWith(create->storage->engine->name, "Replicated") || startsWith(create->storage->engine->name, "Shared"));
if (!replicated_table || !create->storage->engine->arguments)
return;
if (create->storage)
checkTableEngine(*create, *create->storage, query_context);
ASTs & args_ref = create->storage->engine->arguments->children;
ASTs args = args_ref;
if (args.size() < 2)
return;
/// It can be a constant expression. Try to evaluate it, ignore exception if we cannot.
bool has_expression_argument = args_ref[0]->as<ASTFunction>() || args_ref[1]->as<ASTFunction>();
if (has_expression_argument)
if (create->targets)
{
try
{
args[0] = evaluateConstantExpressionAsLiteral(args_ref[0]->clone(), query_context);
args[1] = evaluateConstantExpressionAsLiteral(args_ref[1]->clone(), query_context);
}
catch (...) // NOLINT(bugprone-empty-catch)
{
}
for (const auto & inner_table_engine : create->targets->getInnerEngines())
checkTableEngine(*create, *inner_table_engine, query_context);
}
ASTLiteral * arg1 = args[0]->as<ASTLiteral>();
ASTLiteral * arg2 = args[1]->as<ASTLiteral>();
if (!arg1 || !arg2 || arg1->value.getType() != Field::Types::String || arg2->value.getType() != Field::Types::String)
return;
String maybe_path = arg1->value.get<String>();
String maybe_replica = arg2->value.get<String>();
/// Looks like it's ReplicatedMergeTree with explicit zookeeper_path and replica_name arguments.
/// Let's ensure that some macros are used.
/// NOTE: we cannot check here that substituted values will be actually different on shards and replicas.
Macros::MacroExpansionInfo info;
info.table_id = {getDatabaseName(), create->getTable(), create->uuid};
info.shard = getShardName();
info.replica = getReplicaName();
query_context->getMacros()->expand(maybe_path, info);
bool maybe_shard_macros = info.expanded_other;
info.expanded_other = false;
query_context->getMacros()->expand(maybe_replica, info);
bool maybe_replica_macros = info.expanded_other;
bool enable_functional_tests_helper = getContext()->getConfigRef().has("_functional_tests_helper_database_replicated_replace_args_macros");
if (!enable_functional_tests_helper)
{
if (query_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments)
LOG_WARNING(log, "It's not recommended to explicitly specify zookeeper_path and replica_name in ReplicatedMergeTree arguments");
else
throw Exception(ErrorCodes::INCORRECT_QUERY,
"It's not allowed to specify explicit zookeeper_path and replica_name "
"for ReplicatedMergeTree arguments in Replicated database. If you really want to "
"specify them explicitly, enable setting "
"database_replicated_allow_replicated_engine_arguments.");
}
if (maybe_shard_macros && maybe_replica_macros)
return;
if (enable_functional_tests_helper && !has_expression_argument)
{
if (maybe_path.empty() || maybe_path.back() != '/')
maybe_path += '/';
args_ref[0]->as<ASTLiteral>()->value = maybe_path + "auto_{shard}";
args_ref[1]->as<ASTLiteral>()->value = maybe_replica + "auto_{replica}";
return;
}
throw Exception(ErrorCodes::INCORRECT_QUERY,
"Explicit zookeeper_path and replica_name are specified in ReplicatedMergeTree arguments. "
"If you really want to specify it explicitly, then you should use some macros "
"to distinguish different shards and replicas");
}
}
@ -827,6 +760,85 @@ void DatabaseReplicated::checkQueryValid(const ASTPtr & query, ContextPtr query_
}
}
void DatabaseReplicated::checkTableEngine(const ASTCreateQuery & query, ASTStorage & storage, ContextPtr query_context) const
{
bool replicated_table = storage.engine &&
(startsWith(storage.engine->name, "Replicated") || startsWith(storage.engine->name, "Shared"));
if (!replicated_table || !storage.engine->arguments)
return;
ASTs & args_ref = storage.engine->arguments->children;
ASTs args = args_ref;
if (args.size() < 2)
return;
/// It can be a constant expression. Try to evaluate it, ignore exception if we cannot.
bool has_expression_argument = args_ref[0]->as<ASTFunction>() || args_ref[1]->as<ASTFunction>();
if (has_expression_argument)
{
try
{
args[0] = evaluateConstantExpressionAsLiteral(args_ref[0]->clone(), query_context);
args[1] = evaluateConstantExpressionAsLiteral(args_ref[1]->clone(), query_context);
}
catch (...) // NOLINT(bugprone-empty-catch)
{
}
}
ASTLiteral * arg1 = args[0]->as<ASTLiteral>();
ASTLiteral * arg2 = args[1]->as<ASTLiteral>();
if (!arg1 || !arg2 || arg1->value.getType() != Field::Types::String || arg2->value.getType() != Field::Types::String)
return;
String maybe_path = arg1->value.get<String>();
String maybe_replica = arg2->value.get<String>();
/// Looks like it's ReplicatedMergeTree with explicit zookeeper_path and replica_name arguments.
/// Let's ensure that some macros are used.
/// NOTE: we cannot check here that substituted values will be actually different on shards and replicas.
Macros::MacroExpansionInfo info;
info.table_id = {getDatabaseName(), query.getTable(), query.uuid};
info.shard = getShardName();
info.replica = getReplicaName();
query_context->getMacros()->expand(maybe_path, info);
bool maybe_shard_macros = info.expanded_other;
info.expanded_other = false;
query_context->getMacros()->expand(maybe_replica, info);
bool maybe_replica_macros = info.expanded_other;
bool enable_functional_tests_helper = getContext()->getConfigRef().has("_functional_tests_helper_database_replicated_replace_args_macros");
if (!enable_functional_tests_helper)
{
if (query_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments)
LOG_WARNING(log, "It's not recommended to explicitly specify zookeeper_path and replica_name in ReplicatedMergeTree arguments");
else
throw Exception(ErrorCodes::INCORRECT_QUERY,
"It's not allowed to specify explicit zookeeper_path and replica_name "
"for ReplicatedMergeTree arguments in Replicated database. If you really want to "
"specify them explicitly, enable setting "
"database_replicated_allow_replicated_engine_arguments.");
}
if (maybe_shard_macros && maybe_replica_macros)
return;
if (enable_functional_tests_helper && !has_expression_argument)
{
if (maybe_path.empty() || maybe_path.back() != '/')
maybe_path += '/';
args_ref[0]->as<ASTLiteral>()->value = maybe_path + "auto_{shard}";
args_ref[1]->as<ASTLiteral>()->value = maybe_replica + "auto_{replica}";
return;
}
throw Exception(ErrorCodes::INCORRECT_QUERY,
"Explicit zookeeper_path and replica_name are specified in ReplicatedMergeTree arguments. "
"If you really want to specify it explicitly, then you should use some macros "
"to distinguish different shards and replicas");
}
BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, QueryFlags flags)
{
waitDatabaseStarted();
@ -1312,11 +1324,9 @@ ASTPtr DatabaseReplicated::parseQueryFromMetadataInZooKeeper(const String & node
if (create.uuid == UUIDHelpers::Nil || create.getTable() != TABLE_WITH_UUID_NAME_PLACEHOLDER || create.database)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Got unexpected query from {}: {}", node_name, query);
bool is_materialized_view_with_inner_table = create.is_materialized_view && create.to_table_id.empty();
create.setDatabase(getDatabaseName());
create.setTable(unescapeForFileName(node_name));
create.attach = is_materialized_view_with_inner_table;
create.attach = create.is_materialized_view_with_inner_table();
return ast;
}

1
src/Databases/DatabaseReplicated.h
View File

@ -107,6 +107,7 @@ private:
void fillClusterAuthInfo(String collection_name, const Poco::Util::AbstractConfiguration & config);
void checkQueryValid(const ASTPtr & query, ContextPtr query_context) const;
void checkTableEngine(const ASTCreateQuery & query, ASTStorage & storage, ContextPtr query_context) const;
void recoverLostReplica(const ZooKeeperPtr & current_zookeeper, UInt32 our_log_ptr, UInt32 & max_log_ptr);

8
src/Databases/DatabasesCommon.cpp
View File

@ -289,8 +289,8 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n
tables.erase(it);
table_storage->is_detached = true;
if (table_storage->isSystemStorage() == false)
CurrentMetrics::sub(getAttachedCounterForStorage(table_storage), 1);
if (!table_storage->isSystemStorage() && database_name != DatabaseCatalog::SYSTEM_DATABASE)
CurrentMetrics::sub(getAttachedCounterForStorage(table_storage));
auto table_id = table_storage->getStorageID();
if (table_id.hasUUID())
@ -334,8 +334,8 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c
/// non-Atomic database the is_detached is set to true before RENAME.
table->is_detached = false;
if (table->isSystemStorage() == false && table_id.database_name != DatabaseCatalog::SYSTEM_DATABASE)
CurrentMetrics::add(getAttachedCounterForStorage(table), 1);
if (!table->isSystemStorage() && table_id.database_name != DatabaseCatalog::SYSTEM_DATABASE)
CurrentMetrics::add(getAttachedCounterForStorage(table));
}
void DatabaseWithOwnTablesBase::shutdown()

6
src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
View File

@ -59,7 +59,7 @@ CachedOnDiskReadBufferFromFile::CachedOnDiskReadBufferFromFile(
std::optional<size_t> read_until_position_,
std::shared_ptr<FilesystemCacheLog> cache_log_)
: ReadBufferFromFileBase(use_external_buffer_ ? 0 : settings_.remote_fs_buffer_size, nullptr, 0, file_size_)
#ifdef ABORT_ON_LOGICAL_ERROR
#ifdef DEBUG_OR_SANITIZER_BUILD
, log(getLogger(fmt::format("CachedOnDiskReadBufferFromFile({})", cache_key_)))
#else
, log(getLogger("CachedOnDiskReadBufferFromFile"))
@ -452,7 +452,7 @@ CachedOnDiskReadBufferFromFile::getImplementationBuffer(FileSegment & file_segme
{
case ReadType::CACHED:
{
#ifdef ABORT_ON_LOGICAL_ERROR
#ifdef DEBUG_OR_SANITIZER_BUILD
size_t file_size = getFileSizeFromReadBuffer(*read_buffer_for_file_segment);
if (file_size == 0 || range.left + file_size <= file_offset_of_buffer_end)
throw Exception(
@ -937,7 +937,7 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep()
if (!result)
{
#ifdef ABORT_ON_LOGICAL_ERROR
#ifdef DEBUG_OR_SANITIZER_BUILD
if (read_type == ReadType::CACHED)
{
size_t cache_file_size = getFileSizeFromReadBuffer(*implementation_buffer);

5
src/Functions/DateTimeTransforms.h
View File

@ -1954,7 +1954,10 @@ struct ToRelativeSubsecondNumImpl
return t.value;
if (scale > scale_multiplier)
return t.value / (scale / scale_multiplier);
return t.value * (scale_multiplier / scale);
return static_cast<UInt128>(t.value) * static_cast<UInt128>((scale_multiplier / scale));
/// Casting ^^: All integers are Int64, yet if t.value is big enough the multiplication can still
/// overflow which is UB. This place is too low-level and generic to check if t.value is sane.
/// Therefore just let it overflow safely and don't bother further.
}
static Int64 execute(UInt32 t, const DateLUTImpl &)
{

11
src/Functions/acosh.cpp
View File

@ -5,11 +5,12 @@ namespace DB
{
namespace
{
struct AcoshName
{
static constexpr auto name = "acosh";
};
using FunctionAcosh = FunctionMathUnary<UnaryFunctionVectorized<AcoshName, acosh>>;
struct AcoshName
{
static constexpr auto name = "acosh";
};
using FunctionAcosh = FunctionMathUnary<UnaryFunctionVectorized<AcoshName, acosh>>;
}

1
src/Functions/addMicroseconds.cpp
View File

@ -6,6 +6,7 @@ namespace DB
{
using FunctionAddMicroseconds = FunctionDateOrDateTimeAddInterval<AddMicrosecondsImpl>;
REGISTER_FUNCTION(AddMicroseconds)
{
factory.registerFunction<FunctionAddMicroseconds>();

1
src/Functions/addMilliseconds.cpp
View File

@ -6,6 +6,7 @@ namespace DB
{
using FunctionAddMilliseconds = FunctionDateOrDateTimeAddInterval<AddMillisecondsImpl>;
REGISTER_FUNCTION(AddMilliseconds)
{
factory.registerFunction<FunctionAddMilliseconds>();

1
src/Functions/addNanoseconds.cpp
View File

@ -6,6 +6,7 @@ namespace DB
{
using FunctionAddNanoseconds = FunctionDateOrDateTimeAddInterval<AddNanosecondsImpl>;
REGISTER_FUNCTION(AddNanoseconds)
{
factory.registerFunction<FunctionAddNanoseconds>();

1
src/Functions/aes_encrypt_mysql.cpp
View File

@ -7,7 +7,6 @@
namespace DB
{
namespace
{

9
src/Functions/appendTrailingCharIfAbsent.cpp
View File

@ -57,7 +57,7 @@ private:
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const auto & column = arguments[0].column;
const auto & column_char = arguments[1].column;
@ -80,14 +80,13 @@ private:
auto & dst_data = col_res->getChars();
auto & dst_offsets = col_res->getOffsets();
const auto size = src_offsets.size();
dst_data.resize(src_data.size() + size);
dst_offsets.resize(size);
dst_data.resize(src_data.size() + input_rows_count);
dst_offsets.resize(input_rows_count);
ColumnString::Offset src_offset{};
ColumnString::Offset dst_offset{};
for (const auto i : collections::range(0, size))
for (size_t i = 0; i < input_rows_count; ++i)
{
const auto src_length = src_offsets[i] - src_offset;
memcpySmallAllowReadWriteOverflow15(&dst_data[dst_offset], &src_data[src_offset], src_length);

2
src/Functions/ascii.cpp
View File

@ -45,9 +45,7 @@ struct AsciiImpl
size_t size = data.size() / n;
for (size_t i = 0; i < size; ++i)
{
res[i] = doAscii(data, i * n, n);
}
}
[[noreturn]] static void array(const ColumnString::Offsets & /*offsets*/, PaddedPODArray<ReturnType> & /*res*/)

11
src/Functions/asinh.cpp
View File

@ -5,11 +5,12 @@ namespace DB
{
namespace
{
struct AsinhName
{
static constexpr auto name = "asinh";
};
using FunctionAsinh = FunctionMathUnary<UnaryFunctionVectorized<AsinhName, asinh>>;
struct AsinhName
{
static constexpr auto name = "asinh";
};
using FunctionAsinh = FunctionMathUnary<UnaryFunctionVectorized<AsinhName, asinh>>;
}

11
src/Functions/atan2.cpp
View File

@ -5,11 +5,12 @@ namespace DB
{
namespace
{
struct Atan2Name
{
static constexpr auto name = "atan2";
};
using FunctionAtan2 = FunctionMathBinaryFloat64<BinaryFunctionVectorized<Atan2Name, atan2>>;
struct Atan2Name
{
static constexpr auto name = "atan2";
};
using FunctionAtan2 = FunctionMathBinaryFloat64<BinaryFunctionVectorized<Atan2Name, atan2>>;
}

11
src/Functions/atanh.cpp
View File

@ -5,11 +5,12 @@ namespace DB
{
namespace
{
struct AtanhName
{
static constexpr auto name = "atanh";
};
using FunctionAtanh = FunctionMathUnary<UnaryFunctionVectorized<AtanhName, atanh>>;
struct AtanhName
{
static constexpr auto name = "atanh";
};
using FunctionAtanh = FunctionMathUnary<UnaryFunctionVectorized<AtanhName, atanh>>;
}

2
src/Functions/base58Encode.cpp
View File

@ -3,8 +3,10 @@
namespace DB
{
REGISTER_FUNCTION(Base58Encode)
{
factory.registerFunction<FunctionBase58Conversion<Base58Encode>>();
}
}

2
src/Functions/base64Decode.cpp
View File

@ -5,6 +5,7 @@
namespace DB
{
REGISTER_FUNCTION(Base64Decode)
{
FunctionDocumentation::Description description = R"(Accepts a String and decodes it from base64, according to RFC 4648 (https://datatracker.ietf.org/doc/html/rfc4648#section-4). Throws an exception in case of an error. Alias: FROM_BASE64.)";
@ -19,6 +20,7 @@ REGISTER_FUNCTION(Base64Decode)
/// MySQL compatibility alias.
factory.registerAlias("FROM_BASE64", "base64Decode", FunctionFactory::Case::Insensitive);
}
}
#endif

2
src/Functions/base64Encode.cpp
View File

@ -5,6 +5,7 @@
namespace DB
{
REGISTER_FUNCTION(Base64Encode)
{
FunctionDocumentation::Description description = R"(Encodes a String as base64, according to RFC 4648 (https://datatracker.ietf.org/doc/html/rfc4648#section-4). Alias: TO_BASE64.)";
@ -19,6 +20,7 @@ REGISTER_FUNCTION(Base64Encode)
/// MySQL compatibility alias.
factory.registerAlias("TO_BASE64", "base64Encode", FunctionFactory::Case::Insensitive);
}
}
#endif

2
src/Functions/base64URLDecode.cpp
View File

@ -5,6 +5,7 @@
namespace DB
{
REGISTER_FUNCTION(Base64URLDecode)
{
FunctionDocumentation::Description description = R"(Accepts a base64-encoded URL and decodes it from base64 with URL-specific modifications, according to RFC 4648 (https://datatracker.ietf.org/doc/html/rfc4648#section-5).)";
@ -16,6 +17,7 @@ REGISTER_FUNCTION(Base64URLDecode)
factory.registerFunction<FunctionBase64Conversion<Base64Decode<Base64Variant::URL>>>({description, syntax, arguments, returned_value, examples, categories});
}
}
#endif

2
src/Functions/base64URLEncode.cpp
View File

@ -5,6 +5,7 @@
namespace DB
{
REGISTER_FUNCTION(Base64URLEncode)
{
FunctionDocumentation::Description description = R"(Encodes an URL (String or FixedString) as base64 with URL-specific modifications, according to RFC 4648 (https://datatracker.ietf.org/doc/html/rfc4648#section-5).)";
@ -16,6 +17,7 @@ REGISTER_FUNCTION(Base64URLEncode)
factory.registerFunction<FunctionBase64Conversion<Base64Encode<Base64Variant::URL>>>({description, syntax, arguments, returned_value, examples, categories});
}
}
#endif

8
src/Functions/byteSize.cpp
View File

@ -67,11 +67,11 @@ public:
const IColumn * column = arguments[arg_num].column.get();
if (arg_num == 0)
for (size_t row_num = 0; row_num < input_rows_count; ++row_num)
vec_res[row_num] = column->byteSizeAt(row_num);
for (size_t row = 0; row < input_rows_count; ++row)
vec_res[row] = column->byteSizeAt(row);
else
for (size_t row_num = 0; row_num < input_rows_count; ++row_num)
vec_res[row_num] += column->byteSizeAt(row_num);
for (size_t row = 0; row < input_rows_count; ++row)
vec_res[row] += column->byteSizeAt(row);
}
return result_col;

1
src/Functions/byteSwap.cpp
View File

@ -10,6 +10,7 @@ extern const int NOT_IMPLEMENTED;
namespace
{
template <typename T>
requires std::is_integral_v<T>
T byteSwap(T x)

3
src/Functions/caseWithExpression.cpp
View File

@ -98,8 +98,7 @@ public:
/// Execute transform.
ColumnsWithTypeAndName transform_args{args.front(), src_array_col, dst_array_col, args.back()};
return FunctionFactory::instance().get("transform", context)->build(transform_args)
->execute(transform_args, result_type, input_rows_count);
return FunctionFactory::instance().get("transform", context)->build(transform_args)->execute(transform_args, result_type, input_rows_count);
}
private:

12
src/Functions/convertCharset.cpp
View File

@ -88,7 +88,8 @@ private:
static void convert(const String & from_charset, const String & to_charset,
const ColumnString::Chars & from_chars, const ColumnString::Offsets & from_offsets,
ColumnString::Chars & to_chars, ColumnString::Offsets & to_offsets)
ColumnString::Chars & to_chars, ColumnString::Offsets & to_offsets,
size_t input_rows_count)
{
auto converter_from = getConverter(from_charset);
auto converter_to = getConverter(to_charset);
@ -96,12 +97,11 @@ private:
ColumnString::Offset current_from_offset = 0;
ColumnString::Offset current_to_offset = 0;
size_t size = from_offsets.size();
to_offsets.resize(size);
to_offsets.resize(input_rows_count);
PODArray<UChar> uchars;
for (size_t i = 0; i < size; ++i)
for (size_t i = 0; i < input_rows_count; ++i)
{
size_t from_string_size = from_offsets[i] - current_from_offset - 1;
@ -184,7 +184,7 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const ColumnWithTypeAndName & arg_from = arguments[0];
const ColumnWithTypeAndName & arg_charset_from = arguments[1];
@ -204,7 +204,7 @@ public:
if (const ColumnString * col_from = checkAndGetColumn<ColumnString>(arg_from.column.get()))
{
auto col_to = ColumnString::create();
convert(charset_from, charset_to, col_from->getChars(), col_from->getOffsets(), col_to->getChars(), col_to->getOffsets());
convert(charset_from, charset_to, col_from->getChars(), col_from->getOffsets(), col_to->getChars(), col_to->getOffsets(), input_rows_count);
return col_to;
}
else

11
src/Functions/cosh.cpp
View File

@ -5,11 +5,12 @@ namespace DB
{
namespace
{
struct CoshName
{
static constexpr auto name = "cosh";
};
using FunctionCosh = FunctionMathUnary<UnaryFunctionVectorized<CoshName, cosh>>;
struct CoshName
{
static constexpr auto name = "cosh";
};
using FunctionCosh = FunctionMathUnary<UnaryFunctionVectorized<CoshName, cosh>>;
}

3
src/Functions/countSubstringsCaseInsensitiveUTF8.cpp
View File

@ -13,8 +13,7 @@ struct NameCountSubstringsCaseInsensitiveUTF8
static constexpr auto name = "countSubstringsCaseInsensitiveUTF8";
};
using FunctionCountSubstringsCaseInsensitiveUTF8 = FunctionsStringSearch<
CountSubstringsImpl<NameCountSubstringsCaseInsensitiveUTF8, PositionCaseInsensitiveUTF8>>;
using FunctionCountSubstringsCaseInsensitiveUTF8 = FunctionsStringSearch<CountSubstringsImpl<NameCountSubstringsCaseInsensitiveUTF8, PositionCaseInsensitiveUTF8>>;
}

255
src/Functions/dateDiff.cpp
View File

@ -26,8 +26,6 @@ namespace DB
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int ILLEGAL_COLUMN;
extern const int BAD_ARGUMENTS;
}
@ -45,84 +43,82 @@ public:
template <typename Transform>
void dispatchForColumns(
const IColumn & x, const IColumn & y,
const IColumn & col_x, const IColumn & col_y,
const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
size_t input_rows_count,
ColumnInt64::Container & result) const
{
if (const auto * x_vec_16 = checkAndGetColumn<ColumnDate>(&x))
dispatchForSecondColumn<Transform>(*x_vec_16, y, timezone_x, timezone_y, result);
else if (const auto * x_vec_32 = checkAndGetColumn<ColumnDateTime>(&x))
dispatchForSecondColumn<Transform>(*x_vec_32, y, timezone_x, timezone_y, result);
else if (const auto * x_vec_32_s = checkAndGetColumn<ColumnDate32>(&x))
dispatchForSecondColumn<Transform>(*x_vec_32_s, y, timezone_x, timezone_y, result);
else if (const auto * x_vec_64 = checkAndGetColumn<ColumnDateTime64>(&x))
dispatchForSecondColumn<Transform>(*x_vec_64, y, timezone_x, timezone_y, result);
else if (const auto * x_const_16 = checkAndGetColumnConst<ColumnDate>(&x))
dispatchConstForSecondColumn<Transform>(x_const_16->getValue<UInt16>(), y, timezone_x, timezone_y, result);
else if (const auto * x_const_32 = checkAndGetColumnConst<ColumnDateTime>(&x))
dispatchConstForSecondColumn<Transform>(x_const_32->getValue<UInt32>(), y, timezone_x, timezone_y, result);
else if (const auto * x_const_32_s = checkAndGetColumnConst<ColumnDate32>(&x))
dispatchConstForSecondColumn<Transform>(x_const_32_s->getValue<Int32>(), y, timezone_x, timezone_y, result);
else if (const auto * x_const_64 = checkAndGetColumnConst<ColumnDateTime64>(&x))
dispatchConstForSecondColumn<Transform>(x_const_64->getValue<DecimalField<DateTime64>>(), y, timezone_x, timezone_y, result);
if (const auto * x_vec_16 = checkAndGetColumn<ColumnDate>(&col_x))
dispatchForSecondColumn<Transform>(*x_vec_16, col_y, timezone_x, timezone_y, input_rows_count, result);
else if (const auto * x_vec_32 = checkAndGetColumn<ColumnDateTime>(&col_x))
dispatchForSecondColumn<Transform>(*x_vec_32, col_y, timezone_x, timezone_y, input_rows_count, result);
else if (const auto * x_vec_32_s = checkAndGetColumn<ColumnDate32>(&col_x))
dispatchForSecondColumn<Transform>(*x_vec_32_s, col_y, timezone_x, timezone_y, input_rows_count, result);
else if (const auto * x_vec_64 = checkAndGetColumn<ColumnDateTime64>(&col_x))
dispatchForSecondColumn<Transform>(*x_vec_64, col_y, timezone_x, timezone_y, input_rows_count, result);
else if (const auto * x_const_16 = checkAndGetColumnConst<ColumnDate>(&col_x))
dispatchConstForSecondColumn<Transform>(x_const_16->getValue<UInt16>(), col_y, timezone_x, timezone_y, input_rows_count, result);
else if (const auto * x_const_32 = checkAndGetColumnConst<ColumnDateTime>(&col_x))
dispatchConstForSecondColumn<Transform>(x_const_32->getValue<UInt32>(), col_y, timezone_x, timezone_y, input_rows_count, result);
else if (const auto * x_const_32_s = checkAndGetColumnConst<ColumnDate32>(&col_x))
dispatchConstForSecondColumn<Transform>(x_const_32_s->getValue<Int32>(), col_y, timezone_x, timezone_y, input_rows_count, result);
else if (const auto * x_const_64 = checkAndGetColumnConst<ColumnDateTime64>(&col_x))
dispatchConstForSecondColumn<Transform>(x_const_64->getValue<DecimalField<DateTime64>>(), col_y, timezone_x, timezone_y, input_rows_count, result);
else
throw Exception(ErrorCodes::ILLEGAL_COLUMN,
"Illegal column for first argument of function {}, must be Date, Date32, DateTime or DateTime64",
name);
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for first argument of function {}, must be Date, Date32, DateTime or DateTime64", name);
}
template <typename Transform, typename LeftColumnType>
void dispatchForSecondColumn(
const LeftColumnType & x, const IColumn & y,
const LeftColumnType & x, const IColumn & col_y,
const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
size_t input_rows_count,
ColumnInt64::Container & result) const
{
if (const auto * y_vec_16 = checkAndGetColumn<ColumnDate>(&y))
vectorVector<Transform>(x, *y_vec_16, timezone_x, timezone_y, result);
else if (const auto * y_vec_32 = checkAndGetColumn<ColumnDateTime>(&y))
vectorVector<Transform>(x, *y_vec_32, timezone_x, timezone_y, result);
else if (const auto * y_vec_32_s = checkAndGetColumn<ColumnDate32>(&y))
vectorVector<Transform>(x, *y_vec_32_s, timezone_x, timezone_y, result);
else if (const auto * y_vec_64 = checkAndGetColumn<ColumnDateTime64>(&y))
vectorVector<Transform>(x, *y_vec_64, timezone_x, timezone_y, result);
else if (const auto * y_const_16 = checkAndGetColumnConst<ColumnDate>(&y))
vectorConstant<Transform>(x, y_const_16->getValue<UInt16>(), timezone_x, timezone_y, result);
else if (const auto * y_const_32 = checkAndGetColumnConst<ColumnDateTime>(&y))
vectorConstant<Transform>(x, y_const_32->getValue<UInt32>(), timezone_x, timezone_y, result);
else if (const auto * y_const_32_s = checkAndGetColumnConst<ColumnDate32>(&y))
vectorConstant<Transform>(x, y_const_32_s->getValue<Int32>(), timezone_x, timezone_y, result);
else if (const auto * y_const_64 = checkAndGetColumnConst<ColumnDateTime64>(&y))
vectorConstant<Transform>(x, y_const_64->getValue<DecimalField<DateTime64>>(), timezone_x, timezone_y, result);
if (const auto * y_vec_16 = checkAndGetColumn<ColumnDate>(&col_y))
vectorVector<Transform>(x, *y_vec_16, timezone_x, timezone_y, input_rows_count, result);
else if (const auto * y_vec_32 = checkAndGetColumn<ColumnDateTime>(&col_y))
vectorVector<Transform>(x, *y_vec_32, timezone_x, timezone_y, input_rows_count, result);
else if (const auto * y_vec_32_s = checkAndGetColumn<ColumnDate32>(&col_y))
vectorVector<Transform>(x, *y_vec_32_s, timezone_x, timezone_y, input_rows_count, result);
else if (const auto * y_vec_64 = checkAndGetColumn<ColumnDateTime64>(&col_y))
vectorVector<Transform>(x, *y_vec_64, timezone_x, timezone_y, input_rows_count, result);
else if (const auto * y_const_16 = checkAndGetColumnConst<ColumnDate>(&col_y))
vectorConstant<Transform>(x, y_const_16->getValue<UInt16>(), timezone_x, timezone_y, input_rows_count, result);
else if (const auto * y_const_32 = checkAndGetColumnConst<ColumnDateTime>(&col_y))
vectorConstant<Transform>(x, y_const_32->getValue<UInt32>(), timezone_x, timezone_y, input_rows_count, result);
else if (const auto * y_const_32_s = checkAndGetColumnConst<ColumnDate32>(&col_y))
vectorConstant<Transform>(x, y_const_32_s->getValue<Int32>(), timezone_x, timezone_y, input_rows_count, result);
else if (const auto * y_const_64 = checkAndGetColumnConst<ColumnDateTime64>(&col_y))
vectorConstant<Transform>(x, y_const_64->getValue<DecimalField<DateTime64>>(), timezone_x, timezone_y, input_rows_count, result);
else
throw Exception(ErrorCodes::ILLEGAL_COLUMN,
"Illegal column for second argument of function {}, must be Date, Date32, DateTime or DateTime64",
name);
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for second argument of function {}, must be Date, Date32, DateTime or DateTime64", name);
}
template <typename Transform, typename T1>
void dispatchConstForSecondColumn(
T1 x, const IColumn & y,
T1 x, const IColumn & col_y,
const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
size_t input_rows_count,
ColumnInt64::Container & result) const
{
if (const auto * y_vec_16 = checkAndGetColumn<ColumnDate>(&y))
constantVector<Transform>(x, *y_vec_16, timezone_x, timezone_y, result);
else if (const auto * y_vec_32 = checkAndGetColumn<ColumnDateTime>(&y))
constantVector<Transform>(x, *y_vec_32, timezone_x, timezone_y, result);
else if (const auto * y_vec_32_s = checkAndGetColumn<ColumnDate32>(&y))
constantVector<Transform>(x, *y_vec_32_s, timezone_x, timezone_y, result);
else if (const auto * y_vec_64 = checkAndGetColumn<ColumnDateTime64>(&y))
constantVector<Transform>(x, *y_vec_64, timezone_x, timezone_y, result);
if (const auto * y_vec_16 = checkAndGetColumn<ColumnDate>(&col_y))
constantVector<Transform>(x, *y_vec_16, timezone_x, timezone_y, input_rows_count, result);
else if (const auto * y_vec_32 = checkAndGetColumn<ColumnDateTime>(&col_y))
constantVector<Transform>(x, *y_vec_32, timezone_x, timezone_y, input_rows_count, result);
else if (const auto * y_vec_32_s = checkAndGetColumn<ColumnDate32>(&col_y))
constantVector<Transform>(x, *y_vec_32_s, timezone_x, timezone_y, input_rows_count, result);
else if (const auto * y_vec_64 = checkAndGetColumn<ColumnDateTime64>(&col_y))
constantVector<Transform>(x, *y_vec_64, timezone_x, timezone_y, input_rows_count, result);
else
throw Exception(ErrorCodes::ILLEGAL_COLUMN,
"Illegal column for second argument of function {}, must be Date, Date32, DateTime or DateTime64",
name);
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for second argument of function {}, must be Date, Date32, DateTime or DateTime64", name);
}
template <typename Transform, typename LeftColumnType, typename RightColumnType>
void vectorVector(
const LeftColumnType & x, const RightColumnType & y,
const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
size_t input_rows_count,
ColumnInt64::Container & result) const
{
const auto & x_data = x.getData();
@ -130,14 +126,15 @@ public:
const auto transform_x = TransformDateTime64<Transform>(getScale(x));
const auto transform_y = TransformDateTime64<Transform>(getScale(y));
for (size_t i = 0, size = x.size(); i < size; ++i)
result[i] = calculate(transform_x, transform_y, x_data[i], y_data[i], timezone_x, timezone_y);
for (size_t i = 0; i < input_rows_count; ++i)
result[i] = calculate(transform_x, transform_y, x_data[i], y_data[i], timezone_x, timezone_y);
}
template <typename Transform, typename LeftColumnType, typename T2>
void vectorConstant(
const LeftColumnType & x, T2 y,
const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
size_t input_rows_count,
ColumnInt64::Container & result) const
{
const auto & x_data = x.getData();
@ -145,7 +142,7 @@ public:
const auto transform_y = TransformDateTime64<Transform>(getScale(y));
const auto y_value = stripDecimalFieldValue(y);
for (size_t i = 0, size = x.size(); i < size; ++i)
for (size_t i = 0; i < input_rows_count; ++i)
result[i] = calculate(transform_x, transform_y, x_data[i], y_value, timezone_x, timezone_y);
}
@ -153,6 +150,7 @@ public:
void constantVector(
T1 x, const RightColumnType & y,
const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
size_t input_rows_count,
ColumnInt64::Container & result) const
{
const auto & y_data = y.getData();
@ -160,20 +158,22 @@ public:
const auto transform_y = TransformDateTime64<Transform>(getScale(y));
const auto x_value = stripDecimalFieldValue(x);
for (size_t i = 0, size = y.size(); i < size; ++i)
for (size_t i = 0; i < input_rows_count; ++i)
result[i] = calculate(transform_x, transform_y, x_value, y_data[i], timezone_x, timezone_y);
}
template <typename TransformX, typename TransformY, typename T1, typename T2>
Int64 calculate(const TransformX & transform_x, const TransformY & transform_y, T1 x, T2 y, const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y) const
{
auto res = static_cast<Int64>(transform_y.execute(y, timezone_y)) - static_cast<Int64>(transform_x.execute(x, timezone_x));
if constexpr (is_diff)
return static_cast<Int64>(transform_y.execute(y, timezone_y))
- static_cast<Int64>(transform_x.execute(x, timezone_x));
{
return res;
}
else
{
auto res = static_cast<Int64>(transform_y.execute(y, timezone_y))
- static_cast<Int64>(transform_x.execute(x, timezone_x));
/// Adjust res:
DateTimeComponentsWithFractionalPart a_comp;
DateTimeComponentsWithFractionalPart b_comp;
Int64 adjust_value;
@ -332,95 +332,73 @@ public:
static constexpr auto name = is_relative ? "dateDiff" : "age";
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionDateDiff>(); }
String getName() const override
{
return name;
}
String getName() const override { return name; }
bool isVariadic() const override { return true; }
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
size_t getNumberOfArguments() const override { return 0; }
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0, 3}; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
if (arguments.size() != 3 && arguments.size() != 4)
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Number of arguments for function {} doesn't match: passed {}, should be 3 or 4",
getName(), arguments.size());
FunctionArgumentDescriptors mandatory_args{
{"unit", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
{"startdate", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateOrDate32OrDateTimeOrDateTime64), nullptr, "Date[32] or DateTime[64]"},
{"enddate", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateOrDate32OrDateTimeOrDateTime64), nullptr, "Date[32] or DateTime[64]"},
};
if (!isString(arguments[0]))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"First argument for function {} (unit) must be String",
getName());
FunctionArgumentDescriptors optional_args{
{"timezone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
};
if (!isDate(arguments[1]) && !isDate32(arguments[1]) && !isDateTime(arguments[1]) && !isDateTime64(arguments[1]))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Second argument for function {} must be Date, Date32, DateTime or DateTime64",
getName());
if (!isDate(arguments[2]) && !isDate32(arguments[2]) && !isDateTime(arguments[2]) && !isDateTime64(arguments[2]))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Third argument for function {} must be Date, Date32, DateTime or DateTime64",
getName()
);
if (arguments.size() == 4 && !isString(arguments[3]))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Fourth argument for function {} (timezone) must be String",
getName());
validateFunctionArguments(*this, arguments, mandatory_args, optional_args);
return std::make_shared<DataTypeInt64>();
}
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0, 3}; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const auto * unit_column = checkAndGetColumnConst<ColumnString>(arguments[0].column.get());
if (!unit_column)
throw Exception(ErrorCodes::ILLEGAL_COLUMN,
"First argument for function {} must be constant String",
getName());
const auto * col_unit = checkAndGetColumnConst<ColumnString>(arguments[0].column.get());
if (!col_unit)
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "First argument for function {} must be constant String", getName());
String unit = Poco::toLower(unit_column->getValue<String>());
String unit = Poco::toLower(col_unit->getValue<String>());
const IColumn & x = *arguments[1].column;
const IColumn & y = *arguments[2].column;
const IColumn & col_x = *arguments[1].column;
const IColumn & col_y = *arguments[2].column;
size_t rows = input_rows_count;
auto res = ColumnInt64::create(rows);
auto col_res = ColumnInt64::create(input_rows_count);
const auto & timezone_x = extractTimeZoneFromFunctionArguments(arguments, 3, 1);
const auto & timezone_y = extractTimeZoneFromFunctionArguments(arguments, 3, 2);
if (unit == "year" || unit == "years" || unit == "yy" || unit == "yyyy")
impl.template dispatchForColumns<ToRelativeYearNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
impl.template dispatchForColumns<ToRelativeYearNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
else if (unit == "quarter" || unit == "quarters" || unit == "qq" || unit == "q")
impl.template dispatchForColumns<ToRelativeQuarterNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
impl.template dispatchForColumns<ToRelativeQuarterNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
else if (unit == "month" || unit == "months" || unit == "mm" || unit == "m")
impl.template dispatchForColumns<ToRelativeMonthNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
impl.template dispatchForColumns<ToRelativeMonthNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
else if (unit == "week" || unit == "weeks" || unit == "wk" || unit == "ww")
impl.template dispatchForColumns<ToRelativeWeekNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
impl.template dispatchForColumns<ToRelativeWeekNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
else if (unit == "day" || unit == "days" || unit == "dd" || unit == "d")
impl.template dispatchForColumns<ToRelativeDayNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
impl.template dispatchForColumns<ToRelativeDayNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
else if (unit == "hour" || unit == "hours" || unit == "hh" || unit == "h")
impl.template dispatchForColumns<ToRelativeHourNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
impl.template dispatchForColumns<ToRelativeHourNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
else if (unit == "minute" || unit == "minutes" || unit == "mi" || unit == "n")
impl.template dispatchForColumns<ToRelativeMinuteNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
impl.template dispatchForColumns<ToRelativeMinuteNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
else if (unit == "second" || unit == "seconds" || unit == "ss" || unit == "s")
impl.template dispatchForColumns<ToRelativeSecondNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
impl.template dispatchForColumns<ToRelativeSecondNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
else if (unit == "millisecond" || unit == "milliseconds" || unit == "ms")
impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<millisecond_multiplier>>(x, y, timezone_x, timezone_y, res->getData());
impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<millisecond_multiplier>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
else if (unit == "microsecond" || unit == "microseconds" || unit == "us" || unit == "u")
impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<microsecond_multiplier>>(x, y, timezone_x, timezone_y, res->getData());
impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<microsecond_multiplier>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
else if (unit == "nanosecond" || unit == "nanoseconds" || unit == "ns")
impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<nanosecond_multiplier>>(x, y, timezone_x, timezone_y, res->getData());
impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<nanosecond_multiplier>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
else
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Function {} does not support '{}' unit", getName(), unit);
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function {} does not support '{}' unit", getName(), unit);
return res;
return col_res;
}
private:
DateDiffImpl<is_relative> impl{name};
@ -437,50 +415,35 @@ public:
static constexpr auto name = "timeDiff";
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionTimeDiff>(); }
String getName() const override
{
return name;
}
String getName() const override { return name; }
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {}; }
bool isVariadic() const override { return false; }
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
size_t getNumberOfArguments() const override { return 2; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
if (arguments.size() != 2)
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Number of arguments for function {} doesn't match: passed {}, should be 2",
getName(), arguments.size());
FunctionArgumentDescriptors args{
{"first_datetime", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateOrDate32OrDateTimeOrDateTime64), nullptr, "Date[32] or DateTime[64]"},
{"second_datetime", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateOrDate32OrDateTimeOrDateTime64), nullptr, "Date[32] or DateTime[64]"},
};
if (!isDate(arguments[0]) && !isDate32(arguments[0]) && !isDateTime(arguments[0]) && !isDateTime64(arguments[0]))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"First argument for function {} must be Date, Date32, DateTime or DateTime64",
getName());
if (!isDate(arguments[1]) && !isDate32(arguments[1]) && !isDateTime(arguments[1]) && !isDateTime64(arguments[1]))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Second argument for function {} must be Date, Date32, DateTime or DateTime64",
getName()
);
validateFunctionArguments(*this, arguments, args);
return std::make_shared<DataTypeInt64>();
}
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {}; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const IColumn & x = *arguments[0].column;
const IColumn & y = *arguments[1].column;
const IColumn & col_x = *arguments[0].column;
const IColumn & col_y = *arguments[1].column;
size_t rows = input_rows_count;
auto res = ColumnInt64::create(rows);
auto col_res = ColumnInt64::create(input_rows_count);
impl.dispatchForColumns<ToRelativeSecondNumImpl<ResultPrecision::Extended>>(x, y, DateLUT::instance(), DateLUT::instance(), res->getData());
impl.dispatchForColumns<ToRelativeSecondNumImpl<ResultPrecision::Extended>>(col_x, col_y, DateLUT::instance(), DateLUT::instance(), input_rows_count, col_res->getData());
return res;
return col_res;
}
private:
DateDiffImpl<true> impl{name};

16
src/Functions/dateName.cpp
View File

@ -109,14 +109,14 @@ public:
ColumnPtr executeImpl(
const ColumnsWithTypeAndName & arguments,
const DataTypePtr & result_type,
[[maybe_unused]] size_t input_rows_count) const override
size_t input_rows_count) const override
{
ColumnPtr res;
if (!((res = executeType<DataTypeDate>(arguments, result_type))
|| (res = executeType<DataTypeDate32>(arguments, result_type))
|| (res = executeType<DataTypeDateTime>(arguments, result_type))
|| (res = executeType<DataTypeDateTime64>(arguments, result_type))))
if (!((res = executeType<DataTypeDate>(arguments, result_type, input_rows_count))
|| (res = executeType<DataTypeDate32>(arguments, result_type, input_rows_count))
|| (res = executeType<DataTypeDateTime>(arguments, result_type, input_rows_count))
|| (res = executeType<DataTypeDateTime64>(arguments, result_type, input_rows_count))))
throw Exception(
ErrorCodes::ILLEGAL_COLUMN,
"Illegal column {} of function {}, must be Date or DateTime.",
@ -127,7 +127,7 @@ public:
}
template <typename DataType>
ColumnPtr executeType(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const
ColumnPtr executeType(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const
{
auto * times = checkAndGetColumn<typename DataType::ColumnType>(arguments[1].column.get());
if (!times)
@ -144,7 +144,7 @@ public:
String date_part = date_part_column->getValue<String>();
const DateLUTImpl * time_zone_tmp;
if (std::is_same_v<DataType, DataTypeDateTime64> || std::is_same_v<DataType, DataTypeDateTime>)
if constexpr (std::is_same_v<DataType, DataTypeDateTime64> || std::is_same_v<DataType, DataTypeDateTime>)
time_zone_tmp = &extractTimeZoneFromFunctionArguments(arguments, 2, 1);
else
time_zone_tmp = &DateLUT::instance();
@ -175,7 +175,7 @@ public:
using TimeType = DateTypeToTimeType<DataType>;
callOnDatePartWriter<TimeType>(date_part, [&](const auto & writer)
{
for (size_t i = 0; i < times_data.size(); ++i)
for (size_t i = 0; i < input_rows_count; ++i)
{
if constexpr (std::is_same_v<DataType, DataTypeDateTime64>)
{

22
src/Functions/degrees.cpp
View File

@ -7,18 +7,20 @@ namespace DB
{
namespace
{
struct DegreesName
{
static constexpr auto name = "degrees";
};
Float64 degrees(Float64 r)
{
Float64 degrees = r * (180 / M_PI);
return degrees;
}
struct DegreesName
{
static constexpr auto name = "degrees";
};
Float64 degrees(Float64 r)
{
Float64 degrees = r * (180 / M_PI);
return degrees;
}
using FunctionDegrees = FunctionMathUnary<UnaryFunctionVectorized<DegreesName, degrees>>;
using FunctionDegrees = FunctionMathUnary<UnaryFunctionVectorized<DegreesName, degrees>>;
}
REGISTER_FUNCTION(Degrees)

2
src/Functions/filesystem.cpp
View File

@ -91,7 +91,7 @@ public:
auto col_res = ColumnVector<UInt64>::create(col_str->size());
auto & data = col_res->getData();
for (size_t i = 0; i < col_str->size(); ++i)
for (size_t i = 0; i < input_rows_count; ++i)
{
auto disk_name = col_str->getDataAt(i).toString();
if (auto it = disk_map.find(disk_name); it != disk_map.end())

32
src/Functions/formatDateTime.cpp
View File

@ -848,7 +848,7 @@ public:
return std::make_shared<DataTypeString>();
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, [[maybe_unused]] size_t input_rows_count) const override
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
ColumnPtr res;
if constexpr (support_integer == SupportInteger::Yes)
@ -862,17 +862,17 @@ public:
if (!castType(arguments[0].type.get(), [&](const auto & type)
{
using FromDataType = std::decay_t<decltype(type)>;
if (!(res = executeType<FromDataType>(arguments, result_type)))
if (!(res = executeType<FromDataType>(arguments, result_type, input_rows_count)))
throw Exception(ErrorCodes::ILLEGAL_COLUMN,
"Illegal column {} of function {}, must be Integer, Date, Date32, DateTime or DateTime64.",
arguments[0].column->getName(), getName());
return true;
}))
{
if (!((res = executeType<DataTypeDate>(arguments, result_type))
|| (res = executeType<DataTypeDate32>(arguments, result_type))
|| (res = executeType<DataTypeDateTime>(arguments, result_type))
|| (res = executeType<DataTypeDateTime64>(arguments, result_type))))
if (!((res = executeType<DataTypeDate>(arguments, result_type, input_rows_count))
|| (res = executeType<DataTypeDate32>(arguments, result_type, input_rows_count))
|| (res = executeType<DataTypeDateTime>(arguments, result_type, input_rows_count))
|| (res = executeType<DataTypeDateTime64>(arguments, result_type, input_rows_count))))
throw Exception(ErrorCodes::ILLEGAL_COLUMN,
"Illegal column {} of function {}, must be Integer or DateTime.",
arguments[0].column->getName(), getName());
@ -881,10 +881,10 @@ public:
}
else
{
if (!((res = executeType<DataTypeDate>(arguments, result_type))
|| (res = executeType<DataTypeDate32>(arguments, result_type))
|| (res = executeType<DataTypeDateTime>(arguments, result_type))
|| (res = executeType<DataTypeDateTime64>(arguments, result_type))))
if (!((res = executeType<DataTypeDate>(arguments, result_type, input_rows_count))
|| (res = executeType<DataTypeDate32>(arguments, result_type, input_rows_count))
|| (res = executeType<DataTypeDateTime>(arguments, result_type, input_rows_count))
|| (res = executeType<DataTypeDateTime64>(arguments, result_type, input_rows_count))))
throw Exception(ErrorCodes::ILLEGAL_COLUMN,
"Illegal column {} of function {}, must be Date or DateTime.",
arguments[0].column->getName(), getName());
@ -894,7 +894,7 @@ public:
}
template <typename DataType>
ColumnPtr executeType(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const
ColumnPtr executeType(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const
{
auto non_const_datetime = arguments[0].column->convertToFullColumnIfConst();
auto * times = checkAndGetColumn<typename DataType::ColumnType>(non_const_datetime.get());
@ -955,13 +955,11 @@ public:
else
time_zone = &DateLUT::instance();
const auto & vec = times->getData();
auto col_res = ColumnString::create();
auto & res_data = col_res->getChars();
auto & res_offsets = col_res->getOffsets();
res_data.resize(vec.size() * (out_template_size + 1));
res_offsets.resize(vec.size());
res_data.resize(input_rows_count * (out_template_size + 1));
res_offsets.resize(input_rows_count);
if constexpr (format_syntax == FormatSyntax::MySQL)
{
@ -990,9 +988,11 @@ public:
}
}
const auto & vec = times->getData();
auto * begin = reinterpret_cast<char *>(res_data.data());
auto * pos = begin;
for (size_t i = 0; i < vec.size(); ++i)
for (size_t i = 0; i < input_rows_count; ++i)
{
if (!const_time_zone_column && arguments.size() > 2)
{

10
src/Functions/formatQuery.cpp
View File

@ -75,7 +75,7 @@ public:
if (const ColumnString * col_query_string = checkAndGetColumn<ColumnString>(col_query.get()))
{
auto col_res = ColumnString::create();
formatVector(col_query_string->getChars(), col_query_string->getOffsets(), col_res->getChars(), col_res->getOffsets(), col_null_map);
formatVector(col_query_string->getChars(), col_query_string->getOffsets(), col_res->getChars(), col_res->getOffsets(), col_null_map, input_rows_count);
if (error_handling == ErrorHandling::Null)
return ColumnNullable::create(std::move(col_res), std::move(col_null_map));
@ -92,16 +92,16 @@ private:
const ColumnString::Offsets & offsets,
ColumnString::Chars & res_data,
ColumnString::Offsets & res_offsets,
ColumnUInt8::MutablePtr & res_null_map) const
ColumnUInt8::MutablePtr & res_null_map,
size_t input_rows_count) const
{
const size_t size = offsets.size();
res_offsets.resize(size);
res_offsets.resize(input_rows_count);
res_data.resize(data.size());
size_t prev_offset = 0;
size_t res_data_size = 0;
for (size_t i = 0; i < size; ++i)
for (size_t i = 0; i < input_rows_count; ++i)
{
const char * begin = reinterpret_cast<const char *>(&data[prev_offset]);
const char * end = begin + offsets[i] - prev_offset - 1;

31
src/Functions/formatReadable.h
View File

@ -55,19 +55,19 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
ColumnPtr res;
if (!((res = executeType<UInt8>(arguments))
|| (res = executeType<UInt16>(arguments))
|| (res = executeType<UInt32>(arguments))
|| (res = executeType<UInt64>(arguments))
|| (res = executeType<Int8>(arguments))
|| (res = executeType<Int16>(arguments))
|| (res = executeType<Int32>(arguments))
|| (res = executeType<Int64>(arguments))
|| (res = executeType<Float32>(arguments))
|| (res = executeType<Float64>(arguments))))
if (!((res = executeType<UInt8>(arguments, input_rows_count))
|| (res = executeType<UInt16>(arguments, input_rows_count))
|| (res = executeType<UInt32>(arguments, input_rows_count))
|| (res = executeType<UInt64>(arguments, input_rows_count))
|| (res = executeType<Int8>(arguments, input_rows_count))
|| (res = executeType<Int16>(arguments, input_rows_count))
|| (res = executeType<Int32>(arguments, input_rows_count))
|| (res = executeType<Int64>(arguments, input_rows_count))
|| (res = executeType<Float32>(arguments, input_rows_count))
|| (res = executeType<Float64>(arguments, input_rows_count))))
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}",
arguments[0].column->getName(), getName());
@ -76,7 +76,7 @@ public:
private:
template <typename T>
ColumnPtr executeType(const ColumnsWithTypeAndName & arguments) const
ColumnPtr executeType(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const
{
if (const ColumnVector<T> * col_from = checkAndGetColumn<ColumnVector<T>>(arguments[0].column.get()))
{
@ -85,13 +85,12 @@ private:
const typename ColumnVector<T>::Container & vec_from = col_from->getData();
ColumnString::Chars & data_to = col_to->getChars();
ColumnString::Offsets & offsets_to = col_to->getOffsets();
size_t size = vec_from.size();
data_to.resize(size * 2);
offsets_to.resize(size);
data_to.resize(input_rows_count * 2);
offsets_to.resize(input_rows_count);
WriteBufferFromVector<ColumnString::Chars> buf_to(data_to);
for (size_t i = 0; i < size; ++i)
for (size_t i = 0; i < input_rows_count; ++i)
{
Impl::format(static_cast<double>(vec_from[i]), buf_to);
writeChar(0, buf_to);

Some files were not shown because too many files have changed in this diff Show More