thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 08:02:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into fix-usage-of-plain-metadata

Browse Source
This commit is contained in:
kssenii 2024-03-10 12:05:02 +01:00
commit e5e6625b0c
444 changed files with 6453 additions and 4331 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
.github/workflows/master.yml vendored
View File

@ -305,7 +305,7 @@ jobs:
runner_type: style-checker-aarch64
data: ${{ needs.RunConfig.outputs.data }}
MarkReleaseReady:
if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
if: ${{ !failure() && !cancelled() }}
needs:
- BuilderBinDarwin
- BuilderBinDarwinAarch64
@ -313,9 +313,25 @@ jobs:
- BuilderDebAarch64
runs-on: [self-hosted, style-checker]
steps:
- name: Debug
run: |
echo need with different filters
cat << 'EOF'
${{ toJSON(needs) }}
${{ toJSON(needs.*.result) }}
no failures ${{ !contains(needs.*.result, 'failure') }}
no skips ${{ !contains(needs.*.result, 'skipped') }}
no both ${{ !(contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
EOF
- name: Not ready
# fail the job to be able restart it
if: ${{ contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure') }}
run: exit 1
- name: Check out repository code
if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
uses: ClickHouse/checkout@v1
- name: Mark Commit Release Ready
if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
python3 mark_release_ready.py

18
.github/workflows/release_branches.yml vendored
View File

@ -206,7 +206,7 @@ jobs:
runner_type: style-checker-aarch64
data: ${{ needs.RunConfig.outputs.data }}
MarkReleaseReady:
if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
if: ${{ !failure() && !cancelled() }}
needs:
- BuilderBinDarwin
- BuilderBinDarwinAarch64
@ -214,9 +214,25 @@ jobs:
- BuilderDebAarch64
runs-on: [self-hosted, style-checker-aarch64]
steps:
- name: Debug
run: |
echo need with different filters
cat << 'EOF'
${{ toJSON(needs) }}
${{ toJSON(needs.*.result) }}
no failures ${{ !contains(needs.*.result, 'failure') }}
no skips ${{ !contains(needs.*.result, 'skipped') }}
no both ${{ !(contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
EOF
- name: Not ready
# fail the job to be able restart it
if: ${{ contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure') }}
run: exit 1
- name: Check out repository code
if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
uses: ClickHouse/checkout@v1
- name: Mark Commit Release Ready
if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
python3 mark_release_ready.py

3
.github/workflows/reusable_build.yml vendored
View File

@ -43,7 +43,8 @@ jobs:
runs-on: [self-hosted, '${{inputs.runner_type}}']
steps:
- name: Check out repository code
uses: ClickHouse/checkout@v1
# WIP: temporary try commit with limited perallelization of checkout
uses: ClickHouse/checkout@0be3f7b3098bae494d3ef5d29d2e0676fb606232
with:
clear-repository: true
ref: ${{ fromJson(inputs.data).git_ref }}

2
.gitignore vendored
View File

@ -165,7 +165,7 @@ tests/queries/0_stateless/*.expect.history
tests/integration/**/_gen
# rust
/rust/**/target
/rust/**/target*
# It is autogenerated from *.in
/rust/**/.cargo/config.toml
/rust/**/vendor

3
CMakeLists.txt
View File

@ -319,7 +319,8 @@ if (COMPILER_CLANG)
endif()
endif ()
set (COMPILER_FLAGS "${COMPILER_FLAGS}")
# Disable floating-point expression contraction in order to get consistent floating point calculation results across platforms
set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffp-contract=off")
# Our built-in unwinder only supports DWARF version up to 4.
set (DEBUG_INFO_FLAGS "-g")

23
README.md
View File

@ -31,15 +31,30 @@ curl https://clickhouse.com/ | sh
* [Static Analysis (SonarCloud)](https://sonarcloud.io/project/issues?resolved=false&id=ClickHouse_ClickHouse) proposes C++ quality improvements.
* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
## Monthly Release & Community Call
Every month we get together with the community (users, contributors, customers, those interested in learning more about ClickHouse) to discuss what is coming in the latest release. If you are interested in sharing what you've built on ClickHouse, let us know.
* [v24.3 Community Call](https://clickhouse.com/company/events/v24-3-community-release-call) - Mar 26
* [v24.4 Community Call](https://clickhouse.com/company/events/v24-4-community-release-call) - Apr 30
## Upcoming Events
Keep an eye out for upcoming meetups around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com.
Keep an eye out for upcoming meetups and eventsaround the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
* [ClickHouse Meetup in Bellevue](https://www.meetup.com/clickhouse-seattle-user-group/events/298650371/) - Mar 11
* [ClickHouse Meetup at Ramp's Offices in NYC](https://www.meetup.com/clickhouse-new-york-user-group/events/298640542/) - Mar 19
* [ClickHouse Melbourne Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/299479750/) - Mar 20
* [ClickHouse Meetup in Paris](https://www.meetup.com/clickhouse-france-user-group/events/298997115/) - Mar 21
* [ClickHouse Meetup in Bengaluru](https://www.meetup.com/clickhouse-bangalore-user-group/events/299479850/) - Mar 23
* [ClickHouse Meetup in Zurich](https://www.meetup.com/clickhouse-switzerland-meetup-group/events/299628922/) - Apr 16
* [ClickHouse Meetup in Copenhagen](https://www.meetup.com/clickhouse-denmark-meetup-group/events/299629133/) - Apr 23
* [ClickHouse Meetup in Dubai](https://www.meetup.com/clickhouse-dubai-meetup-group/events/299629189/) - May 28
## Recent Recordings
* **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
* **Recording available**: [**v24.1 Release Webinar**](https://www.youtube.com/watch?v=pBF9g0wGAGs) All the features of 24.1, one convenient video! Watch it now!
* **All release webinar recordings**: [YouTube playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3jAlSy1JxyP8zluvXaN3nxU)
* **Recording available**: [**v24.2 Release Call**](https://www.youtube.com/watch?v=iN2y-TK8f3A) All the features of 24.2, one convenient video! Watch it now!
## Interested in joining ClickHouse and making it your full-time job?

1
base/base/CMakeLists.txt
View File

@ -13,6 +13,7 @@ set (SRCS
cgroupsv2.cpp
coverage.cpp
demangle.cpp
Decimal.cpp
getAvailableMemoryAmount.cpp
getFQDNOrHostName.cpp
getMemoryAmount.cpp

87
base/base/Decimal.cpp Normal file
View File

@ -0,0 +1,87 @@
#include <base/Decimal.h>
#include <base/extended_types.h>
namespace DB
{
/// Explicit template instantiations.
#define FOR_EACH_UNDERLYING_DECIMAL_TYPE(M) \
M(Int32) \
M(Int64) \
M(Int128) \
M(Int256)
#define FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS(M, X) \
M(Int32, X) \
M(Int64, X) \
M(Int128, X) \
M(Int256, X)
template <typename T> const Decimal<T> & Decimal<T>::operator += (const T & x) { value += x; return *this; }
template <typename T> const Decimal<T> & Decimal<T>::operator -= (const T & x) { value -= x; return *this; }
template <typename T> const Decimal<T> & Decimal<T>::operator *= (const T & x) { value *= x; return *this; }
template <typename T> const Decimal<T> & Decimal<T>::operator /= (const T & x) { value /= x; return *this; }
template <typename T> const Decimal<T> & Decimal<T>::operator %= (const T & x) { value %= x; return *this; }
template <typename T> void NO_SANITIZE_UNDEFINED Decimal<T>::addOverflow(const T & x) { value += x; }
/// Maybe this explicit instantiation affects performance since operators cannot be inlined.
template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator += (const Decimal<U> & x) { value += static_cast<T>(x.value); return *this; }
template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator -= (const Decimal<U> & x) { value -= static_cast<T>(x.value); return *this; }
template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator *= (const Decimal<U> & x) { value *= static_cast<T>(x.value); return *this; }
template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator /= (const Decimal<U> & x) { value /= static_cast<T>(x.value); return *this; }
template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator %= (const Decimal<U> & x) { value %= static_cast<T>(x.value); return *this; }
#define DISPATCH(TYPE_T, TYPE_U) \
template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator += (const Decimal<TYPE_U> & x); \
template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator -= (const Decimal<TYPE_U> & x); \
template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator *= (const Decimal<TYPE_U> & x); \
template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator /= (const Decimal<TYPE_U> & x); \
template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator %= (const Decimal<TYPE_U> & x);
#define INVOKE(X) FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS(DISPATCH, X)
FOR_EACH_UNDERLYING_DECIMAL_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
#define DISPATCH(TYPE) template struct Decimal<TYPE>;
FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
#undef DISPATCH
template <typename T> bool operator< (const Decimal<T> & x, const Decimal<T> & y) { return x.value < y.value; }
template <typename T> bool operator> (const Decimal<T> & x, const Decimal<T> & y) { return x.value > y.value; }
template <typename T> bool operator<= (const Decimal<T> & x, const Decimal<T> & y) { return x.value <= y.value; }
template <typename T> bool operator>= (const Decimal<T> & x, const Decimal<T> & y) { return x.value >= y.value; }
template <typename T> bool operator== (const Decimal<T> & x, const Decimal<T> & y) { return x.value == y.value; }
template <typename T> bool operator!= (const Decimal<T> & x, const Decimal<T> & y) { return x.value != y.value; }
#define DISPATCH(TYPE) \
template bool operator< (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
template bool operator> (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
template bool operator<= (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
template bool operator>= (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
template bool operator== (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
template bool operator!= (const Decimal<TYPE> & x, const Decimal<TYPE> & y);
FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
#undef DISPATCH
template <typename T> Decimal<T> operator+ (const Decimal<T> & x, const Decimal<T> & y) { return x.value + y.value; }
template <typename T> Decimal<T> operator- (const Decimal<T> & x, const Decimal<T> & y) { return x.value - y.value; }
template <typename T> Decimal<T> operator* (const Decimal<T> & x, const Decimal<T> & y) { return x.value * y.value; }
template <typename T> Decimal<T> operator/ (const Decimal<T> & x, const Decimal<T> & y) { return x.value / y.value; }
template <typename T> Decimal<T> operator- (const Decimal<T> & x) { return -x.value; }
#define DISPATCH(TYPE) \
template Decimal<TYPE> operator+ (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
template Decimal<TYPE> operator- (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
template Decimal<TYPE> operator* (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
template Decimal<TYPE> operator/ (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
template Decimal<TYPE> operator- (const Decimal<TYPE> & x);
FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
#undef DISPATCH
#undef FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS
#undef FOR_EACH_UNDERLYING_DECIMAL_TYPE
}

94
base/base/Decimal.h
View File

@ -2,6 +2,7 @@
#include <base/extended_types.h>
#include <base/Decimal_fwd.h>
#include <base/types.h>
#include <base/defines.h>
@ -10,6 +11,18 @@ namespace DB
template <class> struct Decimal;
class DateTime64;
#define FOR_EACH_UNDERLYING_DECIMAL_TYPE(M) \
M(Int32) \
M(Int64) \
M(Int128) \
M(Int256)
#define FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS(M, X) \
M(Int32, X) \
M(Int64, X) \
M(Int128, X) \
M(Int256, X)
using Decimal32 = Decimal<Int32>;
using Decimal64 = Decimal<Int64>;
using Decimal128 = Decimal<Int128>;
@ -50,36 +63,73 @@ struct Decimal
return static_cast<U>(value);
}
const Decimal<T> & operator += (const T & x) { value += x; return *this; }
const Decimal<T> & operator -= (const T & x) { value -= x; return *this; }
const Decimal<T> & operator *= (const T & x) { value *= x; return *this; }
const Decimal<T> & operator /= (const T & x) { value /= x; return *this; }
const Decimal<T> & operator %= (const T & x) { value %= x; return *this; }
const Decimal<T> & operator += (const T & x);
const Decimal<T> & operator -= (const T & x);
const Decimal<T> & operator *= (const T & x);
const Decimal<T> & operator /= (const T & x);
const Decimal<T> & operator %= (const T & x);
template <typename U> const Decimal<T> & operator += (const Decimal<U> & x) { value += x.value; return *this; }
template <typename U> const Decimal<T> & operator -= (const Decimal<U> & x) { value -= x.value; return *this; }
template <typename U> const Decimal<T> & operator *= (const Decimal<U> & x) { value *= x.value; return *this; }
template <typename U> const Decimal<T> & operator /= (const Decimal<U> & x) { value /= x.value; return *this; }
template <typename U> const Decimal<T> & operator %= (const Decimal<U> & x) { value %= x.value; return *this; }
template <typename U> const Decimal<T> & operator += (const Decimal<U> & x);
template <typename U> const Decimal<T> & operator -= (const Decimal<U> & x);
template <typename U> const Decimal<T> & operator *= (const Decimal<U> & x);
template <typename U> const Decimal<T> & operator /= (const Decimal<U> & x);
template <typename U> const Decimal<T> & operator %= (const Decimal<U> & x);
/// This is to avoid UB for sumWithOverflow()
void NO_SANITIZE_UNDEFINED addOverflow(const T & x) { value += x; }
void NO_SANITIZE_UNDEFINED addOverflow(const T & x);
T value;
};
template <typename T> inline bool operator< (const Decimal<T> & x, const Decimal<T> & y) { return x.value < y.value; }
template <typename T> inline bool operator> (const Decimal<T> & x, const Decimal<T> & y) { return x.value > y.value; }
template <typename T> inline bool operator<= (const Decimal<T> & x, const Decimal<T> & y) { return x.value <= y.value; }
template <typename T> inline bool operator>= (const Decimal<T> & x, const Decimal<T> & y) { return x.value >= y.value; }
template <typename T> inline bool operator== (const Decimal<T> & x, const Decimal<T> & y) { return x.value == y.value; }
template <typename T> inline bool operator!= (const Decimal<T> & x, const Decimal<T> & y) { return x.value != y.value; }
#define DISPATCH(TYPE) extern template struct Decimal<TYPE>;
FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
#undef DISPATCH
template <typename T> inline Decimal<T> operator+ (const Decimal<T> & x, const Decimal<T> & y) { return x.value + y.value; }
template <typename T> inline Decimal<T> operator- (const Decimal<T> & x, const Decimal<T> & y) { return x.value - y.value; }
template <typename T> inline Decimal<T> operator* (const Decimal<T> & x, const Decimal<T> & y) { return x.value * y.value; }
template <typename T> inline Decimal<T> operator/ (const Decimal<T> & x, const Decimal<T> & y) { return x.value / y.value; }
template <typename T> inline Decimal<T> operator- (const Decimal<T> & x) { return -x.value; }
#define DISPATCH(TYPE_T, TYPE_U) \
extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator += (const Decimal<TYPE_U> & x); \
extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator -= (const Decimal<TYPE_U> & x); \
extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator *= (const Decimal<TYPE_U> & x); \
extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator /= (const Decimal<TYPE_U> & x); \
extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator %= (const Decimal<TYPE_U> & x);
#define INVOKE(X) FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS(DISPATCH, X)
FOR_EACH_UNDERLYING_DECIMAL_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
template <typename T> bool operator< (const Decimal<T> & x, const Decimal<T> & y);
template <typename T> bool operator> (const Decimal<T> & x, const Decimal<T> & y);
template <typename T> bool operator<= (const Decimal<T> & x, const Decimal<T> & y);
template <typename T> bool operator>= (const Decimal<T> & x, const Decimal<T> & y);
template <typename T> bool operator== (const Decimal<T> & x, const Decimal<T> & y);
template <typename T> bool operator!= (const Decimal<T> & x, const Decimal<T> & y);
#define DISPATCH(TYPE) \
extern template bool operator< (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
extern template bool operator> (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
extern template bool operator<= (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
extern template bool operator>= (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
extern template bool operator== (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
extern template bool operator!= (const Decimal<TYPE> & x, const Decimal<TYPE> & y);
FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
#undef DISPATCH
template <typename T> Decimal<T> operator+ (const Decimal<T> & x, const Decimal<T> & y);
template <typename T> Decimal<T> operator- (const Decimal<T> & x, const Decimal<T> & y);
template <typename T> Decimal<T> operator* (const Decimal<T> & x, const Decimal<T> & y);
template <typename T> Decimal<T> operator/ (const Decimal<T> & x, const Decimal<T> & y);
template <typename T> Decimal<T> operator- (const Decimal<T> & x);
#define DISPATCH(TYPE) \
extern template Decimal<TYPE> operator+ (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
extern template Decimal<TYPE> operator- (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
extern template Decimal<TYPE> operator* (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
extern template Decimal<TYPE> operator/ (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
extern template Decimal<TYPE> operator- (const Decimal<TYPE> & x);
FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
#undef DISPATCH
#undef FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS
#undef FOR_EACH_UNDERLYING_DECIMAL_TYPE
/// Distinguishable type to allow function resolution/deduction based on value type,
/// but also relatively easy to convert to/from Decimal64.

38
base/base/extended_types.h
View File

@ -64,6 +64,44 @@ template <> struct is_arithmetic<UInt256> { static constexpr bool value = true;
template <typename T>
inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
#define FOR_EACH_ARITHMETIC_TYPE(M) \
M(DataTypeDate) \
M(DataTypeDate32) \
M(DataTypeDateTime) \
M(DataTypeInt8) \
M(DataTypeUInt8) \
M(DataTypeInt16) \
M(DataTypeUInt16) \
M(DataTypeInt32) \
M(DataTypeUInt32) \
M(DataTypeInt64) \
M(DataTypeUInt64) \
M(DataTypeInt128) \
M(DataTypeUInt128) \
M(DataTypeInt256) \
M(DataTypeUInt256) \
M(DataTypeFloat32) \
M(DataTypeFloat64)
#define FOR_EACH_ARITHMETIC_TYPE_PASS(M, X) \
M(DataTypeDate, X) \
M(DataTypeDate32, X) \
M(DataTypeDateTime, X) \
M(DataTypeInt8, X) \
M(DataTypeUInt8, X) \
M(DataTypeInt16, X) \
M(DataTypeUInt16, X) \
M(DataTypeInt32, X) \
M(DataTypeUInt32, X) \
M(DataTypeInt64, X) \
M(DataTypeUInt64, X) \
M(DataTypeInt128, X) \
M(DataTypeUInt128, X) \
M(DataTypeInt256, X) \
M(DataTypeUInt256, X) \
M(DataTypeFloat32, X) \
M(DataTypeFloat64, X)
template <typename T>
struct make_unsigned // NOLINT(readability-identifier-naming)
{

11
docker/images.json
View File

@ -1,8 +1,12 @@
{
"docker/packager/binary": {
"docker/packager/binary-builder": {
"name": "clickhouse/binary-builder",
"dependent": []
},
"docker/packager/cctools": {
"name": "clickhouse/cctools",
"dependent": []
},
"docker/test/compatibility/centos": {
"name": "clickhouse/test-old-centos",
"dependent": []
@ -30,7 +34,6 @@
"docker/test/util": {
"name": "clickhouse/test-util",
"dependent": [
"docker/packager/binary",
"docker/test/base",
"docker/test/fasttest"
]
@ -67,7 +70,9 @@
},
"docker/test/fasttest": {
"name": "clickhouse/fasttest",
"dependent": []
"dependent": [
"docker/packager/binary-builder"
]
},
"docker/test/style": {
"name": "clickhouse/style-test",

42
docker/packager/binary/Dockerfile → docker/packager/binary-builder/Dockerfile
View File

@ -1,43 +1,6 @@
# docker build -t clickhouse/binary-builder .
ARG FROM_TAG=latest
FROM clickhouse/test-util:latest AS cctools
# The cctools are built always from the clickhouse/test-util:latest and cached inline
# Theoretically, it should improve rebuild speed significantly
ENV CC=clang-${LLVM_VERSION}
ENV CXX=clang++-${LLVM_VERSION}
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# DO NOT PUT ANYTHING BEFORE THE NEXT TWO `RUN` DIRECTIVES
# THE MOST HEAVY OPERATION MUST BE THE FIRST IN THE CACHE
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# libtapi is required to support .tbh format from recent MacOS SDKs
RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \
&& cd apple-libtapi \
&& git checkout 15dfc2a8c9a2a89d06ff227560a69f5265b692f9 \
&& INSTALLPREFIX=/cctools ./build.sh \
&& ./install.sh \
&& cd .. \
&& rm -rf apple-libtapi
# Build and install tools for cross-linking to Darwin (x86-64)
# Build and install tools for cross-linking to Darwin (aarch64)
RUN git clone https://github.com/tpoechtrager/cctools-port.git \
&& cd cctools-port/cctools \
&& git checkout 2a3e1c2a6ff54a30f898b70cfb9ba1692a55fad7 \
&& ./configure --prefix=/cctools --with-libtapi=/cctools \
--target=x86_64-apple-darwin \
&& make install -j$(nproc) \
&& make clean \
&& ./configure --prefix=/cctools --with-libtapi=/cctools \
--target=aarch64-apple-darwin \
&& make install -j$(nproc) \
&& cd ../.. \
&& rm -rf cctools-port
# !!!!!!!!!!!
# END COMPILE
# !!!!!!!!!!!
FROM clickhouse/test-util:$FROM_TAG
FROM clickhouse/fasttest:$FROM_TAG
ENV CC=clang-${LLVM_VERSION}
ENV CXX=clang++-${LLVM_VERSION}
@ -110,7 +73,8 @@ RUN curl -Lo /usr/bin/clang-tidy-cache \
"https://raw.githubusercontent.com/matus-chochlik/ctcache/$CLANG_TIDY_SHA1/clang-tidy-cache" \
&& chmod +x /usr/bin/clang-tidy-cache
COPY --from=cctools /cctools /cctools
# If the cctools is updated, then first build it in the CI, then update here in a different commit
COPY --from=clickhouse/cctools:5a908f73878a /cctools /cctools
RUN mkdir /workdir && chmod 777 /workdir
WORKDIR /workdir

0
docker/packager/binary/build.sh → docker/packager/binary-builder/build.sh
View File

31
docker/packager/cctools/Dockerfile Normal file
View File

@ -0,0 +1,31 @@
# This is a hack to significantly reduce the build time of the clickhouse/binary-builder
# It's based on the assumption that we don't care of the cctools version so much
# It event does not depend on the clickhouse/fasttest in the `docker/images.json`
ARG FROM_TAG=latest
FROM clickhouse/fasttest:$FROM_TAG
ENV CC=clang-${LLVM_VERSION}
ENV CXX=clang++-${LLVM_VERSION}
RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \
&& cd apple-libtapi \
&& git checkout 15dfc2a8c9a2a89d06ff227560a69f5265b692f9 \
&& INSTALLPREFIX=/cctools ./build.sh \
&& ./install.sh \
&& cd .. \
&& rm -rf apple-libtapi
# Build and install tools for cross-linking to Darwin (x86-64)
# Build and install tools for cross-linking to Darwin (aarch64)
RUN git clone https://github.com/tpoechtrager/cctools-port.git \
&& cd cctools-port/cctools \
&& git checkout 2a3e1c2a6ff54a30f898b70cfb9ba1692a55fad7 \
&& ./configure --prefix=/cctools --with-libtapi=/cctools \
--target=x86_64-apple-darwin \
&& make install -j$(nproc) \
&& make clean \
&& ./configure --prefix=/cctools --with-libtapi=/cctools \
--target=aarch64-apple-darwin \
&& make install -j$(nproc) \
&& cd ../.. \
&& rm -rf cctools-port

8
docker/packager/packager
View File

@ -1,16 +1,16 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import subprocess
import os
import argparse
import logging
import os
import subprocess
import sys
from pathlib import Path
from typing import List, Optional
SCRIPT_PATH = Path(__file__).absolute()
IMAGE_TYPE = "binary"
IMAGE_NAME = f"clickhouse/{IMAGE_TYPE}-builder"
IMAGE_TYPE = "binary-builder"
IMAGE_NAME = f"clickhouse/{IMAGE_TYPE}"
class BuildException(Exception):

59
docker/test/fasttest/Dockerfile
View File

@ -6,9 +6,18 @@ FROM clickhouse/test-util:$FROM_TAG
RUN apt-get update \
&& apt-get install \
brotli \
clang-${LLVM_VERSION} \
clang-tidy-${LLVM_VERSION} \
cmake \
expect \
file \
libclang-${LLVM_VERSION}-dev \
libclang-rt-${LLVM_VERSION}-dev \
lld-${LLVM_VERSION} \
llvm-${LLVM_VERSION} \
llvm-${LLVM_VERSION}-dev \
lsof \
ninja-build \
odbcinst \
psmisc \
python3 \
@ -26,14 +35,50 @@ RUN apt-get update \
RUN pip3 install numpy==1.26.3 scipy==1.12.0 pandas==1.5.3 Jinja2==3.1.3
ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"
# This symlink is required by gcc to find the lld linker
RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
# for external_symbolizer_path
RUN ln -s /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer
# FIXME: workaround for "The imported target "merge-fdata" references the file" error
# https://salsa.debian.org/pkg-llvm-team/llvm-toolchain/-/commit/992e52c0b156a5ba9c6a8a54f8c4857ddd3d371d
RUN sed -i '/_IMPORT_CHECK_FILES_FOR_\(mlir-\|llvm-bolt\|merge-fdata\|MLIR\)/ {s|^|#|}' /usr/lib/llvm-${LLVM_VERSION}/lib/cmake/llvm/LLVMExports-*.cmake
RUN mkdir -p /tmp/clickhouse-odbc-tmp \
&& wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
&& cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \
&& odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
&& odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
&& rm -rf /tmp/clickhouse-odbc-tmp
ARG CCACHE_VERSION=4.6.1
RUN mkdir /tmp/ccache \
&& cd /tmp/ccache \
&& curl -L \
-O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz \
-O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz.asc \
&& gpg --recv-keys --keyserver hkps://keyserver.ubuntu.com 5A939A71A46792CF57866A51996DDA075594ADB8 \
&& gpg --verify ccache-4.6.1.tar.xz.asc \
&& tar xf ccache-$CCACHE_VERSION.tar.xz \
&& cd /tmp/ccache/ccache-$CCACHE_VERSION \
&& cmake -DCMAKE_INSTALL_PREFIX=/usr \
-DCMAKE_BUILD_TYPE=None \
-DZSTD_FROM_INTERNET=ON \
-DREDIS_STORAGE_BACKEND=OFF \
-Wno-dev \
-B build \
-S . \
&& make VERBOSE=1 -C build \
&& make install -C build \
&& cd / \
&& rm -rf /tmp/ccache
ARG TARGETARCH
ARG SCCACHE_VERSION=v0.7.7
ENV SCCACHE_IGNORE_SERVER_IO_ERROR=1
# sccache requires a value for the region. So by default we use The Default Region
ENV SCCACHE_REGION=us-east-1
RUN arch=${TARGETARCH:-amd64} \
&& case $arch in \
amd64) rarch=x86_64 ;; \
arm64) rarch=aarch64 ;; \
esac \
&& curl -Ls "https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl.tar.gz" | \
tar xz -C /tmp \
&& mv "/tmp/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl/sccache" /usr/bin \
&& rm "/tmp/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl" -r
# Give suid to gdb to grant it attach permissions
# chmod 777 to make the container user independent

7
docker/test/fuzzer/run-fuzzer.sh
View File

@ -343,10 +343,9 @@ quit
# which is confusing.
task_exit_code=$fuzzer_exit_code
echo "failure" > status.txt
{ rg -ao "Found error:.*" fuzzer.log \
|| rg -ao "Exception:.*" fuzzer.log \
|| echo "Fuzzer failed ($fuzzer_exit_code). See the logs." ; } \
| tail -1 > description.txt
echo "Achtung!" > description.txt
echo "Fuzzer went wrong with error code: ($fuzzer_exit_code). Its process died somehow when the server stayed alive. The server log probably won't tell you much so try to find information in other files." >>description.txt
{ rg -ao "Found error:.*" fuzzer.log || rg -ao "Exception:.*" fuzzer.log; } | tail -1 >>description.txt
fi
if test -f core.*; then

21
docker/test/sqllogic/Dockerfile
View File

@ -24,17 +24,18 @@ RUN pip3 install \
deepdiff \
sqlglot
ARG odbc_repo="https://github.com/ClickHouse/clickhouse-odbc.git"
ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz"
RUN mkdir -p /tmp/clickhouse-odbc-tmp \
&& cd /tmp/clickhouse-odbc-tmp \
&& curl -L ${odbc_driver_url} | tar --strip-components=1 -xz clickhouse-odbc-1.1.6-Linux \
&& mkdir /usr/local/lib64 -p \
&& cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib64/ \
&& odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
&& odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
&& sed -i 's"=libclickhouseodbc"=/usr/local/lib64/libclickhouseodbc"' /etc/odbcinst.ini \
&& rm -rf /tmp/clickhouse-odbc-tmp
RUN git clone --recursive ${odbc_repo} \
&& mkdir -p /clickhouse-odbc/build \
&& cmake -S /clickhouse-odbc -B /clickhouse-odbc/build \
&& ls /clickhouse-odbc/build/driver \
&& make -j 10 -C /clickhouse-odbc/build \
&& ls /clickhouse-odbc/build/driver \
&& mkdir -p /usr/local/lib64/ && cp /clickhouse-odbc/build/driver/lib*.so /usr/local/lib64/ \
&& odbcinst -i -d -f /clickhouse-odbc/packaging/odbcinst.ini.sample \
&& odbcinst -i -s -l -f /clickhouse-odbc/packaging/odbc.ini.sample
ENV TZ=Europe/Amsterdam
ENV MAX_RUN_TIME=9000

22
docker/test/stateless/Dockerfile
View File

@ -3,7 +3,7 @@
ARG FROM_TAG=latest
FROM clickhouse/test-base:$FROM_TAG
ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"
ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz"
# golang version 1.13 on Ubuntu 20 is enough for tests
RUN apt-get update -y \
@ -35,7 +35,6 @@ RUN apt-get update -y \
sudo \
tree \
unixodbc \
wget \
rustc \
cargo \
zstd \
@ -50,11 +49,14 @@ RUN apt-get update -y \
RUN pip3 install numpy==1.26.3 scipy==1.12.0 pandas==1.5.3 Jinja2==3.1.3 pyarrow==15.0.0
RUN mkdir -p /tmp/clickhouse-odbc-tmp \
&& wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
&& cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \
&& odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
&& odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
&& rm -rf /tmp/clickhouse-odbc-tmp
&& cd /tmp/clickhouse-odbc-tmp \
&& curl -L ${odbc_driver_url} | tar --strip-components=1 -xz clickhouse-odbc-1.1.6-Linux \
&& mkdir /usr/local/lib64 -p \
&& cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib64/ \
&& odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
&& odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
&& sed -i 's"=libclickhouseodbc"=/usr/local/lib64/libclickhouseodbc"' /etc/odbcinst.ini \
&& rm -rf /tmp/clickhouse-odbc-tmp
ENV TZ=Europe/Amsterdam
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
@ -70,11 +72,11 @@ ARG TARGETARCH
# Download Minio-related binaries
RUN arch=${TARGETARCH:-amd64} \
&& wget "https://dl.min.io/server/minio/release/linux-${arch}/archive/minio.RELEASE.${MINIO_SERVER_VERSION}" -O ./minio \
&& wget "https://dl.min.io/client/mc/release/linux-${arch}/archive/mc.RELEASE.${MINIO_CLIENT_VERSION}" -O ./mc \
&& curl -L "https://dl.min.io/server/minio/release/linux-${arch}/archive/minio.RELEASE.${MINIO_SERVER_VERSION}" -o ./minio \
&& curl -L "https://dl.min.io/client/mc/release/linux-${arch}/archive/mc.RELEASE.${MINIO_CLIENT_VERSION}" -o ./mc \
&& chmod +x ./mc ./minio
RUN wget --no-verbose 'https://archive.apache.org/dist/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \
RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \
&& tar -xvf hadoop-3.3.1.tar.gz \
&& rm -rf hadoop-3.3.1.tar.gz

1
docker/test/style/Dockerfile
View File

@ -60,5 +60,4 @@ RUN arch=${TARGETARCH:-amd64} \
COPY run.sh /
COPY process_style_check_result.py /
CMD ["/bin/bash", "/run.sh"]

54
docker/test/util/Dockerfile
View File

@ -41,20 +41,11 @@ RUN apt-get update \
bash \
bsdmainutils \
build-essential \
clang-${LLVM_VERSION} \
clang-tidy-${LLVM_VERSION} \
cmake \
gdb \
git \
gperf \
libclang-rt-${LLVM_VERSION}-dev \
lld-${LLVM_VERSION} \
llvm-${LLVM_VERSION} \
llvm-${LLVM_VERSION}-dev \
libclang-${LLVM_VERSION}-dev \
moreutils \
nasm \
ninja-build \
pigz \
rename \
software-properties-common \
@ -63,49 +54,4 @@ RUN apt-get update \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
# This symlink is required by gcc to find the lld linker
RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
# for external_symbolizer_path
RUN ln -s /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer
# FIXME: workaround for "The imported target "merge-fdata" references the file" error
# https://salsa.debian.org/pkg-llvm-team/llvm-toolchain/-/commit/992e52c0b156a5ba9c6a8a54f8c4857ddd3d371d
RUN sed -i '/_IMPORT_CHECK_FILES_FOR_\(mlir-\|llvm-bolt\|merge-fdata\|MLIR\)/ {s|^|#|}' /usr/lib/llvm-${LLVM_VERSION}/lib/cmake/llvm/LLVMExports-*.cmake
ARG CCACHE_VERSION=4.6.1
RUN mkdir /tmp/ccache \
&& cd /tmp/ccache \
&& curl -L \
-O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz \
-O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz.asc \
&& gpg --recv-keys --keyserver hkps://keyserver.ubuntu.com 5A939A71A46792CF57866A51996DDA075594ADB8 \
&& gpg --verify ccache-4.6.1.tar.xz.asc \
&& tar xf ccache-$CCACHE_VERSION.tar.xz \
&& cd /tmp/ccache/ccache-$CCACHE_VERSION \
&& cmake -DCMAKE_INSTALL_PREFIX=/usr \
-DCMAKE_BUILD_TYPE=None \
-DZSTD_FROM_INTERNET=ON \
-DREDIS_STORAGE_BACKEND=OFF \
-Wno-dev \
-B build \
-S . \
&& make VERBOSE=1 -C build \
&& make install -C build \
&& cd / \
&& rm -rf /tmp/ccache
ARG TARGETARCH
ARG SCCACHE_VERSION=v0.5.4
ENV SCCACHE_IGNORE_SERVER_IO_ERROR=1
# sccache requires a value for the region. So by default we use The Default Region
ENV SCCACHE_REGION=us-east-1
RUN arch=${TARGETARCH:-amd64} \
&& case $arch in \
amd64) rarch=x86_64 ;; \
arm64) rarch=aarch64 ;; \
esac \
&& curl -Ls "https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl.tar.gz" | \
tar xz -C /tmp \
&& mv "/tmp/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl/sccache" /usr/bin \
&& rm "/tmp/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl" -r
COPY process_functional_tests_result.py /

1
docs/en/operations/settings/settings.md
View File

@ -3954,6 +3954,7 @@ Possible values:
- `none` — Is similar to throw, but distributed DDL query returns no result set.
- `null_status_on_timeout` — Returns `NULL` as execution status in some rows of result set instead of throwing `TIMEOUT_EXCEEDED` if query is not finished on the corresponding hosts.
- `never_throw` — Do not throw `TIMEOUT_EXCEEDED` and do not rethrow exceptions if query has failed on some hosts.
- `none_only_active` - similar to `none`, but doesn't wait for inactive replicas of the `Replicated` database. Note: with this mode it's impossible to figure out that the query was not executed on some replica and will be executed in background.
- `null_status_on_timeout_only_active` — similar to `null_status_on_timeout`, but doesn't wait for inactive replicas of the `Replicated` database
- `throw_only_active` — similar to `throw`, but doesn't wait for inactive replicas of the `Replicated` database

4
docs/en/operations/utilities/clickhouse-local.md
View File

@ -201,12 +201,12 @@ Arguments:
- `-S`, `--structure` — table structure for input data.
- `--input-format` — input format, `TSV` by default.
- `-f`, `--file` — path to data, `stdin` by default.
- `-F`, `--file` — path to data, `stdin` by default.
- `-q`, `--query` — queries to execute with `;` as delimiter. `--query` can be specified multiple times, e.g. `--query "SELECT 1" --query "SELECT 2"`. Cannot be used simultaneously with `--queries-file`.
- `--queries-file` - file path with queries to execute. `--queries-file` can be specified multiple times, e.g. `--query queries1.sql --query queries2.sql`. Cannot be used simultaneously with `--query`.
- `--multiquery, -n` – If specified, multiple queries separated by semicolons can be listed after the `--query` option. For convenience, it is also possible to omit `--query` and pass the queries directly after `--multiquery`.
- `-N`, `--table` — table name where to put output data, `table` by default.
- `--format`, `--output-format` — output format, `TSV` by default.
- `-f`, `--format`, `--output-format` — output format, `TSV` by default.
- `-d`, `--database` — default database, `_local` by default.
- `--stacktrace` — whether to dump debug output in case of exception.
- `--echo` — print query before execution.

60
docs/en/sql-reference/functions/date-time-functions.md
View File

@ -394,8 +394,7 @@ Result:
## toYear
Converts a date or date with time to the year number (AD) as `UInt16` value.
Returns the year component (AD) of a date or date with time.
**Syntax**
@ -431,7 +430,7 @@ Result:
## toQuarter
Converts a date or date with time to the quarter number (1-4) as `UInt8` value.
Returns the quarter (1-4) of a date or date with time.
**Syntax**
@ -465,10 +464,9 @@ Result:
└──────────────────────────────────────────────┘
```
## toMonth
Converts a date or date with time to the month number (1-12) as `UInt8` value.
Returns the month component (1-12) of a date or date with time.
**Syntax**
@ -504,7 +502,7 @@ Result:
## toDayOfYear
Converts a date or date with time to the number of the day of the year (1-366) as `UInt16` value.
Returns the number of the day within the year (1-366) of a date or date with time.
**Syntax**
@ -540,7 +538,7 @@ Result:
## toDayOfMonth
Converts a date or date with time to the number of the day in the month (1-31) as `UInt8` value.
Returns the number of the day within the month (1-31) of a date or date with time.
**Syntax**
@ -576,7 +574,7 @@ Result:
## toDayOfWeek
Converts a date or date with time to the number of the day in the week as `UInt8` value.
Returns the number of the day within the week of a date or date with time.
The two-argument form of `toDayOfWeek()` enables you to specify whether the week starts on Monday or Sunday, and whether the return value should be in the range from 0 to 6 or 1 to 7. If the mode argument is omitted, the default mode is 0. The time zone of the date can be specified as the third argument.
@ -627,7 +625,7 @@ Result:
## toHour
Converts a date with time to the number of the hour in 24-hour time (0-23) as `UInt8` value.
Returns the hour component (0-24) of a date with time.
Assumes that if clocks are moved ahead, it is by one hour and occurs at 2 a.m., and if clocks are moved back, it is by one hour and occurs at 3 a.m. (which is not always exactly when it occurs - it depends on the timezone).
@ -641,7 +639,7 @@ Alias: `HOUR`
**Arguments**
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
**Returned value**
@ -665,7 +663,7 @@ Result:
## toMinute
Converts a date with time to the number of the minute of the hour (0-59) as `UInt8` value.
Returns the minute component (0-59) a date with time.
**Syntax**
@ -677,7 +675,7 @@ Alias: `MINUTE`
**Arguments**
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
**Returned value**
@ -701,7 +699,7 @@ Result:
## toSecond
Converts a date with time to the second in the minute (0-59) as `UInt8` value. Leap seconds are not considered.
Returns the second component (0-59) of a date with time. Leap seconds are not considered.
**Syntax**
@ -713,7 +711,7 @@ Alias: `SECOND`
**Arguments**
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
**Returned value**
@ -735,6 +733,40 @@ Result:
└─────────────────────────────────────────────┘
```
## toMillisecond
Returns the millisecond component (0-999) of a date with time.
**Syntax**
```sql
toMillisecond(value)
```
*Arguments**
- `value` - [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
Alias: `MILLISECOND`
```sql
SELECT toMillisecond(toDateTime64('2023-04-21 10:20:30.456', 3))
```
Result:
```response
┌──toMillisecond(toDateTime64('2023-04-21 10:20:30.456', 3))─┐
│ 456 │
└────────────────────────────────────────────────────────────┘
```
**Returned value**
- The millisecond in the minute (0 - 59) of the given date/time
Type: `UInt16`
## toUnixTimestamp
Converts a string, a date or a date with time to the [Unix Timestamp](https://en.wikipedia.org/wiki/Unix_time) in `UInt32` representation.

26
docs/en/sql-reference/statements/grant.md
View File

@ -202,6 +202,13 @@ Hierarchy of privileges:
- `S3`
- [dictGet](#grant-dictget)
- [displaySecretsInShowAndSelect](#grant-display-secrets)
- [NAMED COLLECTION ADMIN](#grant-named-collection-admin)
- `CREATE NAMED COLLECTION`
- `DROP NAMED COLLECTION`
- `ALTER NAMED COLLECTION`
- `SHOW NAMED COLLECTIONS`
- `SHOW NAMED COLLECTIONS SECRETS`
- `NAMED COLLECTION`
Examples of how this hierarchy is treated:
@ -498,6 +505,25 @@ and
[`format_display_secrets_in_show_and_select` format setting](../../operations/settings/formats#format_display_secrets_in_show_and_select)
are turned on.
### NAMED COLLECTION ADMIN
Allows a certain operation on a specified named collection. Before version 23.7 it was called NAMED COLLECTION CONTROL, and after 23.7 NAMED COLLECTION ADMIN was added and NAMED COLLECTION CONTROL is preserved as an alias.
- `NAMED COLLECTION ADMIN`. Level: `NAMED_COLLECTION`. Aliases: `NAMED COLLECTION CONTROL`
- `CREATE NAMED COLLECTION`. Level: `NAMED_COLLECTION`
- `DROP NAMED COLLECTION`. Level: `NAMED_COLLECTION`
- `ALTER NAMED COLLECTION`. Level: `NAMED_COLLECTION`
- `SHOW NAMED COLLECTIONS`. Level: `NAMED_COLLECTION`. Aliases: `SHOW NAMED COLLECTIONS`
- `SHOW NAMED COLLECTIONS SECRETS`. Level: `NAMED_COLLECTION`. Aliases: `SHOW NAMED COLLECTIONS SECRETS`
- `NAMED COLLECTION`. Level: `NAMED_COLLECTION`. Aliases: `NAMED COLLECTION USAGE, USE NAMED COLLECTION`
Unlike all other grants (CREATE, DROP, ALTER, SHOW) grant NAMED COLLECTION was added only in 23.7, while all others were added earlier - in 22.12.
**Examples**
Assuming a named collection is called abc, we grant privilege CREATE NAMED COLLECTION to user john.
- `GRANT CREATE NAMED COLLECTION ON abc TO john`
### ALL
Grants all the privileges on regulated entity to a user account or a role.

108
docs/en/sql-reference/window-functions/index.md
View File

@ -5,7 +5,12 @@ sidebar_label: Window Functions
title: Window Functions
---
ClickHouse supports the standard grammar for defining windows and window functions. The following features are currently supported:
Windows functions let you perform calculations across a set of rows that are related to the current row.
Some of the calculations that you can do are similar to those that can be done with an aggregate function, but a window function doesn't cause rows to be grouped into a single output - the individual rows are still returned.
## Standard Window Functions
ClickHouse supports the standard grammar for defining windows and window functions. The table below indicates whether a feature is currently supported.
| Feature | Support or workaround |
|------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@ -25,6 +30,8 @@ ClickHouse supports the standard grammar for defining windows and window functio
## ClickHouse-specific Window Functions
There are also the following window function that's specific to ClickHouse:
### nonNegativeDerivative(metric_column, timestamp_column[, INTERVAL X UNITS])
Finds non-negative derivative for given `metric_column` by `timestamp_column`.
@ -33,40 +40,6 @@ The computed value is the following for each row:
- `0` for 1st row,
- ${metric_i - metric_{i-1} \over timestamp_i - timestamp_{i-1}} * interval$ for $i_th$ row.
## References
### GitHub Issues
The roadmap for the initial support of window functions is [in this issue](https://github.com/ClickHouse/ClickHouse/issues/18097).
All GitHub issues related to window functions have the [comp-window-functions](https://github.com/ClickHouse/ClickHouse/labels/comp-window-functions) tag.
### Tests
These tests contain the examples of the currently supported grammar:
https://github.com/ClickHouse/ClickHouse/blob/master/tests/performance/window_functions.xml
https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/01591_window_functions.sql
### Postgres Docs
https://www.postgresql.org/docs/current/sql-select.html#SQL-WINDOW
https://www.postgresql.org/docs/devel/sql-expressions.html#SYNTAX-WINDOW-FUNCTIONS
https://www.postgresql.org/docs/devel/functions-window.html
https://www.postgresql.org/docs/devel/tutorial-window.html
### MySQL Docs
https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html
https://dev.mysql.com/doc/refman/8.0/en/window-functions-usage.html
https://dev.mysql.com/doc/refman/8.0/en/window-functions-frames.html
## Syntax
```text
@ -80,20 +53,7 @@ WINDOW window_name as ([[PARTITION BY grouping_column] [ORDER BY sorting_column]
- `PARTITION BY` - defines how to break a resultset into groups.
- `ORDER BY` - defines how to order rows inside the group during calculation aggregate_function.
- `ROWS or RANGE` - defines bounds of a frame, aggregate_function is calculated within a frame.
- `WINDOW` - allows to reuse a window definition with multiple expressions.
### Functions
These functions can be used only as a window function.
- `row_number()` - Number the current row within its partition starting from 1.
- `first_value(x)` - Return the first non-NULL value evaluated within its ordered frame.
- `last_value(x)` - Return the last non-NULL value evaluated within its ordered frame.
- `nth_value(x, offset)` - Return the first non-NULL value evaluated against the nth row (offset) in its ordered frame.
- `rank()` - Rank the current row within its partition with gaps.
- `dense_rank()` - Rank the current row within its partition without gaps.
- `lagInFrame(x)` - Return a value evaluated at the row that is at a specified physical offset row before the current row within the ordered frame.
- `leadInFrame(x)` - Return a value evaluated at the row that is offset rows after the current row within the ordered frame.
- `WINDOW` - allows multiple expressions to use the same window definition.
```text
PARTITION
@ -112,8 +72,23 @@ These functions can be used only as a window function.
└─────────────────┘ <--- UNBOUNDED FOLLOWING (END of the PARTITION)
```
### Functions
These functions can be used only as a window function.
- `row_number()` - Number the current row within its partition starting from 1.
- `first_value(x)` - Return the first non-NULL value evaluated within its ordered frame.
- `last_value(x)` - Return the last non-NULL value evaluated within its ordered frame.
- `nth_value(x, offset)` - Return the first non-NULL value evaluated against the nth row (offset) in its ordered frame.
- `rank()` - Rank the current row within its partition with gaps.
- `dense_rank()` - Rank the current row within its partition without gaps.
- `lagInFrame(x)` - Return a value evaluated at the row that is at a specified physical offset row before the current row within the ordered frame.
- `leadInFrame(x)` - Return a value evaluated at the row that is offset rows after the current row within the ordered frame.
## Examples
Let's have a look at some examples of how window functions can be used.
```sql
CREATE TABLE wf_partition
(
@ -589,6 +564,41 @@ ORDER BY
└──────────────┴─────────────────────┴───────┴─────────────────────────┘
```
## References
### GitHub Issues
The roadmap for the initial support of window functions is [in this issue](https://github.com/ClickHouse/ClickHouse/issues/18097).
All GitHub issues related to window functions have the [comp-window-functions](https://github.com/ClickHouse/ClickHouse/labels/comp-window-functions) tag.
### Tests
These tests contain the examples of the currently supported grammar:
https://github.com/ClickHouse/ClickHouse/blob/master/tests/performance/window_functions.xml
https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/01591_window_functions.sql
### Postgres Docs
https://www.postgresql.org/docs/current/sql-select.html#SQL-WINDOW
https://www.postgresql.org/docs/devel/sql-expressions.html#SYNTAX-WINDOW-FUNCTIONS
https://www.postgresql.org/docs/devel/functions-window.html
https://www.postgresql.org/docs/devel/tutorial-window.html
### MySQL Docs
https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html
https://dev.mysql.com/doc/refman/8.0/en/window-functions-usage.html
https://dev.mysql.com/doc/refman/8.0/en/window-functions-frames.html
## Related Content
- Blog: [Working with time series data in ClickHouse](https://clickhouse.com/blog/working-with-time-series-data-and-functions-ClickHouse)

37
docs/zh/operations/settings/settings.md
View File

@ -649,11 +649,22 @@ log_query_threads=1
## max_query_size {#settings-max_query_size}
查询的最大部分，可以被带到RAM用于使用SQL解析器进行解析。
插入查询还包含由单独的流解析器（消耗O(1)RAM）处理的插入数据，这些数据不包含在此限制中。
SQL 解析器解析的查询字符串的最大字节数。 INSERT 查询的 VALUES 子句中的数据由单独的流解析器（消耗 O(1) RAM）处理，并且不受此限制的影响。
默认值：256KiB。
## max_parser_depth {#max_parser_depth}
限制递归下降解析器中的最大递归深度。允许控制堆栈大小。
可能的值：
- 正整数。
- 0 — 递归深度不受限制。
默认值：1000。
## interactive_delay {#interactive-delay}
以微秒为单位的间隔，用于检查请求执行是否已被取消并发送进度。
@ -1064,6 +1075,28 @@ ClickHouse生成异常
默认值：0。
## optimize_functions_to_subcolumns {#optimize_functions_to_subcolumns}
启用或禁用通过将某些函数转换为读取子列的优化。这减少了要读取的数据量。
这些函数可以转化为：
- [length](../../sql-reference/functions/array-functions.md/#array_functions-length) 读取 [size0](../../sql-reference/data-types/array.md/#array-size）子列。
- [empty](../../sql-reference/functions/array-functions.md/#empty函数) 读取 [size0](../../sql-reference/data-types/array.md/#array-size）子列。
- [notEmpty](../../sql-reference/functions/array-functions.md/#notempty函数) 读取 [size0](../../sql-reference/data-types/array.md/#array-size）子列。
- [isNull](../../sql-reference/operators/index.md#operator-is-null) 读取 [null](../../sql-reference/data-types/nullable. md/#finding-null) 子列。
- [isNotNull](../../sql-reference/operators/index.md#is-not-null) 读取 [null](../../sql-reference/data-types/nullable. md/#finding-null) 子列。
- [count](../../sql-reference/aggregate-functions/reference/count.md) 读取 [null](../../sql-reference/data-types/nullable.md/#finding-null) 子列。
- [mapKeys](../../sql-reference/functions/tuple-map-functions.mdx/#mapkeys) 读取 [keys](../../sql-reference/data-types/map.md/#map-subcolumns) 子列。
- [mapValues](../../sql-reference/functions/tuple-map-functions.mdx/#mapvalues) 读取 [values](../../sql-reference/data-types/map.md/#map-subcolumns) 子列。
可能的值：
- 0 — 禁用优化。
- 1 — 优化已启用。
默认值：`0`。
## distributed_replica_error_half_life {#settings-distributed_replica_error_half_life}
- 类型：秒

26
docs/zh/sql-reference/data-types/array.md
View File

@ -1,7 +1,7 @@
---
slug: /zh/sql-reference/data-types/array
---
# 阵列(T) {#data-type-array}
# 数组(T) {#data-type-array}
由 `T` 类型元素组成的数组。
@ -66,3 +66,27 @@ SELECT array(1, 'a')
Received exception from server (version 1.1.54388):
Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: There is no supertype for types UInt8, String because some of them are String/FixedString and some of them are not.
```
## 数组大小 {#array-size}
可以使用 `size0` 子列找到数组的大小，而无需读取整个列。对于多维数组，您可以使用 `sizeN-1`，其中 `N` 是所需的维度。
**例子**
SQL查询：
```sql
CREATE TABLE t_arr (`arr` Array(Array(Array(UInt32)))) ENGINE = MergeTree ORDER BY tuple();
INSERT INTO t_arr VALUES ([[[12, 13, 0, 1],[12]]]);
SELECT arr.size0, arr.size1, arr.size2 FROM t_arr;
```
结果：
``` text
┌─arr.size0─┬─arr.size1─┬─arr.size2─┐
│ 1 │ [2] │ [[4,1]] │
└───────────┴───────────┴───────────┘
```

27
docs/zh/sql-reference/data-types/nullable.md
View File

@ -20,6 +20,33 @@ slug: /zh/sql-reference/data-types/nullable
掩码文件中的条目允许ClickHouse区分每个表行的对应数据类型的«NULL»和默认值由于有额外的文件，«Nullable»列比普通列消耗更多的存储空间
## null子列 {#finding-null}
通过使用 `null` 子列可以在列中查找 `NULL` 值，而无需读取整个列。如果对应的值为 `NULL`，则返回 `1`，否则返回 `0`。
**示例**
SQL查询:
``` sql
CREATE TABLE nullable (`n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple();
INSERT INTO nullable VALUES (1) (NULL) (2) (NULL);
SELECT n.null FROM nullable;
```
结果:
``` text
┌─n.null─┐
│ 0 │
│ 1 │
│ 0 │
│ 1 │
└────────┘
```
## 用法示例 {#yong-fa-shi-li}
``` sql

2
programs/copier/Internals.cpp
View File

@ -259,7 +259,7 @@ ShardPriority getReplicasPriority(const Cluster::Addresses & replicas, const std
res.is_remote = 1;
for (const auto & replica : replicas)
{
if (isLocalAddress(DNSResolver::instance().resolveHost(replica.host_name)))
if (isLocalAddress(DNSResolver::instance().resolveHostAllInOriginOrder(replica.host_name).front()))
{
res.is_remote = 0;
break;

9
rust/Cargo.lock generated
View File

@ -6,6 +6,7 @@ version = 3
name = "_ch_rust_prql"
version = "0.1.0"
dependencies = [
"anstream",
"prqlc",
"serde_json",
]
@ -698,9 +699,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.7.1"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7"
dependencies = [
"adler",
]
@ -751,9 +752,9 @@ dependencies = [
[[package]]
name = "object"
version = "0.32.1"
version = "0.32.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
dependencies = [
"memchr",
]

1
rust/prql/Cargo.toml
View File

@ -4,6 +4,7 @@ name = "_ch_rust_prql"
version = "0.1.0"
[dependencies]
anstream = {version = "0.6.12"}
prqlc = {version = "0.11.3", default-features = false}
serde_json = "1.0"

54
rust/prql/src/lib.rs
View File

@ -39,6 +39,11 @@ pub unsafe extern "C" fn prql_to_sql_impl(
};
if let Ok(sql_str) = prqlc::compile(&query_str, &opts) {
// NOTE: Over at PRQL we're considering to un-deprecate & re-enable the
// `color: false` option. If that happens, we can remove the `strip_str`
// here, which strips color codes from the output.
use anstream::adapter::strip_str;
let sql_str = strip_str(&sql_str).to_string();
set_output(sql_str, out, out_size);
0
} else {
@ -54,17 +59,50 @@ pub unsafe extern "C" fn prql_to_sql(
out: *mut *mut u8,
out_size: *mut u64,
) -> i64 {
let ret = panic::catch_unwind(|| {
return prql_to_sql_impl(query, size, out, out_size);
});
return match ret {
// NOTE: using cxxbridge we can return proper Result<> type.
Err(_err) => 1,
Ok(res) => res,
}
// NOTE: using cxxbridge we can return proper Result<> type.
panic::catch_unwind(|| prql_to_sql_impl(query, size, out, out_size)).unwrap_or_else(|_| {
set_output("prqlc panicked".to_string(), out, out_size);
1
})
}
#[no_mangle]
pub unsafe extern "C" fn prql_free_pointer(ptr_to_free: *mut u8) {
std::mem::drop(CString::from_raw(ptr_to_free as *mut c_char));
}
#[cfg(test)]
mod tests {
use super::*;
use std::ffi::{CStr, CString};
/// A test helper to offer a rust interface to the C bindings
fn run_compile(query: &str) -> (String, i64) {
let query_cstr = CString::new(query).unwrap();
let query_ptr = query_cstr.as_ptr() as *const u8;
let query_size = query_cstr.to_bytes_with_nul().len() as u64 - 1; // Excluding the null terminator
let mut out: *mut u8 = std::ptr::null_mut();
let mut out_size = 0_u64;
unsafe {
let success = prql_to_sql(query_ptr, query_size, &mut out, &mut out_size);
let output = CStr::from_ptr(out as *const i8)
.to_str()
.unwrap()
.to_string();
prql_free_pointer(out);
(output, success)
}
}
#[test]
fn test_prql_to_sql() {
assert!(run_compile("from x").0.contains("SELECT"));
assert!(run_compile("asdf").1 == 1);
// In prqlc 0.11.3, this is a panic, so that allows us to test that the
// panic is caught. When we upgrade prqlc, it won't be a panic any
// longer.
assert!(run_compile("x -> y").1 == 1);
}
}

2
src/Access/Common/AllowedClientHosts.cpp
View File

@ -55,7 +55,7 @@ namespace
{
IPAddress addr_v6 = toIPv6(address);
auto host_addresses = DNSResolver::instance().resolveHostAll(host);
auto host_addresses = DNSResolver::instance().resolveHostAllInOriginOrder(host);
for (const auto & addr : host_addresses)
{

90
src/Access/Common/QuotaDefs.cpp
View File

@ -49,71 +49,135 @@ String QuotaTypeInfo::valueToStringWithName(QuotaValue value) const
const QuotaTypeInfo & QuotaTypeInfo::get(QuotaType type)
{
static constexpr auto make_info = [](const char * raw_name_, UInt64 output_denominator_)
static constexpr auto make_info = [](const char * raw_name_, String current_usage_description_, String max_allowed_usage_description_, UInt64 output_denominator_)
{
String init_name = raw_name_;
boost::to_lower(init_name);
String init_keyword = raw_name_;
boost::replace_all(init_keyword, "_", " ");
bool init_output_as_float = (output_denominator_ != 1);
return QuotaTypeInfo{raw_name_, std::move(init_name), std::move(init_keyword), init_output_as_float, output_denominator_};
return QuotaTypeInfo
{
.raw_name = raw_name_,
.name = std::move(init_name),
.keyword = std::move(init_keyword),
.current_usage_description = std::move(current_usage_description_),
.max_allowed_usage_description = std::move(max_allowed_usage_description_),
.output_as_float = init_output_as_float,
.output_denominator = output_denominator_
};
};
switch (type)
{
case QuotaType::QUERIES:
{
static const auto info = make_info("QUERIES", 1);
static const auto info = make_info(
"QUERIES",
"The current number of executed queries.",
"The maximum allowed number of queries of all types allowed to be executed.",
1
);
return info;
}
case QuotaType::QUERY_SELECTS:
{
static const auto info = make_info("QUERY_SELECTS", 1);
static const auto info = make_info(
"QUERY_SELECTS",
"The current number of executed SELECT queries.",
"The maximum allowed number of SELECT queries allowed to be executed.",
1
);
return info;
}
case QuotaType::QUERY_INSERTS:
{
static const auto info = make_info("QUERY_INSERTS", 1);
static const auto info = make_info(
"QUERY_INSERTS",
"The current number of executed INSERT queries.",
"The maximum allowed number of INSERT queries allowed to be executed.",
1
);
return info;
}
case QuotaType::ERRORS:
{
static const auto info = make_info("ERRORS", 1);
static const auto info = make_info(
"ERRORS",
"The current number of queries resulted in an error.",
"The maximum number of queries resulted in an error allowed within the specified period of time.",
1
);
return info;
}
case QuotaType::RESULT_ROWS:
{
static const auto info = make_info("RESULT_ROWS", 1);
static const auto info = make_info(
"RESULT_ROWS",
"The current total number of rows in the result set of all queries within the current period of time.",
"The maximum total number of rows in the result set of all queries allowed within the specified period of time.",
1
);
return info;
}
case QuotaType::RESULT_BYTES:
{
static const auto info = make_info("RESULT_BYTES", 1);
static const auto info = make_info(
"RESULT_BYTES",
"The current total number of bytes in the result set of all queries within the current period of time.",
"The maximum total number of bytes in the result set of all queries allowed within the specified period of time.",
1
);
return info;
}
case QuotaType::READ_ROWS:
{
static const auto info = make_info("READ_ROWS", 1);
static const auto info = make_info(
"READ_ROWS",
"The current total number of rows read during execution of all queries within the current period of time.",
"The maximum number of rows to read during execution of all queries allowed within the specified period of time.",
1
);
return info;
}
case QuotaType::READ_BYTES:
{
static const auto info = make_info("READ_BYTES", 1);
static const auto info = make_info(
"READ_BYTES",
"The current total number of bytes read during execution of all queries within the current period of time.",
"The maximum number of bytes to read during execution of all queries allowed within the specified period of time.",
1
);
return info;
}
case QuotaType::EXECUTION_TIME:
{
static const auto info = make_info("EXECUTION_TIME", 1000000000 /* execution_time is stored in nanoseconds */);
static const auto info = make_info(
"EXECUTION_TIME",
"The current total amount of time (in nanoseconds) spent to execute queries within the current period of time",
"The maximum amount of time (in nanoseconds) allowed for all queries to execute within the specified period of time",
1000000000 /* execution_time is stored in nanoseconds */
);
return info;
}
case QuotaType::WRITTEN_BYTES:
{
static const auto info = make_info("WRITTEN_BYTES", 1);
static const auto info = make_info(
"WRITTEN_BYTES",
"The current total number of bytes written during execution of all queries within the current period of time.",
"The maximum number of bytes to written during execution of all queries allowed within the specified period of time.",
1
);
return info;
}
case QuotaType::FAILED_SEQUENTIAL_AUTHENTICATIONS:
{
static const auto info = make_info("FAILED_SEQUENTIAL_AUTHENTICATIONS", 1);
static const auto info = make_info(
"FAILED_SEQUENTIAL_AUtheNTICATIONS",
"The current number of consecutive authentication failures within the current period of time.",
"The maximum number of consecutive authentication failures allowed within the specified period of time.",
1
);
return info;
}
case QuotaType::MAX: break;

2
src/Access/Common/QuotaDefs.h
View File

@ -33,6 +33,8 @@ struct QuotaTypeInfo
const char * const raw_name = "";
const String name; /// Lowercased with underscores, e.g. "result_rows".
const String keyword; /// Uppercased with spaces, e.g. "RESULT ROWS".
const String current_usage_description;
const String max_allowed_usage_description;
const bool output_as_float = false;
const UInt64 output_denominator = 1;
String valueToString(QuotaValue value) const;

9
src/Access/Common/RowPolicyDefs.cpp
View File

@ -33,7 +33,7 @@ String toString(RowPolicyFilterType type)
const RowPolicyFilterTypeInfo & RowPolicyFilterTypeInfo::get(RowPolicyFilterType type_)
{
static constexpr auto make_info = [](const char * raw_name_)
static constexpr auto make_info = [](const char * raw_name_, const String & comment_)
{
String init_name = raw_name_;
boost::to_lower(init_name);
@ -41,14 +41,17 @@ const RowPolicyFilterTypeInfo & RowPolicyFilterTypeInfo::get(RowPolicyFilterType
String init_command = init_name.substr(0, underscore_pos);
boost::to_upper(init_command);
bool init_is_check = (std::string_view{init_name}.substr(underscore_pos + 1) == "check");
return RowPolicyFilterTypeInfo{raw_name_, std::move(init_name), std::move(init_command), init_is_check};
return RowPolicyFilterTypeInfo{raw_name_, std::move(init_name), std::move(init_command), comment_, init_is_check};
};
switch (type_)
{
case RowPolicyFilterType::SELECT_FILTER:
{
static const auto info = make_info("SELECT_FILTER");
static const auto info = make_info(
"SELECT_FILTER",
"Expression which is used for filtering in SELECT queries."
);
return info;
}
#if 0 /// Row-level security for INSERT, UPDATE, DELETE is not implemented yet.

1
src/Access/Common/RowPolicyDefs.h
View File

@ -52,6 +52,7 @@ struct RowPolicyFilterTypeInfo
const char * const raw_name;
const String name; /// Lowercased with underscores, e.g. "select_filter".
const String command; /// Uppercased without last word, e.g. "SELECT".
const String description;
const bool is_check; /// E.g. false for SELECT_FILTER.
static const RowPolicyFilterTypeInfo & get(RowPolicyFilterType type);
};

114
src/Analyzer/ConstantNode.cpp
View File

@ -1,5 +1,7 @@
#include <Analyzer/ConstantNode.h>
#include <Analyzer/FunctionNode.h>
#include <Common/assert_cast.h>
#include <Common/FieldVisitorToString.h>
#include <Common/SipHash.h>
@ -38,52 +40,9 @@ ConstantNode::ConstantNode(Field value_)
: ConstantNode(value_, applyVisitor(FieldToDataType(), value_))
{}
void ConstantNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const
{
buffer << std::string(indent, ' ') << "CONSTANT id: " << format_state.getNodeId(this);
if (hasAlias())
buffer << ", alias: " << getAlias();
buffer << ", constant_value: " << constant_value->getValue().dump();
buffer << ", constant_value_type: " << constant_value->getType()->getName();
if (getSourceExpression())
{
buffer << '\n' << std::string(indent + 2, ' ') << "EXPRESSION" << '\n';
getSourceExpression()->dumpTreeImpl(buffer, format_state, indent + 4);
}
}
bool ConstantNode::isEqualImpl(const IQueryTreeNode & rhs) const
{
const auto & rhs_typed = assert_cast<const ConstantNode &>(rhs);
return *constant_value == *rhs_typed.constant_value && value_string == rhs_typed.value_string;
}
void ConstantNode::updateTreeHashImpl(HashState & hash_state) const
{
auto type_name = constant_value->getType()->getName();
hash_state.update(type_name.size());
hash_state.update(type_name);
hash_state.update(value_string.size());
hash_state.update(value_string);
}
QueryTreeNodePtr ConstantNode::cloneImpl() const
{
return std::make_shared<ConstantNode>(constant_value, source_expression);
}
ASTPtr ConstantNode::toASTImpl(const ConvertToASTOptions & options) const
bool ConstantNode::requiresCastCall() const
{
const auto & constant_value_literal = constant_value->getValue();
auto constant_value_ast = std::make_shared<ASTLiteral>(constant_value_literal);
if (!options.add_cast_for_constants)
return constant_value_ast;
bool need_to_add_cast_function = false;
auto constant_value_literal_type = constant_value_literal.getType();
WhichDataType constant_value_type(constant_value->getType());
@ -131,7 +90,72 @@ ASTPtr ConstantNode::toASTImpl(const ConvertToASTOptions & options) const
// Add cast if constant was created as a result of constant folding.
// Constant folding may lead to type transformation and literal on shard
// may have a different type.
if (need_to_add_cast_function || source_expression != nullptr)
return need_to_add_cast_function || source_expression != nullptr;
}
bool ConstantNode::receivedFromInitiatorServer() const
{
if (!hasSourceExpression())
return false;
auto * cast_function = getSourceExpression()->as<FunctionNode>();
if (!cast_function || cast_function->getFunctionName() != "_CAST")
return false;
return true;
}
void ConstantNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const
{
buffer << std::string(indent, ' ') << "CONSTANT id: " << format_state.getNodeId(this);
if (hasAlias())
buffer << ", alias: " << getAlias();
buffer << ", constant_value: ";
if (mask_id)
buffer << "[HIDDEN id: " << mask_id << "]";
else
buffer << constant_value->getValue().dump();
buffer << ", constant_value_type: " << constant_value->getType()->getName();
if (!mask_id && getSourceExpression())
{
buffer << '\n' << std::string(indent + 2, ' ') << "EXPRESSION" << '\n';
getSourceExpression()->dumpTreeImpl(buffer, format_state, indent + 4);
}
}
bool ConstantNode::isEqualImpl(const IQueryTreeNode & rhs) const
{
const auto & rhs_typed = assert_cast<const ConstantNode &>(rhs);
return *constant_value == *rhs_typed.constant_value && value_string == rhs_typed.value_string;
}
void ConstantNode::updateTreeHashImpl(HashState & hash_state) const
{
auto type_name = constant_value->getType()->getName();
hash_state.update(type_name.size());
hash_state.update(type_name);
hash_state.update(value_string.size());
hash_state.update(value_string);
}
QueryTreeNodePtr ConstantNode::cloneImpl() const
{
return std::make_shared<ConstantNode>(constant_value, source_expression);
}
ASTPtr ConstantNode::toASTImpl(const ConvertToASTOptions & options) const
{
const auto & constant_value_literal = constant_value->getValue();
auto constant_value_ast = std::make_shared<ASTLiteral>(constant_value_literal);
if (!options.add_cast_for_constants)
return constant_value_ast;
if (requiresCastCall())
{
auto constant_type_name_ast = std::make_shared<ASTLiteral>(constant_value->getType()->getName());
return makeASTFunction("_CAST", std::move(constant_value_ast), std::move(constant_type_name_ast));

12
src/Analyzer/ConstantNode.h
View File

@ -75,6 +75,17 @@ public:
return constant_value->getType();
}
/// Check if conversion to AST requires wrapping with _CAST function.
bool requiresCastCall() const;
/// Check if constant is a result of _CAST function constant folding.
bool receivedFromInitiatorServer() const;
void setMaskId(size_t id)
{
mask_id = id;
}
void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override;
protected:
@ -90,6 +101,7 @@ private:
ConstantValuePtr constant_value;
String value_string;
QueryTreeNodePtr source_expression;
size_t mask_id = 0;
static constexpr size_t children_size = 0;
};

372
src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h Normal file
View File

@ -0,0 +1,372 @@
#pragma once
#include <Parsers/FunctionSecretArgumentsFinder.h>
#include <Analyzer/ConstantNode.h>
#include <Analyzer/FunctionNode.h>
#include <Analyzer/IQueryTreeNode.h>
#include <Analyzer/IdentifierNode.h>
#include <Analyzer/ListNode.h>
#include <Common/KnownObjectNames.h>
#include <Core/QualifiedTableName.h>
#include <boost/algorithm/string/predicate.hpp>
namespace DB
{
/// Finds arguments of a specified function which should not be displayed for most users for security reasons.
/// That involves passwords and secret keys.
class FunctionSecretArgumentsFinderTreeNode
{
public:
explicit FunctionSecretArgumentsFinderTreeNode(const FunctionNode & function_) : function(function_), arguments(function.getArguments())
{
if (arguments.getNodes().empty())
return;
findFunctionSecretArguments();
}
struct Result
{
/// Result constructed by default means no arguments will be hidden.
size_t start = static_cast<size_t>(-1);
size_t count = 0; /// Mostly it's either 0 or 1. There are only a few cases where `count` can be greater than 1 (e.g. see `encrypt`).
/// In all known cases secret arguments are consecutive
bool are_named = false; /// Arguments like `password = 'password'` are considered as named arguments.
/// E.g. "headers" in `url('..', headers('foo' = '[HIDDEN]'))`
std::vector<std::string> nested_maps;
bool hasSecrets() const
{
return count != 0 || !nested_maps.empty();
}
};
FunctionSecretArgumentsFinder::Result getResult() const { return result; }
private:
const FunctionNode & function;
const ListNode & arguments;
FunctionSecretArgumentsFinder::Result result;
void markSecretArgument(size_t index, bool argument_is_named = false)
{
if (index >= arguments.getNodes().size())
return;
if (!result.count)
{
result.start = index;
result.are_named = argument_is_named;
}
chassert(index >= result.start); /// We always check arguments consecutively
result.count = index + 1 - result.start;
if (!argument_is_named)
result.are_named = false;
}
void findFunctionSecretArguments()
{
const auto & name = function.getFunctionName();
if ((name == "mysql") || (name == "postgresql") || (name == "mongodb"))
{
/// mysql('host:port', 'database', 'table', 'user', 'password', ...)
/// postgresql('host:port', 'database', 'table', 'user', 'password', ...)
/// mongodb('host:port', 'database', 'collection', 'user', 'password', ...)
findMySQLFunctionSecretArguments();
}
else if ((name == "s3") || (name == "cosn") || (name == "oss") ||
(name == "deltaLake") || (name == "hudi") || (name == "iceberg"))
{
/// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
findS3FunctionSecretArguments(/* is_cluster_function= */ false);
}
else if (name == "s3Cluster")
{
/// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', ...)
findS3FunctionSecretArguments(/* is_cluster_function= */ true);
}
else if ((name == "remote") || (name == "remoteSecure"))
{
/// remote('addresses_expr', 'db', 'table', 'user', 'password', ...)
findRemoteFunctionSecretArguments();
}
else if ((name == "encrypt") || (name == "decrypt") ||
(name == "aes_encrypt_mysql") || (name == "aes_decrypt_mysql") ||
(name == "tryDecrypt"))
{
/// encrypt('mode', 'plaintext', 'key' [, iv, aad])
findEncryptionFunctionSecretArguments();
}
else if (name == "url")
{
findURLSecretArguments();
}
}
void findMySQLFunctionSecretArguments()
{
if (isNamedCollectionName(0))
{
/// mysql(named_collection, ..., password = 'password', ...)
findSecretNamedArgument("password", 1);
}
else
{
/// mysql('host:port', 'database', 'table', 'user', 'password', ...)
markSecretArgument(4);
}
}
/// Returns the number of arguments excluding "headers" and "extra_credentials" (which should
/// always be at the end). Marks "headers" as secret, if found.
size_t excludeS3OrURLNestedMaps()
{
const auto & nodes = arguments.getNodes();
size_t count = nodes.size();
while (count > 0)
{
const FunctionNode * f = nodes.at(count - 1)->as<FunctionNode>();
if (!f)
break;
if (f->getFunctionName() == "headers")
result.nested_maps.push_back(f->getFunctionName());
else if (f->getFunctionName() != "extra_credentials")
break;
count -= 1;
}
return count;
}
void findS3FunctionSecretArguments(bool is_cluster_function)
{
/// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument.
size_t url_arg_idx = is_cluster_function ? 1 : 0;
if (!is_cluster_function && isNamedCollectionName(0))
{
/// s3(named_collection, ..., secret_access_key = 'secret_access_key', ...)
findSecretNamedArgument("secret_access_key", 1);
return;
}
/// We should check other arguments first because we don't need to do any replacement in case of
/// s3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)])
/// s3('url', 'format', 'structure' [, 'compression'] [, extra_credentials(..)] [, headers(..)])
size_t count = excludeS3OrURLNestedMaps();
if ((url_arg_idx + 3 <= count) && (count <= url_arg_idx + 4))
{
String second_arg;
if (tryGetStringFromArgument(url_arg_idx + 1, &second_arg))
{
if (boost::iequals(second_arg, "NOSIGN"))
return; /// The argument after 'url' is "NOSIGN".
if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg))
return; /// The argument after 'url' is a format: s3('url', 'format', ...)
}
}
/// We're going to replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures:
/// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
/// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression')
if (url_arg_idx + 2 < count)
markSecretArgument(url_arg_idx + 2);
}
void findURLSecretArguments()
{
if (!isNamedCollectionName(0))
excludeS3OrURLNestedMaps();
}
bool tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier = true) const
{
if (arg_idx >= arguments.getNodes().size())
return false;
return tryGetStringFromArgument(arguments.getNodes()[arg_idx], res, allow_identifier);
}
static bool tryGetStringFromArgument(const QueryTreeNodePtr argument, String * res, bool allow_identifier = true)
{
if (const auto * literal = argument->as<ConstantNode>())
{
if (literal->getValue().getType() != Field::Types::String)
return false;
if (res)
*res = literal->getValue().safeGet<String>();
return true;
}
if (allow_identifier)
{
if (const auto * id = argument->as<IdentifierNode>())
{
if (res)
*res = id->getIdentifier().getFullName();
return true;
}
}
return false;
}
void findRemoteFunctionSecretArguments()
{
if (isNamedCollectionName(0))
{
/// remote(named_collection, ..., password = 'password', ...)
findSecretNamedArgument("password", 1);
return;
}
/// We're going to replace 'password' with '[HIDDEN'] for the following signatures:
/// remote('addresses_expr', db.table, 'user' [, 'password'] [, sharding_key])
/// remote('addresses_expr', 'db', 'table', 'user' [, 'password'] [, sharding_key])
/// remote('addresses_expr', table_function(), 'user' [, 'password'] [, sharding_key])
/// But we should check the number of arguments first because we don't need to do any replacements in case of
/// remote('addresses_expr', db.table)
if (arguments.getNodes().size() < 3)
return;
size_t arg_num = 1;
/// Skip 1 or 2 arguments with table_function() or db.table or 'db', 'table'.
const auto * table_function = arguments.getNodes()[arg_num]->as<FunctionNode>();
if (table_function && KnownTableFunctionNames::instance().exists(table_function->getFunctionName()))
{
++arg_num;
}
else
{
std::optional<String> database;
std::optional<QualifiedTableName> qualified_table_name;
if (!tryGetDatabaseNameOrQualifiedTableName(arg_num, database, qualified_table_name))
{
/// We couldn't evaluate the argument so we don't know whether it is 'db.table' or just 'db'.
/// Hence we can't figure out whether we should skip one argument 'user' or two arguments 'table', 'user'
/// before the argument 'password'. So it's safer to wipe two arguments just in case.
/// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string
/// before wiping it (because the `password` argument is always a literal string).
if (tryGetStringFromArgument(arg_num + 2, nullptr, /* allow_identifier= */ false))
{
/// Wipe either `password` or `user`.
markSecretArgument(arg_num + 2);
}
if (tryGetStringFromArgument(arg_num + 3, nullptr, /* allow_identifier= */ false))
{
/// Wipe either `password` or `sharding_key`.
markSecretArgument(arg_num + 3);
}
return;
}
/// Skip the current argument (which is either a database name or a qualified table name).
++arg_num;
if (database)
{
/// Skip the 'table' argument if the previous argument was a database name.
++arg_num;
}
}
/// Skip username.
++arg_num;
/// Do our replacement:
/// remote('addresses_expr', db.table, 'user', 'password', ...) -> remote('addresses_expr', db.table, 'user', '[HIDDEN]', ...)
/// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string
/// before wiping it (because the `password` argument is always a literal string).
bool can_be_password = tryGetStringFromArgument(arg_num, nullptr, /* allow_identifier= */ false);
if (can_be_password)
markSecretArgument(arg_num);
}
/// Tries to get either a database name or a qualified table name from an argument.
/// Empty string is also allowed (it means the default database).
/// The function is used by findRemoteFunctionSecretArguments() to determine how many arguments to skip before a password.
bool tryGetDatabaseNameOrQualifiedTableName(
size_t arg_idx,
std::optional<String> & res_database,
std::optional<QualifiedTableName> & res_qualified_table_name) const
{
res_database.reset();
res_qualified_table_name.reset();
String str;
if (!tryGetStringFromArgument(arg_idx, &str, /* allow_identifier= */ true))
return false;
if (str.empty())
{
res_database = "";
return true;
}
auto qualified_table_name = QualifiedTableName::tryParseFromString(str);
if (!qualified_table_name)
return false;
if (qualified_table_name->database.empty())
res_database = std::move(qualified_table_name->table);
else
res_qualified_table_name = std::move(qualified_table_name);
return true;
}
void findEncryptionFunctionSecretArguments()
{
if (arguments.getNodes().empty())
return;
/// We replace all arguments after 'mode' with '[HIDDEN]':
/// encrypt('mode', 'plaintext', 'key' [, iv, aad]) -> encrypt('mode', '[HIDDEN]')
result.start = 1;
result.count = arguments.getNodes().size() - 1;
}
/// Whether a specified argument can be the name of a named collection?
bool isNamedCollectionName(size_t arg_idx) const
{
if (arguments.getNodes().size() <= arg_idx)
return false;
const auto * identifier = arguments.getNodes()[arg_idx]->as<IdentifierNode>();
return identifier != nullptr;
}
/// Looks for a secret argument with a specified name. This function looks for arguments in format `key=value` where the key is specified.
void findSecretNamedArgument(const std::string_view & key, size_t start = 0)
{
for (size_t i = start; i < arguments.getNodes().size(); ++i)
{
const auto & argument = arguments.getNodes()[i];
const auto * equals_func = argument->as<FunctionNode>();
if (!equals_func || (equals_func->getFunctionName() != "equals"))
continue;
const auto * expr_list = equals_func->getArguments().as<ListNode>();
if (!expr_list)
continue;
const auto & equal_args = expr_list->getNodes();
if (equal_args.size() != 2)
continue;
String found_key;
if (!tryGetStringFromArgument(equal_args[0], &found_key))
continue;
if (found_key == key)
markSecretArgument(i, /* argument_is_named= */ true);
}
}
};
}

88
src/Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.cpp
View File

@ -94,7 +94,8 @@ public:
if (!func_node || func_node->getArguments().getNodes().size() != 1)
return;
const auto * column_id = func_node->getArguments().getNodes()[0]->as<ColumnNode>();
const auto & argument_node = func_node->getArguments().getNodes()[0];
const auto * column_id = argument_node->as<ColumnNode>();
if (!column_id)
return;
@ -119,7 +120,7 @@ public:
if (!preimage_range)
return;
const auto new_node = generateOptimizedDateFilter(comparator, *column_id, *preimage_range);
const auto new_node = generateOptimizedDateFilter(comparator, argument_node, *preimage_range);
if (!new_node)
return;
@ -128,20 +129,22 @@ public:
}
private:
QueryTreeNodePtr
generateOptimizedDateFilter(const String & comparator, const ColumnNode & column_node, const std::pair<Field, Field> & range) const
QueryTreeNodePtr generateOptimizedDateFilter(
const String & comparator, const QueryTreeNodePtr & column_node, const std::pair<Field, Field> & range) const
{
const DateLUTImpl & date_lut = DateLUT::instance("UTC");
String start_date_or_date_time;
String end_date_or_date_time;
if (isDateOrDate32(column_node.getColumnType().get()))
const auto & column_node_typed = column_node->as<ColumnNode &>();
const auto & column_type = column_node_typed.getColumnType().get();
if (isDateOrDate32(column_type))
{
start_date_or_date_time = date_lut.dateToString(range.first.get<DateLUTImpl::Time>());
end_date_or_date_time = date_lut.dateToString(range.second.get<DateLUTImpl::Time>());
}
else if (isDateTime(column_node.getColumnType().get()) || isDateTime64(column_node.getColumnType().get()))
else if (isDateTime(column_type) || isDateTime64(column_type))
{
start_date_or_date_time = date_lut.timeToString(range.first.get<DateLUTImpl::Time>());
end_date_or_date_time = date_lut.timeToString(range.second.get<DateLUTImpl::Time>());
@ -151,69 +154,29 @@ private:
if (comparator == "equals")
{
const auto lhs = std::make_shared<FunctionNode>("greaterOrEquals");
lhs->getArguments().getNodes().push_back(std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
lhs->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(start_date_or_date_time));
resolveOrdinaryFunctionNode(*lhs, lhs->getFunctionName());
const auto rhs = std::make_shared<FunctionNode>("less");
rhs->getArguments().getNodes().push_back(std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
rhs->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(end_date_or_date_time));
resolveOrdinaryFunctionNode(*rhs, rhs->getFunctionName());
const auto new_date_filter = std::make_shared<FunctionNode>("and");
new_date_filter->getArguments().getNodes() = {lhs, rhs};
resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
return new_date_filter;
return createFunctionNode(
"and",
createFunctionNode("greaterOrEquals", column_node, std::make_shared<ConstantNode>(start_date_or_date_time)),
createFunctionNode("less", column_node, std::make_shared<ConstantNode>(end_date_or_date_time)));
}
else if (comparator == "notEquals")
{
const auto lhs = std::make_shared<FunctionNode>("less");
lhs->getArguments().getNodes().push_back(std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
lhs->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(start_date_or_date_time));
resolveOrdinaryFunctionNode(*lhs, lhs->getFunctionName());
const auto rhs = std::make_shared<FunctionNode>("greaterOrEquals");
rhs->getArguments().getNodes().push_back(std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
rhs->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(end_date_or_date_time));
resolveOrdinaryFunctionNode(*rhs, rhs->getFunctionName());
const auto new_date_filter = std::make_shared<FunctionNode>("or");
new_date_filter->getArguments().getNodes() = {lhs, rhs};
resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
return new_date_filter;
return createFunctionNode(
"or",
createFunctionNode("less", column_node, std::make_shared<ConstantNode>(start_date_or_date_time)),
createFunctionNode("greaterOrEquals", column_node, std::make_shared<ConstantNode>(end_date_or_date_time)));
}
else if (comparator == "greater")
{
const auto new_date_filter = std::make_shared<FunctionNode>("greaterOrEquals");
new_date_filter->getArguments().getNodes().push_back(
std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
new_date_filter->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(end_date_or_date_time));
resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
return new_date_filter;
return createFunctionNode("greaterOrEquals", column_node, std::make_shared<ConstantNode>(end_date_or_date_time));
}
else if (comparator == "lessOrEquals")
{
const auto new_date_filter = std::make_shared<FunctionNode>("less");
new_date_filter->getArguments().getNodes().push_back(
std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
new_date_filter->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(end_date_or_date_time));
resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
return new_date_filter;
return createFunctionNode("less", column_node, std::make_shared<ConstantNode>(end_date_or_date_time));
}
else if (comparator == "less" || comparator == "greaterOrEquals")
{
const auto new_date_filter = std::make_shared<FunctionNode>(comparator);
new_date_filter->getArguments().getNodes().push_back(
std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
new_date_filter->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(start_date_or_date_time));
resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
return new_date_filter;
return createFunctionNode(comparator, column_node, std::make_shared<ConstantNode>(start_date_or_date_time));
}
else [[unlikely]]
{
@ -224,10 +187,17 @@ private:
}
}
void resolveOrdinaryFunctionNode(FunctionNode & function_node, const String & function_name) const
template <typename... Args>
QueryTreeNodePtr createFunctionNode(const String & function_name, Args &&... args) const
{
auto function = FunctionFactory::instance().get(function_name, getContext());
function_node.resolveAsFunction(function->build(function_node.getArgumentColumns()));
const auto function_node = std::make_shared<FunctionNode>(function_name);
auto & new_arguments = function_node->getArguments().getNodes();
new_arguments.reserve(sizeof...(args));
(new_arguments.push_back(std::forward<Args>(args)), ...);
function_node->resolveAsFunction(function->build(function_node->getArgumentColumns()));
return function_node;
}
};

184
src/Analyzer/Passes/QueryAnalysisPass.cpp
View File

@ -3,6 +3,7 @@
#include <Common/checkStackSize.h>
#include <Common/NamePrompter.h>
#include <Common/ProfileEvents.h>
#include <Analyzer/FunctionSecretArgumentsFinderTreeNode.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteHelpers.h>
@ -706,7 +707,10 @@ struct IdentifierResolveScope
{
subquery_depth = parent_scope->subquery_depth;
context = parent_scope->context;
projection_mask_map = parent_scope->projection_mask_map;
}
else
projection_mask_map = std::make_shared<std::map<IQueryTreeNode::Hash, size_t>>();
if (auto * union_node = scope_node->as<UnionNode>())
{
@ -718,6 +722,11 @@ struct IdentifierResolveScope
group_by_use_nulls = context->getSettingsRef().group_by_use_nulls &&
(query_node->isGroupByWithGroupingSets() || query_node->isGroupByWithRollup() || query_node->isGroupByWithCube());
}
if (context)
join_use_nulls = context->getSettingsRef().join_use_nulls;
else if (parent_scope)
join_use_nulls = parent_scope->join_use_nulls;
}
QueryTreeNodePtr scope_node;
@ -772,6 +781,8 @@ struct IdentifierResolveScope
/// Apply nullability to aggregation keys
bool group_by_use_nulls = false;
/// Join retutns NULLs instead of default values
bool join_use_nulls = false;
/// JOINs count
size_t joins_count = 0;
@ -784,6 +795,9 @@ struct IdentifierResolveScope
*/
QueryTreeNodePtr expression_join_tree_node;
/// Node hash to mask id map
std::shared_ptr<std::map<IQueryTreeNode::Hash, size_t>> projection_mask_map;
[[maybe_unused]] const IdentifierResolveScope * getNearestQueryScope() const
{
const IdentifierResolveScope * scope_to_check = this;
@ -1068,6 +1082,8 @@ private:
class QueryAnalyzer
{
public:
explicit QueryAnalyzer(bool only_analyze_) : only_analyze(only_analyze_) {}
void resolve(QueryTreeNodePtr & node, const QueryTreeNodePtr & table_expression, ContextPtr context)
{
IdentifierResolveScope scope(node, nullptr /*parent_scope*/);
@ -1430,6 +1446,7 @@ private:
/// Global scalar subquery to scalar value map
std::unordered_map<QueryTreeNodePtrWithHash, Block> scalar_subquery_to_scalar_value;
const bool only_analyze;
};
/// Utility functions implementation
@ -1977,80 +1994,96 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden
auto interpreter = std::make_unique<InterpreterSelectQueryAnalyzer>(node->toAST(), subquery_context, subquery_context->getViewSource(), options);
auto io = interpreter->execute();
PullingAsyncPipelineExecutor executor(io.pipeline);
io.pipeline.setProgressCallback(context->getProgressCallback());
io.pipeline.setProcessListElement(context->getProcessListElement());
Block block;
while (block.rows() == 0 && executor.pull(block))
if (only_analyze)
{
}
if (block.rows() == 0)
{
auto types = interpreter->getSampleBlock().getDataTypes();
if (types.size() != 1)
types = {std::make_shared<DataTypeTuple>(types)};
auto & type = types[0];
if (!type->isNullable())
/// If query is only analyzed, then constants are not correct.
scalar_block = interpreter->getSampleBlock();
for (auto & column : scalar_block)
{
if (!type->canBeInsideNullable())
throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY,
"Scalar subquery returned empty result of type {} which cannot be Nullable",
type->getName());
type = makeNullable(type);
if (column.column->empty())
{
auto mut_col = column.column->cloneEmpty();
mut_col->insertDefault();
column.column = std::move(mut_col);
}
}
auto scalar_column = type->createColumn();
scalar_column->insert(Null());
scalar_block.insert({std::move(scalar_column), type, "null"});
}
else
{
if (block.rows() != 1)
throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, "Scalar subquery returned more than one row");
Block block;
Block tmp_block;
while (tmp_block.rows() == 0 && executor.pull(tmp_block))
while (block.rows() == 0 && executor.pull(block))
{
}
if (tmp_block.rows() != 0)
throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, "Scalar subquery returned more than one row");
block = materializeBlock(block);
size_t columns = block.columns();
if (columns == 1)
if (block.rows() == 0)
{
auto & column = block.getByPosition(0);
/// Here we wrap type to nullable if we can.
/// It is needed cause if subquery return no rows, it's result will be Null.
/// In case of many columns, do not check it cause tuple can't be nullable.
if (!column.type->isNullable() && column.type->canBeInsideNullable())
auto types = interpreter->getSampleBlock().getDataTypes();
if (types.size() != 1)
types = {std::make_shared<DataTypeTuple>(types)};
auto & type = types[0];
if (!type->isNullable())
{
column.type = makeNullable(column.type);
column.column = makeNullable(column.column);
if (!type->canBeInsideNullable())
throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY,
"Scalar subquery returned empty result of type {} which cannot be Nullable",
type->getName());
type = makeNullable(type);
}
scalar_block = block;
auto scalar_column = type->createColumn();
scalar_column->insert(Null());
scalar_block.insert({std::move(scalar_column), type, "null"});
}
else
{
/** Make unique column names for tuple.
*
* Example: SELECT (SELECT 2 AS x, x)
*/
makeUniqueColumnNamesInBlock(block);
if (block.rows() != 1)
throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, "Scalar subquery returned more than one row");
scalar_block.insert({
ColumnTuple::create(block.getColumns()),
std::make_shared<DataTypeTuple>(block.getDataTypes(), block.getNames()),
"tuple"});
Block tmp_block;
while (tmp_block.rows() == 0 && executor.pull(tmp_block))
{
}
if (tmp_block.rows() != 0)
throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, "Scalar subquery returned more than one row");
block = materializeBlock(block);
size_t columns = block.columns();
if (columns == 1)
{
auto & column = block.getByPosition(0);
/// Here we wrap type to nullable if we can.
/// It is needed cause if subquery return no rows, it's result will be Null.
/// In case of many columns, do not check it cause tuple can't be nullable.
if (!column.type->isNullable() && column.type->canBeInsideNullable())
{
column.type = makeNullable(column.type);
column.column = makeNullable(column.column);
}
scalar_block = block;
}
else
{
/** Make unique column names for tuple.
*
* Example: SELECT (SELECT 2 AS x, x)
*/
makeUniqueColumnNamesInBlock(block);
scalar_block.insert({
ColumnTuple::create(block.getColumns()),
std::make_shared<DataTypeTuple>(block.getDataTypes(), block.getNames()),
"tuple"});
}
}
}
@ -3286,7 +3319,6 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo
QueryTreeNodePtr resolved_identifier;
JoinKind join_kind = from_join_node.getKind();
bool join_use_nulls = scope.context->getSettingsRef().join_use_nulls;
/// If columns from left or right table were missed Object(Nullable('json')) subcolumns, they will be replaced
/// to ConstantNode(NULL), which can't be cast to ColumnNode, so we resolve it here.
@ -3451,7 +3483,7 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo
if (join_node_in_resolve_process || !resolved_identifier)
return resolved_identifier;
if (join_use_nulls)
if (scope.join_use_nulls)
{
resolved_identifier = resolved_identifier->clone();
convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side);
@ -4439,7 +4471,7 @@ ProjectionNames QueryAnalyzer::resolveMatcher(QueryTreeNodePtr & matcher_node, I
else
matched_expression_nodes_with_names = resolveUnqualifiedMatcher(matcher_node, scope);
if (scope.context->getSettingsRef().join_use_nulls)
if (scope.join_use_nulls)
{
/** If we are resolving matcher came from the result of JOIN and `join_use_nulls` is set,
* we need to convert joined column type to Nullable.
@ -5124,22 +5156,31 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
}
/// Resolve function arguments
bool allow_table_expressions = is_special_function_in;
auto arguments_projection_names = resolveExpressionNodeList(function_node_ptr->getArgumentsNode(),
scope,
true /*allow_lambda_expression*/,
allow_table_expressions /*allow_table_expression*/);
if (function_node_ptr->toAST()->hasSecretParts())
/// Mask arguments if needed
if (!scope.context->getSettingsRef().format_display_secrets_in_show_and_select)
{
for (auto & argument : arguments_projection_names)
if (FunctionSecretArgumentsFinder::Result secret_arguments = FunctionSecretArgumentsFinderTreeNode(*function_node_ptr).getResult(); secret_arguments.count)
{
SipHash hash;
hash.update(argument);
argument = getHexUIntLowercase(hash.get128());
auto & argument_nodes = function_node_ptr->getArgumentsNode()->as<ListNode &>().getNodes();
for (size_t n = secret_arguments.start; n < secret_arguments.start + secret_arguments.count; ++n)
{
if (auto * constant = argument_nodes[n]->as<ConstantNode>())
{
auto mask = scope.projection_mask_map->insert({constant->getTreeHash(), scope.projection_mask_map->size() + 1}).first->second;
constant->setMaskId(mask);
arguments_projection_names[n] = "[HIDDEN id: " + std::to_string(mask) + "]";
}
}
}
}
auto & function_node = *function_node_ptr;
/// Replace right IN function argument if it is table or table function with subquery that read ordinary columns
@ -7559,8 +7600,22 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier
}
if (query_node_typed.getPrewhere())
{
/** Expression in PREWHERE with JOIN should not be modified by join_use_nulls.
* Example: SELECT * FROM t1 JOIN t2 USING (id) PREWHERE a = 1
* Column `a` should be resolved from table and should not change its type to Nullable.
*/
bool join_use_nulls = scope.join_use_nulls;
bool use_identifier_lookup_to_result_cache = scope.use_identifier_lookup_to_result_cache;
scope.join_use_nulls = false;
scope.use_identifier_lookup_to_result_cache = false;
resolveExpressionNode(query_node_typed.getPrewhere(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
scope.join_use_nulls = join_use_nulls;
scope.use_identifier_lookup_to_result_cache = use_identifier_lookup_to_result_cache;
}
if (query_node_typed.getWhere())
resolveExpressionNode(query_node_typed.getWhere(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
@ -7750,13 +7805,16 @@ void QueryAnalyzer::resolveUnion(const QueryTreeNodePtr & union_node, Identifier
}
QueryAnalysisPass::QueryAnalysisPass(QueryTreeNodePtr table_expression_)
QueryAnalysisPass::QueryAnalysisPass(QueryTreeNodePtr table_expression_, bool only_analyze_)
: table_expression(std::move(table_expression_))
, only_analyze(only_analyze_)
{}
QueryAnalysisPass::QueryAnalysisPass(bool only_analyze_) : only_analyze(only_analyze_) {}
void QueryAnalysisPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
{
QueryAnalyzer analyzer;
QueryAnalyzer analyzer(only_analyze);
analyzer.resolve(query_tree_node, table_expression, context);
createUniqueTableAliases(query_tree_node, table_expression, context);
}

5
src/Analyzer/Passes/QueryAnalysisPass.h
View File

@ -71,13 +71,13 @@ public:
/** Construct query analysis pass for query or union analysis.
* Available columns are extracted from query node join tree.
*/
QueryAnalysisPass() = default;
explicit QueryAnalysisPass(bool only_analyze_ = false);
/** Construct query analysis pass for expression or list of expressions analysis.
* Available expression columns are extracted from table expression.
* Table expression node must have query, union, table, table function type.
*/
explicit QueryAnalysisPass(QueryTreeNodePtr table_expression_);
QueryAnalysisPass(QueryTreeNodePtr table_expression_, bool only_analyze_ = false);
String getName() override
{
@ -93,6 +93,7 @@ public:
private:
QueryTreeNodePtr table_expression;
const bool only_analyze;
};
}

4
src/Analyzer/QueryTreePassManager.cpp
View File

@ -246,9 +246,9 @@ void QueryTreePassManager::dump(WriteBuffer & buffer, size_t up_to_pass_index)
}
}
void addQueryTreePasses(QueryTreePassManager & manager)
void addQueryTreePasses(QueryTreePassManager & manager, bool only_analyze)
{
manager.addPass(std::make_unique<QueryAnalysisPass>());
manager.addPass(std::make_unique<QueryAnalysisPass>(only_analyze));
manager.addPass(std::make_unique<GroupingFunctionsResolvePass>());
manager.addPass(std::make_unique<RemoveUnusedProjectionColumnsPass>());

2
src/Analyzer/QueryTreePassManager.h
View File

@ -47,6 +47,6 @@ private:
std::vector<QueryTreePassPtr> passes;
};
void addQueryTreePasses(QueryTreePassManager & manager);
void addQueryTreePasses(QueryTreePassManager & manager, bool only_analyze = false);
}

16
src/Backups/BackupsWorker.cpp
View File

@ -18,6 +18,7 @@
#include <Interpreters/executeDDLQueryOnCluster.h>
#include <Parsers/ASTBackupQuery.h>
#include <Parsers/ASTFunction.h>
#include <Common/CurrentThread.h>
#include <Common/Exception.h>
#include <Common/Macros.h>
#include <Common/logger_useful.h>
@ -486,7 +487,7 @@ OperationID BackupsWorker::startMakingBackup(const ASTPtr & query, const Context
/// process_list_element_holder is used to make an element in ProcessList live while BACKUP is working asynchronously.
auto process_list_element = context_in_use->getProcessListElement();
scheduleFromThreadPool<void>(
thread_pool.scheduleOrThrowOnError(
[this,
backup_query,
backup_id,
@ -502,6 +503,8 @@ OperationID BackupsWorker::startMakingBackup(const ASTPtr & query, const Context
BackupMutablePtr backup_async;
try
{
setThreadName("BackupWorker");
CurrentThread::QueryScope query_scope(context_in_use);
doBackup(
backup_async,
backup_query,
@ -517,8 +520,7 @@ OperationID BackupsWorker::startMakingBackup(const ASTPtr & query, const Context
{
on_exception(backup_async, backup_id, backup_name_for_logging, backup_settings, backup_coordination);
}
},
thread_pool, "BackupWorker");
});
}
else
{
@ -864,7 +866,7 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt
/// process_list_element_holder is used to make an element in ProcessList live while RESTORE is working asynchronously.
auto process_list_element = context_in_use->getProcessListElement();
scheduleFromThreadPool<void>(
thread_pool.scheduleOrThrowOnError(
[this,
restore_query,
restore_id,
@ -878,6 +880,8 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt
{
try
{
setThreadName("RestorerWorker");
CurrentThread::QueryScope query_scope(context_in_use);
doRestore(
restore_query,
restore_id,
@ -891,9 +895,7 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt
{
on_exception(restore_id, backup_name_for_logging, restore_settings, restore_coordination);
}
},
thread_pool,
"RestoreWorker");
});
}
else
{

2
src/Client/ConnectionParameters.cpp
View File

@ -115,7 +115,7 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati
/// At the same time, I want clickhouse-local to always work, regardless.
/// TODO: get rid of glibc, or replace getaddrinfo to c-ares.
compression = config.getBool("compression", host != "localhost" && !isLocalAddress(DNSResolver::instance().resolveHost(host)))
compression = config.getBool("compression", host != "localhost" && !isLocalAddress(DNSResolver::instance().resolveHostAllInOriginOrder(host).front()))
? Protocol::Compression::Enable : Protocol::Compression::Disable;
timeouts = ConnectionTimeouts()

25
src/Columns/ColumnUnique.cpp Normal file
View File

@ -0,0 +1,25 @@
#include <Columns/ColumnUnique.h>
namespace DB
{
/// Explicit template instantiations.
template class ColumnUnique<ColumnInt8>;
template class ColumnUnique<ColumnUInt8>;
template class ColumnUnique<ColumnInt16>;
template class ColumnUnique<ColumnUInt16>;
template class ColumnUnique<ColumnInt32>;
template class ColumnUnique<ColumnUInt32>;
template class ColumnUnique<ColumnInt64>;
template class ColumnUnique<ColumnUInt64>;
template class ColumnUnique<ColumnInt128>;
template class ColumnUnique<ColumnUInt128>;
template class ColumnUnique<ColumnInt256>;
template class ColumnUnique<ColumnUInt256>;
template class ColumnUnique<ColumnFloat32>;
template class ColumnUnique<ColumnFloat64>;
template class ColumnUnique<ColumnString>;
template class ColumnUnique<ColumnFixedString>;
template class ColumnUnique<ColumnDateTime64>;
}

21
src/Columns/ColumnUnique.h
View File

@ -15,6 +15,8 @@
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>
#include <Common/FieldVisitors.h>
#include "Columns/ColumnsDateTime.h"
#include "Columns/ColumnsNumber.h"
#include <base/range.h>
#include <base/unaligned.h>
@ -736,4 +738,23 @@ UInt128 ColumnUnique<ColumnType>::IncrementalHash::getHash(const ColumnType & co
return cur_hash;
}
extern template class ColumnUnique<ColumnInt8>;
extern template class ColumnUnique<ColumnUInt8>;
extern template class ColumnUnique<ColumnInt16>;
extern template class ColumnUnique<ColumnUInt16>;
extern template class ColumnUnique<ColumnInt32>;
extern template class ColumnUnique<ColumnUInt32>;
extern template class ColumnUnique<ColumnInt64>;
extern template class ColumnUnique<ColumnUInt64>;
extern template class ColumnUnique<ColumnInt128>;
extern template class ColumnUnique<ColumnUInt128>;
extern template class ColumnUnique<ColumnInt256>;
extern template class ColumnUnique<ColumnUInt256>;
extern template class ColumnUnique<ColumnFloat32>;
extern template class ColumnUnique<ColumnFloat64>;
extern template class ColumnUnique<ColumnString>;
extern template class ColumnUnique<ColumnFixedString>;
extern template class ColumnUnique<ColumnDateTime64>;
}

11
src/Common/DNSResolver.cpp
View File

@ -202,10 +202,10 @@ DNSResolver::DNSResolver() : impl(std::make_unique<DNSResolver::Impl>()), log(ge
Poco::Net::IPAddress DNSResolver::resolveHost(const std::string & host)
{
return pickAddress(resolveHostAll(host));
return pickAddress(resolveHostAll(host)); // random order -> random pick
}
DNSResolver::IPAddresses DNSResolver::resolveHostAll(const std::string & host)
DNSResolver::IPAddresses DNSResolver::resolveHostAllInOriginOrder(const std::string & host)
{
if (impl->disable_cache)
return resolveIPAddressImpl(host);
@ -214,6 +214,13 @@ DNSResolver::IPAddresses DNSResolver::resolveHostAll(const std::string & host)
return resolveIPAddressWithCache(impl->cache_host, host);
}
DNSResolver::IPAddresses DNSResolver::resolveHostAll(const std::string & host)
{
auto addresses = resolveHostAllInOriginOrder(host);
std::shuffle(addresses.begin(), addresses.end(), thread_local_rng);
return addresses;
}
Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host_and_port)
{
if (impl->disable_cache)

3
src/Common/DNSResolver.h
View File

@ -34,6 +34,9 @@ public:
Poco::Net::IPAddress resolveHost(const std::string & host);
/// Accepts host names like 'example.com' or '127.0.0.1' or '::1' and resolves all its IPs
/// resolveHostAllInOriginOrder returns addresses with the same order as system call returns it
IPAddresses resolveHostAllInOriginOrder(const std::string & host);
/// resolveHostAll returns addresses in random order
IPAddresses resolveHostAll(const std::string & host);
/// Accepts host names like 'example.com:port' or '127.0.0.1:port' or '[::1]:port' and resolves its IP and port

40
src/Common/DateLUTImpl.h
View File

@ -3,13 +3,13 @@
#include <base/DayNum.h>
#include <base/defines.h>
#include <base/types.h>
#include <Core/DecimalFunctions.h>
#include <ctime>
#include <cassert>
#include <string>
#include <type_traits>
#define DATE_SECONDS_PER_DAY 86400 /// Number of seconds in a day, 60 * 60 * 24
#define DATE_LUT_MIN_YEAR 1900 /// 1900 since majority of financial organizations consider 1900 as an initial year.
@ -280,9 +280,9 @@ private:
static_assert(std::is_integral_v<DateOrTime> && std::is_integral_v<Divisor>);
assert(divisor > 0);
if (likely(offset_is_whole_number_of_hours_during_epoch))
if (offset_is_whole_number_of_hours_during_epoch) [[likely]]
{
if (likely(x >= 0))
if (x >= 0) [[likely]]
return static_cast<DateOrTime>(x / divisor * divisor);
/// Integer division for negative numbers rounds them towards zero (up).
@ -576,10 +576,10 @@ public:
unsigned toSecond(Time t) const
{
if (likely(offset_is_whole_number_of_minutes_during_epoch))
if (offset_is_whole_number_of_minutes_during_epoch) [[likely]]
{
Time res = t % 60;
if (likely(res >= 0))
if (res >= 0) [[likely]]
return static_cast<unsigned>(res);
return static_cast<unsigned>(res) + 60;
}
@ -593,6 +593,30 @@ public:
return time % 60;
}
template <typename DateOrTime>
unsigned toMillisecond(const DateOrTime & datetime, Int64 scale_multiplier) const
{
constexpr Int64 millisecond_multiplier = 1'000;
constexpr Int64 microsecond_multiplier = 1'000 * millisecond_multiplier;
constexpr Int64 divider = microsecond_multiplier / millisecond_multiplier;
auto components = DB::DecimalUtils::splitWithScaleMultiplier(datetime, scale_multiplier);
if (datetime.value < 0 && components.fractional)
{
components.fractional = scale_multiplier + (components.whole ? Int64(-1) : Int64(1)) * components.fractional;
--components.whole;
}
Int64 fractional = components.fractional;
if (scale_multiplier > microsecond_multiplier)
fractional = fractional / (scale_multiplier / microsecond_multiplier);
else if (scale_multiplier < microsecond_multiplier)
fractional = fractional * (microsecond_multiplier / scale_multiplier);
UInt16 millisecond = static_cast<UInt16>(fractional / divider);
return millisecond;
}
unsigned toMinute(Time t) const
{
if (t >= 0 && offset_is_whole_number_of_hours_during_epoch)
@ -1122,9 +1146,9 @@ public:
DateOrTime toStartOfMinuteInterval(DateOrTime t, UInt64 minutes) const
{
Int64 divisor = 60 * minutes;
if (likely(offset_is_whole_number_of_minutes_during_epoch))
if (offset_is_whole_number_of_minutes_during_epoch) [[likely]]
{
if (likely(t >= 0))
if (t >= 0) [[likely]]
return static_cast<DateOrTime>(t / divisor * divisor);
return static_cast<DateOrTime>((t + 1 - divisor) / divisor * divisor);
}
@ -1339,7 +1363,7 @@ public:
UInt8 saturateDayOfMonth(Int16 year, UInt8 month, UInt8 day_of_month) const
{
if (likely(day_of_month <= 28))
if (day_of_month <= 28) [[likely]]
return day_of_month;
UInt8 days_in_month = daysInMonth(year, month);

23
src/Common/FieldVisitorConvertToNumber.cpp Normal file
View File

@ -0,0 +1,23 @@
#include <Common/FieldVisitorConvertToNumber.h>
#include "base/Decimal.h"
namespace DB
{
/// Explicit template instantiations.
template class FieldVisitorConvertToNumber<Int8>;
template class FieldVisitorConvertToNumber<UInt8>;
template class FieldVisitorConvertToNumber<Int16>;
template class FieldVisitorConvertToNumber<UInt16>;
template class FieldVisitorConvertToNumber<Int32>;
template class FieldVisitorConvertToNumber<UInt32>;
template class FieldVisitorConvertToNumber<Int64>;
template class FieldVisitorConvertToNumber<UInt64>;
template class FieldVisitorConvertToNumber<Int128>;
template class FieldVisitorConvertToNumber<UInt128>;
template class FieldVisitorConvertToNumber<Int256>;
template class FieldVisitorConvertToNumber<UInt256>;
template class FieldVisitorConvertToNumber<Float32>;
template class FieldVisitorConvertToNumber<Float64>;
}

15
src/Common/FieldVisitorConvertToNumber.h
View File

@ -117,4 +117,19 @@ public:
T operator() (const bool & x) const { return T(x); }
};
extern template class FieldVisitorConvertToNumber<Int8>;
extern template class FieldVisitorConvertToNumber<UInt8>;
extern template class FieldVisitorConvertToNumber<Int16>;
extern template class FieldVisitorConvertToNumber<UInt16>;
extern template class FieldVisitorConvertToNumber<Int32>;
extern template class FieldVisitorConvertToNumber<UInt32>;
extern template class FieldVisitorConvertToNumber<Int64>;
extern template class FieldVisitorConvertToNumber<UInt64>;
extern template class FieldVisitorConvertToNumber<Int128>;
extern template class FieldVisitorConvertToNumber<UInt128>;
extern template class FieldVisitorConvertToNumber<Int256>;
extern template class FieldVisitorConvertToNumber<UInt256>;
extern template class FieldVisitorConvertToNumber<Float32>;
extern template class FieldVisitorConvertToNumber<Float64>;
}

1
src/Common/ProfileEvents.cpp
View File

@ -475,6 +475,7 @@ The server successfully detected this situation and will download merged part fr
M(FileSegmentUseMicroseconds, "File segment use() time") \
M(FileSegmentRemoveMicroseconds, "File segment remove() time") \
M(FileSegmentHolderCompleteMicroseconds, "File segments holder complete() time") \
M(FileSegmentFailToIncreasePriority, "Number of times the priority was not increased due to a high contention on the cache lock") \
M(FilesystemCacheHoldFileSegments, "Filesystem cache file segments count, which were hold") \
M(FilesystemCacheUnusedHoldFileSegments, "Filesystem cache file segments count, which were hold, but not used (because of seek or LIMIT n, etc)") \
\

2
src/Coordination/KeeperStateManager.cpp
View File

@ -30,7 +30,7 @@ bool isLocalhost(const std::string & hostname)
{
try
{
return isLocalAddress(DNSResolver::instance().resolveHost(hostname));
return isLocalAddress(DNSResolver::instance().resolveHostAllInOriginOrder(hostname).front());
}
catch (...)
{

4
src/Core/Settings.h
View File

@ -186,6 +186,7 @@ class IColumn;
\
M(String, cluster_for_parallel_replicas, "", "Cluster for a shard in which current server is located", 0) \
M(UInt64, allow_experimental_parallel_reading_from_replicas, 0, "Use all the replicas from a shard for SELECT query execution. Reading is parallelized and coordinated dynamically. 0 - disabled, 1 - enabled, silently disable them in case of failure, 2 - enabled, throw an exception in case of failure", 0) \
M(Bool, parallel_replicas_allow_in_with_subquery, true, "If true, subquery for IN will be executed on every follower replica.", 0) \
M(Float, parallel_replicas_single_task_marks_count_multiplier, 2, "A multiplier which will be added during calculation for minimal number of marks to retrieve from coordinator. This will be applied only for remote replicas.", 0) \
M(Bool, parallel_replicas_for_non_replicated_merge_tree, false, "If true, ClickHouse will use parallel replicas algorithm also for non-replicated MergeTree tables", 0) \
M(UInt64, parallel_replicas_min_number_of_rows_per_replica, 0, "Limit the number of replicas used in a query to (estimated rows to read / min_number_of_rows_per_replica). The max is still limited by 'max_parallel_replicas'", 0) \
@ -597,6 +598,7 @@ class IColumn;
M(Bool, normalize_function_names, true, "Normalize function names to their canonical names", 0) \
M(Bool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there are constants there", 0) \
M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views if the block is not a duplicate for the table. Use true to always deduplicate in dependent tables.", 0) \
M(Bool, throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert, true, "Throw exception on INSERT query when the setting `deduplicate_blocks_in_dependent_materialized_views` is enabled along with `async_insert`. It guarantees correctness, because these features can't work together.", 0) \
M(Bool, update_insert_deduplication_token_in_dependent_materialized_views, false, "Should update insert deduplication token with table identifier during insert in dependent materialized views.", 0) \
M(Bool, materialized_views_ignore_errors, false, "Allows to ignore errors for MATERIALIZED VIEW, and deliver original block to the table regardless of MVs", 0) \
M(Bool, ignore_materialized_views_with_dropped_target_table, false, "Ignore MVs with dropped target table during pushing to views", 0) \
@ -697,7 +699,7 @@ class IColumn;
M(Bool, database_replicated_allow_replicated_engine_arguments, true, "Allow to create only Replicated tables in database with engine Replicated with explicit arguments", 0) \
M(Bool, cloud_mode, false, "Only available in ClickHouse Cloud", 0) \
M(UInt64, cloud_mode_engine, 1, "Only available in ClickHouse Cloud", 0) \
M(DistributedDDLOutputMode, distributed_ddl_output_mode, DistributedDDLOutputMode::THROW, "Format of distributed DDL query result, one of: 'none', 'throw', 'null_status_on_timeout', 'never_throw'", 0) \
M(DistributedDDLOutputMode, distributed_ddl_output_mode, DistributedDDLOutputMode::THROW, "Format of distributed DDL query result, one of: 'none', 'throw', 'null_status_on_timeout', 'never_throw', 'none_only_active', 'throw_only_active', 'null_status_on_timeout_only_active'", 0) \
M(UInt64, distributed_ddl_entry_format_version, 5, "Compatibility version of distributed DDL (ON CLUSTER) queries", 0) \
\
M(UInt64, external_storage_max_read_rows, 0, "Limit maximum number of rows when table with external engine should flush history data. Now supported only for MySQL table engine, database engine, dictionary and MaterializedMySQL. If equal to 0, this setting is disabled", 0) \

2
src/Core/SettingsChangesHistory.h
View File

@ -90,6 +90,8 @@ static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> sett
{"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"},
{"page_cache_inject_eviction", false, false, "Added userspace page cache"},
{"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"},
{"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication is dependent materialized view cannot work together with async inserts."},
{"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
}},
{"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
{"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},

1
src/Core/SettingsEnums.cpp
View File

@ -118,6 +118,7 @@ IMPLEMENT_SETTING_ENUM(DistributedDDLOutputMode, ErrorCodes::BAD_ARGUMENTS,
{"null_status_on_timeout", DistributedDDLOutputMode::NULL_STATUS_ON_TIMEOUT},
{"throw_only_active", DistributedDDLOutputMode::THROW_ONLY_ACTIVE},
{"null_status_on_timeout_only_active", DistributedDDLOutputMode::NULL_STATUS_ON_TIMEOUT_ONLY_ACTIVE},
{"none_only_active", DistributedDDLOutputMode::NONE_ONLY_ACTIVE},
{"never_throw", DistributedDDLOutputMode::NEVER_THROW}})
IMPLEMENT_SETTING_ENUM(StreamingHandleErrorMode, ErrorCodes::BAD_ARGUMENTS,

1
src/Core/SettingsEnums.h
View File

@ -177,6 +177,7 @@ enum class DistributedDDLOutputMode
NEVER_THROW,
THROW_ONLY_ACTIVE,
NULL_STATUS_ON_TIMEOUT_ONLY_ACTIVE,
NONE_ONLY_ACTIVE,
};
DECLARE_SETTING_ENUM(DistributedDDLOutputMode)

6
src/DataTypes/DataTypeDecimalBase.h
View File

@ -207,4 +207,10 @@ inline DataTypePtr createDecimal(UInt64 precision_value, UInt64 scale_value)
return std::make_shared<DecimalType<Decimal256>>(precision_value, scale_value);
}
extern template class DataTypeDecimalBase<Decimal32>;
extern template class DataTypeDecimalBase<Decimal64>;
extern template class DataTypeDecimalBase<Decimal128>;
extern template class DataTypeDecimalBase<Decimal256>;
extern template class DataTypeDecimalBase<DateTime64>;
}

256
src/DataTypes/DataTypesDecimal.cpp
View File

@ -112,6 +112,256 @@ static DataTypePtr createExact(const ASTPtr & arguments)
return createDecimal<DataTypeDecimal>(precision, scale);
}
template <typename FromDataType, typename ToDataType, typename ReturnType>
requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
ReturnType convertDecimalsImpl(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result)
{
using FromFieldType = typename FromDataType::FieldType;
using ToFieldType = typename ToDataType::FieldType;
using MaxFieldType = std::conditional_t<(sizeof(FromFieldType) > sizeof(ToFieldType)), FromFieldType, ToFieldType>;
using MaxNativeType = typename MaxFieldType::NativeType;
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
MaxNativeType converted_value;
if (scale_to > scale_from)
{
converted_value = DecimalUtils::scaleMultiplier<MaxNativeType>(scale_to - scale_from);
if (common::mulOverflow(static_cast<MaxNativeType>(value.value), converted_value, converted_value))
{
if constexpr (throw_exception)
throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow while multiplying {} by scale {}",
std::string(ToDataType::family_name), toString(value.value), toString(converted_value));
else
return ReturnType(false);
}
}
else if (scale_to == scale_from)
{
converted_value = value.value;
}
else
{
converted_value = value.value / DecimalUtils::scaleMultiplier<MaxNativeType>(scale_from - scale_to);
}
if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType))
{
if (converted_value < std::numeric_limits<typename ToFieldType::NativeType>::min() ||
converted_value > std::numeric_limits<typename ToFieldType::NativeType>::max())
{
if constexpr (throw_exception)
throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow: {} is not in range ({}, {})",
std::string(ToDataType::family_name), toString(converted_value),
toString(std::numeric_limits<typename ToFieldType::NativeType>::min()),
toString(std::numeric_limits<typename ToFieldType::NativeType>::max()));
else
return ReturnType(false);
}
}
result = static_cast<typename ToFieldType::NativeType>(converted_value);
return ReturnType(true);
}
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
template void convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, void>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result); \
template bool convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, bool>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result);
#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
FOR_EACH_DECIMAL_TYPE(INVOKE);
#undef DISPATCH
template <typename FromDataType, typename ToDataType>
requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
typename ToDataType::FieldType convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to)
{
using ToFieldType = typename ToDataType::FieldType;
ToFieldType result;
convertDecimalsImpl<FromDataType, ToDataType, void>(value, scale_from, scale_to, result);
return result;
}
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
template typename TO_DATA_TYPE::FieldType convertDecimals<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to);
#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
FOR_EACH_DECIMAL_TYPE(INVOKE);
#undef DISPATCH
template <typename FromDataType, typename ToDataType>
requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
bool tryConvertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result)
{
return convertDecimalsImpl<FromDataType, ToDataType, bool>(value, scale_from, scale_to, result);
}
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
template bool tryConvertDecimals<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result);
#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
FOR_EACH_DECIMAL_TYPE(INVOKE);
#undef DISPATCH
template <typename FromDataType, typename ToDataType, typename ReturnType>
requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
ReturnType convertFromDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType & result)
{
using FromFieldType = typename FromDataType::FieldType;
using ToFieldType = typename ToDataType::FieldType;
return DecimalUtils::convertToImpl<ToFieldType, FromFieldType, ReturnType>(value, scale, result);
}
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
template void convertFromDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result); \
template bool convertFromDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);
#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
FOR_EACH_ARITHMETIC_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
template <typename FromDataType, typename ToDataType>
requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
inline typename ToDataType::FieldType convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
{
typename ToDataType::FieldType result;
convertFromDecimalImpl<FromDataType, ToDataType, void>(value, scale, result);
return result;
}
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
template typename TO_DATA_TYPE::FieldType convertFromDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale);
#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
FOR_EACH_ARITHMETIC_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
template <typename FromDataType, typename ToDataType>
requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
inline bool tryConvertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
{
return convertFromDecimalImpl<FromDataType, ToDataType, bool>(value, scale, result);
}
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
template bool tryConvertFromDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType& result);
#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
FOR_EACH_ARITHMETIC_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
template <typename FromDataType, typename ToDataType, typename ReturnType>
requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
ReturnType convertToDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType & result)
{
using FromFieldType = typename FromDataType::FieldType;
using ToFieldType = typename ToDataType::FieldType;
using ToNativeType = typename ToFieldType::NativeType;
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
if constexpr (std::is_floating_point_v<FromFieldType>)
{
if (!std::isfinite(value))
{
if constexpr (throw_exception)
throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow. Cannot convert infinity or NaN to decimal", ToDataType::family_name);
else
return ReturnType(false);
}
auto out = value * static_cast<FromFieldType>(DecimalUtils::scaleMultiplier<ToNativeType>(scale));
if (out <= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::min()) ||
out >= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::max()))
{
if constexpr (throw_exception)
throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow. Float is out of Decimal range", ToDataType::family_name);
else
return ReturnType(false);
}
result = static_cast<ToNativeType>(out);
return ReturnType(true);
}
else
{
if constexpr (is_big_int_v<FromFieldType>)
return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal256>, ToDataType, ReturnType>(static_cast<Int256>(value), 0, scale, result));
else if constexpr (std::is_same_v<FromFieldType, UInt64>)
return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal128>, ToDataType, ReturnType>(static_cast<Int128>(value), 0, scale, result));
else
return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal64>, ToDataType, ReturnType>(static_cast<Int64>(value), 0, scale, result));
}
}
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
template void convertToDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result); \
template bool convertToDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);
#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
FOR_EACH_DECIMAL_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
template <typename FromDataType, typename ToDataType>
requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
inline typename ToDataType::FieldType convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
{
typename ToDataType::FieldType result;
convertToDecimalImpl<FromDataType, ToDataType, void>(value, scale, result);
return result;
}
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
template typename TO_DATA_TYPE::FieldType convertToDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale);
#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
FOR_EACH_DECIMAL_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
template <typename FromDataType, typename ToDataType>
requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
inline bool tryConvertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
{
return convertToDecimalImpl<FromDataType, ToDataType, bool>(value, scale, result);
}
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
template bool tryConvertToDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType& result);
#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
FOR_EACH_DECIMAL_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
template <typename T>
DataTypePtr createDecimalMaxPrecision(UInt64 scale)
{
return std::make_shared<DataTypeDecimal<T>>(DecimalUtils::max_precision<T>, scale);
}
template DataTypePtr createDecimalMaxPrecision<Decimal32>(UInt64 scale);
template DataTypePtr createDecimalMaxPrecision<Decimal64>(UInt64 scale);
template DataTypePtr createDecimalMaxPrecision<Decimal128>(UInt64 scale);
template DataTypePtr createDecimalMaxPrecision<Decimal256>(UInt64 scale);
/// Explicit template instantiations.
template class DataTypeDecimal<Decimal32>;
template class DataTypeDecimal<Decimal64>;
template class DataTypeDecimal<Decimal128>;
template class DataTypeDecimal<Decimal256>;
void registerDataTypeDecimal(DataTypeFactory & factory)
{
factory.registerDataType("Decimal32", createExact<Decimal32>, DataTypeFactory::CaseInsensitive);
@ -125,10 +375,4 @@ void registerDataTypeDecimal(DataTypeFactory & factory)
factory.registerAlias("FIXED", "Decimal", DataTypeFactory::CaseInsensitive);
}
/// Explicit template instantiations.
template class DataTypeDecimal<Decimal32>;
template class DataTypeDecimal<Decimal64>;
template class DataTypeDecimal<Decimal128>;
template class DataTypeDecimal<Decimal256>;
}

237
src/DataTypes/DataTypesDecimal.h
View File

@ -3,7 +3,11 @@
#include <base/arithmeticOverflow.h>
#include <base/extended_types.h>
#include <Common/typeid_cast.h>
#include <base/Decimal.h>
#include <base/Decimal_fwd.h>
#include <DataTypes/IDataType.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDate32.h>
#include <DataTypes/DataTypeDecimalBase.h>
#include <DataTypes/DataTypeDateTime64.h>
@ -13,7 +17,6 @@ namespace DB
namespace ErrorCodes
{
extern const int DECIMAL_OVERFLOW;
extern const int LOGICAL_ERROR;
}
@ -99,171 +102,145 @@ inline UInt32 getDecimalScale(const DataTypeDecimal<T> & data_type)
return data_type.getScale();
}
#define FOR_EACH_DECIMAL_TYPE(M) \
M(DataTypeDecimal<DateTime64>) \
M(DataTypeDateTime64) \
M(DataTypeDecimal32) \
M(DataTypeDecimal64) \
M(DataTypeDecimal128) \
M(DataTypeDecimal256)
#define FOR_EACH_DECIMAL_TYPE_PASS(M, X) \
M(DataTypeDecimal<DateTime64>, X) \
M(DataTypeDateTime64, X) \
M(DataTypeDecimal32, X) \
M(DataTypeDecimal64, X) \
M(DataTypeDecimal128, X) \
M(DataTypeDecimal256, X)
template <typename FromDataType, typename ToDataType, typename ReturnType = void>
requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
inline ReturnType convertDecimalsImpl(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result)
{
using FromFieldType = typename FromDataType::FieldType;
using ToFieldType = typename ToDataType::FieldType;
using MaxFieldType = std::conditional_t<(sizeof(FromFieldType) > sizeof(ToFieldType)), FromFieldType, ToFieldType>;
using MaxNativeType = typename MaxFieldType::NativeType;
ReturnType convertDecimalsImpl(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result);
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
extern template void convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, void>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result); \
extern template bool convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, bool>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result);
#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
FOR_EACH_DECIMAL_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
MaxNativeType converted_value;
if (scale_to > scale_from)
{
converted_value = DecimalUtils::scaleMultiplier<MaxNativeType>(scale_to - scale_from);
if (common::mulOverflow(static_cast<MaxNativeType>(value.value), converted_value, converted_value))
{
if constexpr (throw_exception)
throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow while multiplying {} by scale {}",
std::string(ToDataType::family_name), toString(value.value), toString(converted_value));
else
return ReturnType(false);
}
}
else if (scale_to == scale_from)
{
converted_value = value.value;
}
else
{
converted_value = value.value / DecimalUtils::scaleMultiplier<MaxNativeType>(scale_from - scale_to);
}
if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType))
{
if (converted_value < std::numeric_limits<typename ToFieldType::NativeType>::min() ||
converted_value > std::numeric_limits<typename ToFieldType::NativeType>::max())
{
if constexpr (throw_exception)
throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow: {} is not in range ({}, {})",
std::string(ToDataType::family_name), toString(converted_value),
toString(std::numeric_limits<typename ToFieldType::NativeType>::min()),
toString(std::numeric_limits<typename ToFieldType::NativeType>::max()));
else
return ReturnType(false);
}
}
result = static_cast<typename ToFieldType::NativeType>(converted_value);
return ReturnType(true);
}
template <typename FromDataType, typename ToDataType>
requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
inline typename ToDataType::FieldType convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to)
{
using ToFieldType = typename ToDataType::FieldType;
ToFieldType result;
typename ToDataType::FieldType convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to);
convertDecimalsImpl<FromDataType, ToDataType, void>(value, scale_from, scale_to, result);
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
extern template typename TO_DATA_TYPE::FieldType convertDecimals<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to);
#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
FOR_EACH_DECIMAL_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
return result;
}
template <typename FromDataType, typename ToDataType>
requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
inline bool tryConvertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result)
{
return convertDecimalsImpl<FromDataType, ToDataType, bool>(value, scale_from, scale_to, result);
}
bool tryConvertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result);
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
extern template bool tryConvertDecimals<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result);
#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
FOR_EACH_DECIMAL_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
template <typename FromDataType, typename ToDataType, typename ReturnType>
requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
inline ReturnType convertFromDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
{
using FromFieldType = typename FromDataType::FieldType;
using ToFieldType = typename ToDataType::FieldType;
ReturnType convertFromDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType & result);
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
extern template void convertFromDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result); \
extern template bool convertFromDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);
#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
FOR_EACH_ARITHMETIC_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
return DecimalUtils::convertToImpl<ToFieldType, FromFieldType, ReturnType>(value, scale, result);
}
template <typename FromDataType, typename ToDataType>
requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
inline typename ToDataType::FieldType convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
{
typename ToDataType::FieldType result;
typename ToDataType::FieldType convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale);
convertFromDecimalImpl<FromDataType, ToDataType, void>(value, scale, result);
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
extern template typename TO_DATA_TYPE::FieldType convertFromDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale);
#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
FOR_EACH_ARITHMETIC_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
return result;
}
template <typename FromDataType, typename ToDataType>
requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
inline bool tryConvertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
{
return convertFromDecimalImpl<FromDataType, ToDataType, bool>(value, scale, result);
}
bool tryConvertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result);
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
extern template bool tryConvertFromDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType& result);
#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
FOR_EACH_ARITHMETIC_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
template <typename FromDataType, typename ToDataType, typename ReturnType>
requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
inline ReturnType convertToDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
{
using FromFieldType = typename FromDataType::FieldType;
using ToFieldType = typename ToDataType::FieldType;
using ToNativeType = typename ToFieldType::NativeType;
ReturnType convertToDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result);
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
extern template void convertToDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result); \
extern template bool convertToDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);
#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
FOR_EACH_DECIMAL_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
if constexpr (std::is_floating_point_v<FromFieldType>)
{
if (!std::isfinite(value))
{
if constexpr (throw_exception)
throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow. Cannot convert infinity or NaN to decimal", ToDataType::family_name);
else
return ReturnType(false);
}
auto out = value * static_cast<FromFieldType>(DecimalUtils::scaleMultiplier<ToNativeType>(scale));
if (out <= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::min()) ||
out >= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::max()))
{
if constexpr (throw_exception)
throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow. Float is out of Decimal range", ToDataType::family_name);
else
return ReturnType(false);
}
result = static_cast<ToNativeType>(out);
return ReturnType(true);
}
else
{
if constexpr (is_big_int_v<FromFieldType>)
return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal256>, ToDataType, ReturnType>(static_cast<Int256>(value), 0, scale, result));
else if constexpr (std::is_same_v<FromFieldType, UInt64>)
return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal128>, ToDataType, ReturnType>(static_cast<Int128>(value), 0, scale, result));
else
return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal64>, ToDataType, ReturnType>(static_cast<Int64>(value), 0, scale, result));
}
}
template <typename FromDataType, typename ToDataType>
requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
inline typename ToDataType::FieldType convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
{
typename ToDataType::FieldType result;
convertToDecimalImpl<FromDataType, ToDataType, void>(value, scale, result);
return result;
}
typename ToDataType::FieldType convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale);
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
extern template typename TO_DATA_TYPE::FieldType convertToDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale);
#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
FOR_EACH_DECIMAL_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
template <typename FromDataType, typename ToDataType>
requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
inline bool tryConvertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
{
return convertToDecimalImpl<FromDataType, ToDataType, bool>(value, scale, result);
}
bool tryConvertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result);
#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
extern template bool tryConvertToDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType& result);
#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
FOR_EACH_DECIMAL_TYPE(INVOKE);
#undef INVOKE
#undef DISPATCH
template <typename T>
inline DataTypePtr createDecimalMaxPrecision(UInt64 scale)
{
return std::make_shared<DataTypeDecimal<T>>(DecimalUtils::max_precision<T>, scale);
}
DataTypePtr createDecimalMaxPrecision(UInt64 scale);
extern template DataTypePtr createDecimalMaxPrecision<Decimal32>(UInt64 scale);
extern template DataTypePtr createDecimalMaxPrecision<Decimal64>(UInt64 scale);
extern template DataTypePtr createDecimalMaxPrecision<Decimal128>(UInt64 scale);
extern template DataTypePtr createDecimalMaxPrecision<Decimal256>(UInt64 scale);
extern template class DataTypeDecimal<Decimal32>;
extern template class DataTypeDecimal<Decimal64>;
extern template class DataTypeDecimal<Decimal128>;
extern template class DataTypeDecimal<Decimal256>;
}

17
src/DataTypes/DataTypesNumber.cpp
View File

@ -102,4 +102,21 @@ void registerDataTypeNumbers(DataTypeFactory & factory)
factory.registerAlias("DOUBLE PRECISION", "Float64", DataTypeFactory::CaseInsensitive);
}
/// Explicit template instantiations.
template class DataTypeNumber<UInt8>;
template class DataTypeNumber<UInt16>;
template class DataTypeNumber<UInt32>;
template class DataTypeNumber<UInt64>;
template class DataTypeNumber<Int8>;
template class DataTypeNumber<Int16>;
template class DataTypeNumber<Int32>;
template class DataTypeNumber<Int64>;
template class DataTypeNumber<Float32>;
template class DataTypeNumber<Float64>;
template class DataTypeNumber<UInt128>;
template class DataTypeNumber<Int128>;
template class DataTypeNumber<UInt256>;
template class DataTypeNumber<Int256>;
}

16
src/DataTypes/DataTypesNumber.h
View File

@ -55,6 +55,22 @@ private:
bool unsigned_can_be_signed = false;
};
extern template class DataTypeNumber<UInt8>;
extern template class DataTypeNumber<UInt16>;
extern template class DataTypeNumber<UInt32>;
extern template class DataTypeNumber<UInt64>;
extern template class DataTypeNumber<Int8>;
extern template class DataTypeNumber<Int16>;
extern template class DataTypeNumber<Int32>;
extern template class DataTypeNumber<Int64>;
extern template class DataTypeNumber<Float32>;
extern template class DataTypeNumber<Float64>;
extern template class DataTypeNumber<UInt128>;
extern template class DataTypeNumber<Int128>;
extern template class DataTypeNumber<UInt256>;
extern template class DataTypeNumber<Int256>;
using DataTypeUInt8 = DataTypeNumber<UInt8>;
using DataTypeUInt16 = DataTypeNumber<UInt16>;
using DataTypeUInt32 = DataTypeNumber<UInt32>;

87
src/DataTypes/IDataType.cpp
View File

@ -267,4 +267,91 @@ SerializationPtr IDataType::getSerialization(const NameAndTypePair & column)
return column.type->getDefaultSerialization();
}
#define FOR_TYPES_OF_TYPE(M) \
M(TypeIndex) \
M(const IDataType &) \
M(const DataTypePtr &) \
M(WhichDataType)
#define DISPATCH(TYPE) \
bool isUInt8(TYPE data_type) { return WhichDataType(data_type).isUInt8(); } \
bool isUInt16(TYPE data_type) { return WhichDataType(data_type).isUInt16(); } \
bool isUInt32(TYPE data_type) { return WhichDataType(data_type).isUInt32(); } \
bool isUInt64(TYPE data_type) { return WhichDataType(data_type).isUInt64(); } \
bool isNativeUInt(TYPE data_type) { return WhichDataType(data_type).isNativeUInt(); } \
bool isUInt(TYPE data_type) { return WhichDataType(data_type).isUInt(); } \
\
bool isInt8(TYPE data_type) { return WhichDataType(data_type).isInt8(); } \
bool isInt16(TYPE data_type) { return WhichDataType(data_type).isInt16(); } \
bool isInt32(TYPE data_type) { return WhichDataType(data_type).isInt32(); } \
bool isInt64(TYPE data_type) { return WhichDataType(data_type).isInt64(); } \
bool isNativeInt(TYPE data_type) { return WhichDataType(data_type).isNativeInt(); } \
bool isInt(TYPE data_type) { return WhichDataType(data_type).isInt(); } \
\
bool isInteger(TYPE data_type) { return WhichDataType(data_type).isInteger(); } \
bool isNativeInteger(TYPE data_type) { return WhichDataType(data_type).isNativeInteger(); } \
\
bool isDecimal(TYPE data_type) { return WhichDataType(data_type).isDecimal(); } \
\
bool isFloat(TYPE data_type) { return WhichDataType(data_type).isFloat(); } \
\
bool isNativeNumber(TYPE data_type) { return WhichDataType(data_type).isNativeNumber(); } \
bool isNumber(TYPE data_type) { return WhichDataType(data_type).isNumber(); } \
\
bool isEnum8(TYPE data_type) { return WhichDataType(data_type).isEnum8(); } \
bool isEnum16(TYPE data_type) { return WhichDataType(data_type).isEnum16(); } \
bool isEnum(TYPE data_type) { return WhichDataType(data_type).isEnum(); } \
\
bool isDate(TYPE data_type) { return WhichDataType(data_type).isDate(); } \
bool isDate32(TYPE data_type) { return WhichDataType(data_type).isDate32(); } \
bool isDateOrDate32(TYPE data_type) { return WhichDataType(data_type).isDateOrDate32(); } \
bool isDateTime(TYPE data_type) { return WhichDataType(data_type).isDateTime(); } \
bool isDateTime64(TYPE data_type) { return WhichDataType(data_type).isDateTime64(); } \
bool isDateTimeOrDateTime64(TYPE data_type) { return WhichDataType(data_type).isDateTimeOrDateTime64(); } \
bool isDateOrDate32OrDateTimeOrDateTime64(TYPE data_type) { return WhichDataType(data_type).isDateOrDate32OrDateTimeOrDateTime64(); } \
\
bool isString(TYPE data_type) { return WhichDataType(data_type).isString(); } \
bool isFixedString(TYPE data_type) { return WhichDataType(data_type).isFixedString(); } \
bool isStringOrFixedString(TYPE data_type) { return WhichDataType(data_type).isStringOrFixedString(); } \
\
bool isUUID(TYPE data_type) { return WhichDataType(data_type).isUUID(); } \
bool isIPv4(TYPE data_type) { return WhichDataType(data_type).isIPv4(); } \
bool isIPv6(TYPE data_type) { return WhichDataType(data_type).isIPv6(); } \
bool isArray(TYPE data_type) { return WhichDataType(data_type).isArray(); } \
bool isTuple(TYPE data_type) { return WhichDataType(data_type).isTuple(); } \
bool isMap(TYPE data_type) {return WhichDataType(data_type).isMap(); } \
bool isInterval(TYPE data_type) {return WhichDataType(data_type).isInterval(); } \
bool isObject(TYPE data_type) { return WhichDataType(data_type).isObject(); } \
bool isVariant(TYPE data_type) { return WhichDataType(data_type).isVariant(); } \
bool isNothing(TYPE data_type) { return WhichDataType(data_type).isNothing(); } \
\
bool isColumnedAsNumber(TYPE data_type) \
{ \
WhichDataType which(data_type); \
return which.isInteger() || which.isFloat() || which.isDateOrDate32OrDateTimeOrDateTime64() || which.isUUID() || which.isIPv4() || which.isIPv6(); \
} \
\
bool isColumnedAsDecimal(TYPE data_type) \
{ \
WhichDataType which(data_type); \
return which.isDecimal() || which.isDateTime64(); \
} \
\
bool isNotCreatable(TYPE data_type) \
{ \
WhichDataType which(data_type); \
return which.isNothing() || which.isFunction() || which.isSet(); \
} \
\
bool isNotDecimalButComparableToDecimal(TYPE data_type) \
{ \
WhichDataType which(data_type); \
return which.isInt() || which.isUInt() || which.isFloat(); \
} \
FOR_TYPES_OF_TYPE(DISPATCH)
#undef DISPATCH
#undef FOR_TYPES_OF_TYPE
}

142
src/DataTypes/IDataType.h
View File

@ -424,71 +424,76 @@ struct WhichDataType
/// IDataType helpers (alternative for IDataType virtual methods with single point of truth)
template <typename T> inline bool isUInt8(const T & data_type) { return WhichDataType(data_type).isUInt8(); }
template <typename T> inline bool isUInt16(const T & data_type) { return WhichDataType(data_type).isUInt16(); }
template <typename T> inline bool isUInt32(const T & data_type) { return WhichDataType(data_type).isUInt32(); }
template <typename T> inline bool isUInt64(const T & data_type) { return WhichDataType(data_type).isUInt64(); }
template <typename T> inline bool isNativeUInt(const T & data_type) { return WhichDataType(data_type).isNativeUInt(); }
template <typename T> inline bool isUInt(const T & data_type) { return WhichDataType(data_type).isUInt(); }
#define FOR_TYPES_OF_TYPE(M) \
M(TypeIndex) \
M(const IDataType &) \
M(const DataTypePtr &) \
M(WhichDataType)
template <typename T> inline bool isInt8(const T & data_type) { return WhichDataType(data_type).isInt8(); }
template <typename T> inline bool isInt16(const T & data_type) { return WhichDataType(data_type).isInt16(); }
template <typename T> inline bool isInt32(const T & data_type) { return WhichDataType(data_type).isInt32(); }
template <typename T> inline bool isInt64(const T & data_type) { return WhichDataType(data_type).isInt64(); }
template <typename T> inline bool isNativeInt(const T & data_type) { return WhichDataType(data_type).isNativeInt(); }
template <typename T> inline bool isInt(const T & data_type) { return WhichDataType(data_type).isInt(); }
#define DISPATCH(TYPE) \
bool isUInt8(TYPE data_type); \
bool isUInt16(TYPE data_type); \
bool isUInt32(TYPE data_type); \
bool isUInt64(TYPE data_type); \
bool isNativeUInt(TYPE data_type); \
bool isUInt(TYPE data_type); \
\
bool isInt8(TYPE data_type); \
bool isInt16(TYPE data_type); \
bool isInt32(TYPE data_type); \
bool isInt64(TYPE data_type); \
bool isNativeInt(TYPE data_type); \
bool isInt(TYPE data_type); \
\
bool isInteger(TYPE data_type); \
bool isNativeInteger(TYPE data_type); \
\
bool isDecimal(TYPE data_type); \
\
bool isFloat(TYPE data_type); \
\
bool isNativeNumber(TYPE data_type); \
bool isNumber(TYPE data_type); \
\
bool isEnum8(TYPE data_type); \
bool isEnum16(TYPE data_type); \
bool isEnum(TYPE data_type); \
\
bool isDate(TYPE data_type); \
bool isDate32(TYPE data_type); \
bool isDateOrDate32(TYPE data_type); \
bool isDateTime(TYPE data_type); \
bool isDateTime64(TYPE data_type); \
bool isDateTimeOrDateTime64(TYPE data_type); \
bool isDateOrDate32OrDateTimeOrDateTime64(TYPE data_type); \
\
bool isString(TYPE data_type); \
bool isFixedString(TYPE data_type); \
bool isStringOrFixedString(TYPE data_type); \
\
bool isUUID(TYPE data_type); \
bool isIPv4(TYPE data_type); \
bool isIPv6(TYPE data_type); \
bool isArray(TYPE data_type); \
bool isTuple(TYPE data_type); \
bool isMap(TYPE data_type); \
bool isInterval(TYPE data_type); \
bool isObject(TYPE data_type); \
bool isVariant(TYPE data_type); \
bool isNothing(TYPE data_type); \
\
bool isColumnedAsNumber(TYPE data_type); \
\
bool isColumnedAsDecimal(TYPE data_type); \
\
bool isNotCreatable(TYPE data_type); \
\
bool isNotDecimalButComparableToDecimal(TYPE data_type); \
template <typename T> inline bool isInteger(const T & data_type) { return WhichDataType(data_type).isInteger(); }
template <typename T> inline bool isNativeInteger(const T & data_type) { return WhichDataType(data_type).isNativeInteger(); }
FOR_TYPES_OF_TYPE(DISPATCH)
template <typename T> inline bool isDecimal(const T & data_type) { return WhichDataType(data_type).isDecimal(); }
template <typename T> inline bool isFloat(const T & data_type) { return WhichDataType(data_type).isFloat(); }
template <typename T> inline bool isNativeNumber(const T & data_type) { return WhichDataType(data_type).isNativeNumber(); }
template <typename T> inline bool isNumber(const T & data_type) { return WhichDataType(data_type).isNumber(); }
template <typename T> inline bool isEnum8(const T & data_type) { return WhichDataType(data_type).isEnum8(); }
template <typename T> inline bool isEnum16(const T & data_type) { return WhichDataType(data_type).isEnum16(); }
template <typename T> inline bool isEnum(const T & data_type) { return WhichDataType(data_type).isEnum(); }
template <typename T> inline bool isDate(const T & data_type) { return WhichDataType(data_type).isDate(); }
template <typename T> inline bool isDate32(const T & data_type) { return WhichDataType(data_type).isDate32(); }
template <typename T> inline bool isDateOrDate32(const T & data_type) { return WhichDataType(data_type).isDateOrDate32(); }
template <typename T> inline bool isDateTime(const T & data_type) { return WhichDataType(data_type).isDateTime(); }
template <typename T> inline bool isDateTime64(const T & data_type) { return WhichDataType(data_type).isDateTime64(); }
template <typename T> inline bool isDateTimeOrDateTime64(const T & data_type) { return WhichDataType(data_type).isDateTimeOrDateTime64(); }
template <typename T> inline bool isDateOrDate32OrDateTimeOrDateTime64(const T & data_type) { return WhichDataType(data_type).isDateOrDate32OrDateTimeOrDateTime64(); }
template <typename T> inline bool isString(const T & data_type) { return WhichDataType(data_type).isString(); }
template <typename T> inline bool isFixedString(const T & data_type) { return WhichDataType(data_type).isFixedString(); }
template <typename T> inline bool isStringOrFixedString(const T & data_type) { return WhichDataType(data_type).isStringOrFixedString(); }
template <typename T> inline bool isUUID(const T & data_type) { return WhichDataType(data_type).isUUID(); }
template <typename T> inline bool isIPv4(const T & data_type) { return WhichDataType(data_type).isIPv4(); }
template <typename T> inline bool isIPv6(const T & data_type) { return WhichDataType(data_type).isIPv6(); }
template <typename T> inline bool isArray(const T & data_type) { return WhichDataType(data_type).isArray(); }
template <typename T> inline bool isTuple(const T & data_type) { return WhichDataType(data_type).isTuple(); }
template <typename T> inline bool isMap(const T & data_type) {return WhichDataType(data_type).isMap(); }
template <typename T> inline bool isInterval(const T & data_type) {return WhichDataType(data_type).isInterval(); }
template <typename T> inline bool isObject(const T & data_type) { return WhichDataType(data_type).isObject(); }
template <typename T> inline bool isVariant(const T & data_type) { return WhichDataType(data_type).isVariant(); }
template <typename T> inline bool isNothing(const T & data_type) { return WhichDataType(data_type).isNothing(); }
template <typename T>
inline bool isColumnedAsNumber(const T & data_type)
{
WhichDataType which(data_type);
return which.isInteger() || which.isFloat() || which.isDateOrDate32OrDateTimeOrDateTime64() || which.isUUID() || which.isIPv4() || which.isIPv6();
}
template <typename T>
inline bool isColumnedAsDecimal(const T & data_type)
{
WhichDataType which(data_type);
return which.isDecimal() || which.isDateTime64();
}
#undef DISPATCH
#undef FOR_TYPES_OF_TYPE
// Same as isColumnedAsDecimal but also checks value type of underlyig column.
template <typename T, typename DataType>
@ -498,19 +503,6 @@ inline bool isColumnedAsDecimalT(const DataType & data_type)
return (which.isDecimal() || which.isDateTime64()) && which.idx == TypeToTypeIndex<T>;
}
template <typename T>
inline bool isNotCreatable(const T & data_type)
{
WhichDataType which(data_type);
return which.isNothing() || which.isFunction() || which.isSet();
}
inline bool isNotDecimalButComparableToDecimal(const DataTypePtr & data_type)
{
WhichDataType which(data_type);
return which.isInt() || which.isUInt() || which.isFloat();
}
inline bool isBool(const DataTypePtr & data_type)
{
return data_type->getName() == "Bool";

6
src/DataTypes/Serializations/SerializationDecimalBase.h
View File

@ -29,4 +29,10 @@ public:
void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override;
};
extern template class SerializationDecimalBase<Decimal32>;
extern template class SerializationDecimalBase<Decimal64>;
extern template class SerializationDecimalBase<Decimal128>;
extern template class SerializationDecimalBase<Decimal256>;
extern template class SerializationDecimalBase<DateTime64>;
}

18
src/Dictionaries/HashedArrayDictionary.cpp
View File

@ -1078,7 +1078,7 @@ void HashedArrayDictionary<dictionary_key_type, sharded>::calculateBytesAllocate
bytes_allocated += container.allocated_bytes();
}
bucket_count = container.capacity();
bucket_count += container.capacity();
}
};
@ -1089,6 +1089,13 @@ void HashedArrayDictionary<dictionary_key_type, sharded>::calculateBytesAllocate
bytes_allocated += container.size();
}
/// `bucket_count` should be a sum over all shards,
/// but it should not be a sum over all attributes, since it is used to
/// calculate load_factor like this: `element_count / bucket_count`
/// While element_count is a sum over all shards, not over all attributes.
if (attributes.size())
bucket_count /= attributes.size();
if (update_field_loaded_block)
bytes_allocated += update_field_loaded_block->allocatedBytes();
@ -1167,17 +1174,24 @@ void registerDictionaryArrayHashed(DictionaryFactory & factory)
if (shards <= 0 || 128 < shards)
throw Exception(ErrorCodes::BAD_ARGUMENTS,"{}: SHARDS parameter should be within [1, 128]", full_name);
HashedArrayDictionaryStorageConfiguration configuration{require_nonempty, dict_lifetime, static_cast<size_t>(shards)};
Int64 shard_load_queue_backlog = config.getInt(config_prefix + dictionary_layout_prefix + ".shard_load_queue_backlog", 10000);
if (shard_load_queue_backlog <= 0)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: SHARD_LOAD_QUEUE_BACKLOG parameter should be greater then zero", full_name);
if (source_ptr->hasUpdateField() && shards > 1)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: SHARDS parameter does not supports for updatable source (UPDATE_FIELD)", full_name);
HashedArrayDictionaryStorageConfiguration configuration{require_nonempty, dict_lifetime, static_cast<size_t>(shards), static_cast<UInt64>(shard_load_queue_backlog)};
ContextMutablePtr context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix);
const auto & settings = context->getSettingsRef();
const auto * clickhouse_source = dynamic_cast<const ClickHouseDictionarySource *>(source_ptr.get());
configuration.use_async_executor = clickhouse_source && clickhouse_source->isLocal() && settings.dictionary_use_async_executor;
if (settings.max_execution_time.totalSeconds() > 0)
configuration.load_timeout = std::chrono::seconds(settings.max_execution_time.totalSeconds());
if (dictionary_key_type == DictionaryKeyType::Simple)
{
if (shards > 1)

1
src/Dictionaries/HashedArrayDictionary.h
View File

@ -29,6 +29,7 @@ struct HashedArrayDictionaryStorageConfiguration
size_t shards = 1;
size_t shard_load_queue_backlog = 10000;
bool use_async_executor = false;
std::chrono::seconds load_timeout{0};
};
template <DictionaryKeyType dictionary_key_type, bool sharded>

1
src/Dictionaries/HashedDictionary.h
View File

@ -67,6 +67,7 @@ struct HashedDictionaryConfiguration
const bool require_nonempty;
const DictionaryLifetime lifetime;
bool use_async_executor = false;
const std::chrono::seconds load_timeout{0};
};
template <DictionaryKeyType dictionary_key_type, bool sparse, bool sharded>

72
src/Dictionaries/HashedDictionaryParallelLoader.h
View File

@ -31,6 +31,7 @@ template <DictionaryKeyType dictionary_key_type, bool sparse, bool sharded> clas
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int TIMEOUT_EXCEEDED;
}
}
@ -50,9 +51,10 @@ public:
, shards(dictionary.configuration.shards)
, pool(CurrentMetrics::HashedDictionaryThreads, CurrentMetrics::HashedDictionaryThreadsActive, CurrentMetrics::HashedDictionaryThreadsScheduled, shards)
, shards_queues(shards)
, loading_timeout(dictionary.configuration.load_timeout)
{
UInt64 backlog = dictionary.configuration.shard_load_queue_backlog;
LOG_TRACE(dictionary.log, "Will load the {} dictionary using {} threads (with {} backlog)", dictionary_name, shards, backlog);
LOG_TRACE(dictionary.log, "Will load the {} dictionary using {} threads (with {} backlog and timeout {} sec)", dictionary_name, shards, backlog, loading_timeout.count());
shards_slots.resize(shards);
iota(shards_slots.data(), shards_slots.size(), UInt64(0));
@ -62,7 +64,11 @@ public:
shards_queues[shard].emplace(backlog);
pool.scheduleOrThrowOnError([this, shard, thread_group = CurrentThread::getGroup()]
{
WorkerStatistic statistic;
SCOPE_EXIT_SAFE(
LOG_TRACE(dictionary.log, "Finished worker for dictionary {} shard {}, processed {} blocks, {} rows, total time {}ms",
dictionary_name, shard, statistic.total_blocks, statistic.total_rows, statistic.total_elapsed_ms);
if (thread_group)
CurrentThread::detachFromGroupIfNotDetached();
);
@ -74,7 +80,9 @@ public:
CurrentThread::attachToGroupIfDetached(thread_group);
setThreadName("HashedDictLoad");
threadWorker(shard);
LOG_TRACE(dictionary.log, "Starting worker for dictionary {}, shard {}", dictionary_name, shard);
threadWorker(shard, statistic);
});
}
}
@ -87,8 +95,28 @@ public:
for (size_t shard = 0; shard < shards; ++shard)
{
if (!shards_queues[shard]->push(std::move(shards_blocks[shard])))
throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not push to shards queue #{}", shard);
const auto & current_block = shards_blocks[shard];
while (!shards_queues[shard]->tryPush(current_block, /* milliseconds= */ 100))
{
if (shards_queues[shard]->isFinished())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not push to finished shards queue #{}, dictionary {}", shard, dictionary_name);
/// We need to check if some workers failed
if (pool.active() != shards)
{
LOG_DEBUG(dictionary.log, "Some workers for dictionary {} failed, stopping all workers", dictionary_name);
stop_all_workers = true;
pool.wait(); /// We expect exception to be thrown from the failed worker thread
throw Exception(ErrorCodes::LOGICAL_ERROR, "Worker threads for dictionary {} are not active", dictionary_name);
}
if (loading_timeout.count() && std::chrono::milliseconds(total_loading_time.elapsedMilliseconds()) > loading_timeout)
{
stop_all_workers = true;
pool.wait();
throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout {} sec for dictionary {} loading is expired", loading_timeout.count(), dictionary_name);
}
}
}
}
@ -124,27 +152,49 @@ private:
String dictionary_name;
const size_t shards;
ThreadPool pool;
std::atomic_bool stop_all_workers{false};
std::vector<std::optional<ConcurrentBoundedQueue<Block>>> shards_queues;
std::chrono::seconds loading_timeout;
Stopwatch total_loading_time;
std::vector<UInt64> shards_slots;
DictionaryKeysArenaHolder<dictionary_key_type> arena_holder;
void threadWorker(size_t shard)
struct WorkerStatistic
{
UInt64 total_elapsed_ms = 0;
UInt64 total_blocks = 0;
UInt64 total_rows = 0;
};
void threadWorker(size_t shard, WorkerStatistic & statistic)
{
Block block;
DictionaryKeysArenaHolder<dictionary_key_type> arena_holder_;
auto & shard_queue = *shards_queues[shard];
while (shard_queue.pop(block))
while (true)
{
if (!shard_queue.tryPop(block, /* milliseconds= */ 100))
{
/// Check if we need to stop
if (stop_all_workers || shard_queue.isFinished())
break;
/// Timeout expired, but the queue is not finished yet, try again
continue;
}
Stopwatch watch;
dictionary.blockToAttributes(block, arena_holder_, shard);
UInt64 elapsed_ms = watch.elapsedMilliseconds();
if (elapsed_ms > 1'000)
LOG_TRACE(dictionary.log, "Block processing for shard #{} is slow {}ms (rows {}).", shard, elapsed_ms, block.rows());
}
if (!shard_queue.isFinished())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not pull non finished shards queue #{}", shard);
statistic.total_elapsed_ms += elapsed_ms;
statistic.total_blocks += 1;
statistic.total_rows += block.rows();
if (elapsed_ms > 1'000)
LOG_TRACE(dictionary.log, "Block processing for shard #{} is slow {}ms (rows {})", shard, elapsed_ms, block.rows());
}
}
/// Split block to shards smaller block, using 'selector'.

1
src/Dictionaries/registerHashedDictionary.cpp
View File

@ -77,6 +77,7 @@ void registerDictionaryHashed(DictionaryFactory & factory)
require_nonempty,
dict_lifetime,
use_async_executor,
std::chrono::seconds(settings.max_execution_time.totalSeconds()),
};
if (source_ptr->hasUpdateField() && shards > 1)

9
src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
View File

@ -23,7 +23,6 @@ namespace ErrorCodes
extern const int CANNOT_OPEN_FILE;
extern const int FILE_DOESNT_EXIST;
extern const int BAD_FILE_TYPE;
extern const int FILE_ALREADY_EXISTS;
extern const int CANNOT_PARSE_INPUT_ASSERTION_FAILED;
extern const int LOGICAL_ERROR;
}
@ -593,14 +592,8 @@ void DiskObjectStorageTransaction::moveDirectory(const std::string & from_path,
void DiskObjectStorageTransaction::moveFile(const String & from_path, const String & to_path)
{
operations_to_execute.emplace_back(
std::make_unique<PureMetadataObjectStorageOperation>(object_storage, metadata_storage, [from_path, to_path, this](MetadataTransactionPtr tx)
std::make_unique<PureMetadataObjectStorageOperation>(object_storage, metadata_storage, [from_path, to_path](MetadataTransactionPtr tx)
{
if (metadata_storage.exists(to_path))
throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "File already exists: {}", to_path);
if (!metadata_storage.exists(from_path))
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "File {} doesn't exist, cannot move", from_path);
tx->moveFile(from_path, to_path);
}));
}

15
src/Functions/DateTimeTransforms.cpp
View File

@ -10,16 +10,17 @@ namespace ErrorCodes
void throwDateIsNotSupported(const char * name)
{
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type Date of argument for function {}", name);
}
void throwDateTimeIsNotSupported(const char * name)
{
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type DateTime of argument for function {}", name);
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal argument of type Date for function {}", name);
}
void throwDate32IsNotSupported(const char * name)
{
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type Date32 of argument for function {}", name);
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal argument of type Date32 for function {}", name);
}
void throwDateTimeIsNotSupported(const char * name)
{
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal argument of type DateTime for function {}", name);
}
}

71
src/Functions/DateTimeTransforms.h
View File

@ -6,6 +6,7 @@
#include <Common/DateLUTImpl.h>
#include <Common/DateLUT.h>
#include <Common/IntervalKind.h>
#include "base/Decimal.h"
#include <Columns/ColumnNullable.h>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnVector.h>
@ -54,8 +55,8 @@ constexpr time_t MAX_DATE_TIMESTAMP = 5662310399; // 2149-06-06 23:59:59 U
constexpr time_t MAX_DATETIME_DAY_NUM = 49710; // 2106-02-07
[[noreturn]] void throwDateIsNotSupported(const char * name);
[[noreturn]] void throwDateTimeIsNotSupported(const char * name);
[[noreturn]] void throwDate32IsNotSupported(const char * name);
[[noreturn]] void throwDateTimeIsNotSupported(const char * name);
/// This factor transformation will say that the function is monotone everywhere.
struct ZeroTransform
@ -481,7 +482,7 @@ struct ToStartOfInterval<IntervalKind::Kind::Nanosecond>
}
static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64)
{
throwDateIsNotSupported(TO_START_OF_INTERVAL_NAME);
throwDate32IsNotSupported(TO_START_OF_INTERVAL_NAME);
}
static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64)
{
@ -516,7 +517,7 @@ struct ToStartOfInterval<IntervalKind::Kind::Microsecond>
}
static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64)
{
throwDateIsNotSupported(TO_START_OF_INTERVAL_NAME);
throwDate32IsNotSupported(TO_START_OF_INTERVAL_NAME);
}
static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64)
{
@ -559,7 +560,7 @@ struct ToStartOfInterval<IntervalKind::Kind::Millisecond>
}
static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64)
{
throwDateIsNotSupported(TO_START_OF_INTERVAL_NAME);
throwDate32IsNotSupported(TO_START_OF_INTERVAL_NAME);
}
static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64)
{
@ -602,7 +603,7 @@ struct ToStartOfInterval<IntervalKind::Kind::Second>
}
static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64)
{
throwDateIsNotSupported(TO_START_OF_INTERVAL_NAME);
throwDate32IsNotSupported(TO_START_OF_INTERVAL_NAME);
}
static UInt32 execute(UInt32 t, Int64 seconds, const DateLUTImpl & time_zone, Int64)
{
@ -623,7 +624,7 @@ struct ToStartOfInterval<IntervalKind::Kind::Minute>
}
static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64)
{
throwDateIsNotSupported(TO_START_OF_INTERVAL_NAME);
throwDate32IsNotSupported(TO_START_OF_INTERVAL_NAME);
}
static UInt32 execute(UInt32 t, Int64 minutes, const DateLUTImpl & time_zone, Int64)
{
@ -644,7 +645,7 @@ struct ToStartOfInterval<IntervalKind::Kind::Hour>
}
static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64)
{
throwDateIsNotSupported(TO_START_OF_INTERVAL_NAME);
throwDate32IsNotSupported(TO_START_OF_INTERVAL_NAME);
}
static UInt32 execute(UInt32 t, Int64 hours, const DateLUTImpl & time_zone, Int64)
{
@ -777,7 +778,7 @@ struct ToTimeImpl
}
static UInt32 execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static UInt32 execute(UInt16, const DateLUTImpl &)
{
@ -802,7 +803,7 @@ struct ToStartOfMinuteImpl
}
static UInt32 execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static UInt32 execute(UInt16, const DateLUTImpl &)
{
@ -849,7 +850,7 @@ struct ToStartOfSecondImpl
}
static UInt32 execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static UInt32 execute(UInt16, const DateLUTImpl &)
{
@ -897,7 +898,7 @@ struct ToStartOfMillisecondImpl
}
static UInt32 execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static UInt32 execute(UInt16, const DateLUTImpl &)
{
@ -941,7 +942,7 @@ struct ToStartOfMicrosecondImpl
}
static UInt32 execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static UInt32 execute(UInt16, const DateLUTImpl &)
{
@ -979,7 +980,7 @@ struct ToStartOfNanosecondImpl
}
static UInt32 execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static UInt32 execute(UInt16, const DateLUTImpl &)
{
@ -1004,7 +1005,7 @@ struct ToStartOfFiveMinutesImpl
}
static UInt32 execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static UInt32 execute(UInt16, const DateLUTImpl &)
{
@ -1036,7 +1037,7 @@ struct ToStartOfTenMinutesImpl
}
static UInt32 execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static UInt32 execute(UInt16, const DateLUTImpl &)
{
@ -1068,7 +1069,7 @@ struct ToStartOfFifteenMinutesImpl
}
static UInt32 execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static UInt32 execute(UInt16, const DateLUTImpl &)
{
@ -1103,7 +1104,7 @@ struct TimeSlotImpl
static UInt32 execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static UInt32 execute(UInt16, const DateLUTImpl &)
@ -1142,7 +1143,7 @@ struct ToStartOfHourImpl
static UInt32 execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static UInt32 execute(UInt16, const DateLUTImpl &)
@ -1429,7 +1430,7 @@ struct ToHourImpl
}
static UInt8 execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static UInt8 execute(UInt16, const DateLUTImpl &)
{
@ -1456,7 +1457,7 @@ struct TimezoneOffsetImpl
static time_t execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static time_t execute(UInt16, const DateLUTImpl &)
@ -1482,7 +1483,7 @@ struct ToMinuteImpl
}
static UInt8 execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static UInt8 execute(UInt16, const DateLUTImpl &)
{
@ -1507,7 +1508,7 @@ struct ToSecondImpl
}
static UInt8 execute(Int32, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
throwDate32IsNotSupported(name);
}
static UInt8 execute(UInt16, const DateLUTImpl &)
{
@ -1518,6 +1519,32 @@ struct ToSecondImpl
using FactorTransform = ToStartOfMinuteImpl;
};
struct ToMillisecondImpl
{
static constexpr auto name = "toMillisecond";
static UInt16 execute(const DateTime64 & datetime64, Int64 scale_multiplier, const DateLUTImpl & time_zone)
{
return time_zone.toMillisecond<DateTime64>(datetime64, scale_multiplier);
}
static UInt16 execute(UInt32, const DateLUTImpl &)
{
return 0;
}
static UInt16 execute(Int32, const DateLUTImpl &)
{
throwDate32IsNotSupported(name);
}
static UInt16 execute(UInt16, const DateLUTImpl &)
{
throwDateIsNotSupported(name);
}
static constexpr bool hasPreimage() { return false; }
using FactorTransform = ZeroTransform;
};
struct ToISOYearImpl
{
static constexpr auto name = "toISOYear";

2
src/Functions/FunctionBase64Conversion.h
View File

@ -100,7 +100,7 @@ public:
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
FunctionArgumentDescriptors mandatory_arguments{
{"value", &isStringOrFixedString<IDataType>, nullptr, "String or FixedString"}
{"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String or FixedString"}
};
validateFunctionArgumentTypes(*this, arguments, mandatory_arguments);

6
src/Functions/FunctionHelpers.h
View File

@ -108,8 +108,10 @@ struct FunctionArgumentDescriptor
{
const char * argument_name;
std::function<bool (const IDataType &)> type_validator_func;
std::function<bool (const IColumn &)> column_validator_func;
using TypeValidator = bool (*)(const IDataType &);
TypeValidator type_validator_func;
using ColumnValidator = bool (*)(const IColumn &);
ColumnValidator column_validator_func;
const char * expected_type_description;

6
src/Functions/FunctionStringReplace.h
View File

@ -35,9 +35,9 @@ public:
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
FunctionArgumentDescriptors args{
{"haystack", &isStringOrFixedString<IDataType>, nullptr, "String or FixedString"},
{"pattern", &isString<IDataType>, nullptr, "String"},
{"replacement", &isString<IDataType>, nullptr, "String"}
{"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String or FixedString"},
{"pattern", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
{"replacement", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
};
validateFunctionArgumentTypes(*this, arguments, args);

12
src/Functions/FunctionTokens.h
View File

@ -74,6 +74,8 @@ public:
size_t getNumberOfArguments() const override { return Generator::getNumberOfArguments(); }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return Generator::getArgumentsThatAreAlwaysConstant(); }
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
Generator::checkArguments(*this, arguments);
@ -184,12 +186,12 @@ static inline void checkArgumentsWithSeparatorAndOptionalMaxSubstrings(
const IFunction & func, const ColumnsWithTypeAndName & arguments)
{
FunctionArgumentDescriptors mandatory_args{
{"separator", &isString<IDataType>, isColumnConst, "const String"},
{"s", &isString<IDataType>, nullptr, "String"}
{"separator", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"},
{"s", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
};
FunctionArgumentDescriptors optional_args{
{"max_substrings", &isNativeInteger<IDataType>, isColumnConst, "const Number"},
{"max_substrings", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), isColumnConst, "const Number"},
};
validateFunctionArgumentTypes(func, arguments, mandatory_args, optional_args);
@ -198,11 +200,11 @@ static inline void checkArgumentsWithSeparatorAndOptionalMaxSubstrings(
static inline void checkArgumentsWithOptionalMaxSubstrings(const IFunction & func, const ColumnsWithTypeAndName & arguments)
{
FunctionArgumentDescriptors mandatory_args{
{"s", &isString<IDataType>, nullptr, "String"},
{"s", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
};
FunctionArgumentDescriptors optional_args{
{"max_substrings", &isNativeInteger<IDataType>, isColumnConst, "const Number"},
{"max_substrings", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), isColumnConst, "const Number"},
};
validateFunctionArgumentTypes(func, arguments, mandatory_args, optional_args);

2
src/Functions/FunctionUnixTimestamp64.h
View File

@ -45,7 +45,7 @@ public:
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
FunctionArgumentDescriptors args{
{"value", &isDateTime64<IDataType>, nullptr, "DateTime64"}
{"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateTime64), nullptr, "DateTime64"}
};
validateFunctionArgumentTypes(*this, arguments, args);

20
src/Functions/FunctionsAES.h
View File

@ -154,21 +154,21 @@ private:
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
auto optional_args = FunctionArgumentDescriptors{
{"IV", &isStringOrFixedString<IDataType>, nullptr, "Initialization vector binary string"},
{"IV", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "Initialization vector binary string"},
};
if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::OpenSSL)
{
optional_args.emplace_back(FunctionArgumentDescriptor{
"AAD", &isStringOrFixedString<IDataType>, nullptr, "Additional authenticated data binary string for GCM mode"
"AAD", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "Additional authenticated data binary string for GCM mode"
});
}
validateFunctionArgumentTypes(*this, arguments,
FunctionArgumentDescriptors{
{"mode", &isStringOrFixedString<IDataType>, isColumnConst, "encryption mode string"},
{"input", &isStringOrFixedString<IDataType>, {}, "plaintext"},
{"key", &isStringOrFixedString<IDataType>, {}, "encryption key binary string"},
{"mode", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "encryption mode string"},
{"input", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), {}, "plaintext"},
{"key", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), {}, "encryption key binary string"},
},
optional_args
);
@ -425,21 +425,21 @@ private:
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
auto optional_args = FunctionArgumentDescriptors{
{"IV", &isStringOrFixedString<IDataType>, nullptr, "Initialization vector binary string"},
{"IV", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "Initialization vector binary string"},
};
if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::OpenSSL)
{
optional_args.emplace_back(FunctionArgumentDescriptor{
"AAD", &isStringOrFixedString<IDataType>, nullptr, "Additional authenticated data binary string for GCM mode"
"AAD", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "Additional authenticated data binary string for GCM mode"
});
}
validateFunctionArgumentTypes(*this, arguments,
FunctionArgumentDescriptors{
{"mode", &isStringOrFixedString<IDataType>, isColumnConst, "decryption mode string"},
{"input", &isStringOrFixedString<IDataType>, {}, "ciphertext"},
{"key", &isStringOrFixedString<IDataType>, {}, "decryption key binary string"},
{"mode", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "decryption mode string"},
{"input", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), {}, "ciphertext"},
{"key", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), {}, "decryption key binary string"},
},
optional_args
);

12
src/Functions/FunctionsConversion.h
View File

@ -2129,12 +2129,12 @@ public:
if constexpr (to_decimal)
{
mandatory_args.push_back({"scale", &isNativeInteger<IDataType>, &isColumnConst, "const Integer"});
mandatory_args.push_back({"scale", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), &isColumnConst, "const Integer"});
}
if (!to_decimal && isDateTime64<Name, ToDataType>(arguments))
{
mandatory_args.push_back({"scale", &isNativeInteger<IDataType>, &isColumnConst, "const Integer"});
mandatory_args.push_back({"scale", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), &isColumnConst, "const Integer"});
}
// toString(DateTime or DateTime64, [timezone: String])
@ -2150,7 +2150,7 @@ public:
// toDateTime64(value, scale : Integer[, timezone: String])
|| std::is_same_v<ToDataType, DataTypeDateTime64>)
{
optional_args.push_back({"timezone", &isString<IDataType>, nullptr, "String"});
optional_args.push_back({"timezone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"});
}
validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);
@ -2498,11 +2498,11 @@ public:
if (isDateTime64<Name, ToDataType>(arguments))
{
validateFunctionArgumentTypes(*this, arguments,
FunctionArgumentDescriptors{{"string", &isStringOrFixedString<IDataType>, nullptr, "String or FixedString"}},
FunctionArgumentDescriptors{{"string", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String or FixedString"}},
// optional
FunctionArgumentDescriptors{
{"precision", &isUInt8<IDataType>, isColumnConst, "const UInt8"},
{"timezone", &isStringOrFixedString<IDataType>, isColumnConst, "const String or FixedString"},
{"precision", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isUInt8), isColumnConst, "const UInt8"},
{"timezone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "const String or FixedString"},
});
UInt64 scale = to_datetime64 ? DataTypeDateTime64::default_scale : 0;

2
src/Functions/JSONArrayLength.cpp
View File

@ -45,7 +45,7 @@ namespace
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
auto args = FunctionArgumentDescriptors{
{"json", &isString<IDataType>, nullptr, "String"},
{"json", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
};
validateFunctionArgumentTypes(*this, arguments, args);

4
src/Functions/URL/URLHierarchy.cpp
View File

@ -24,10 +24,12 @@ public:
static bool isVariadic() { return false; }
static size_t getNumberOfArguments() { return 1; }
static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {}; }
static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
{
FunctionArgumentDescriptors mandatory_args{
{"URL", &isString<IDataType>, nullptr, "String"},
{"URL", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
};
validateFunctionArgumentTypes(func, arguments, mandatory_args);

4
src/Functions/URL/URLPathHierarchy.cpp
View File

@ -22,10 +22,12 @@ public:
static bool isVariadic() { return false; }
static size_t getNumberOfArguments() { return 1; }
static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {}; }
static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
{
FunctionArgumentDescriptors mandatory_args{
{"URL", &isString<IDataType>, nullptr, "String"},
{"URL", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
};
validateFunctionArgumentTypes(func, arguments, mandatory_args);

4
src/Functions/URL/extractURLParameterNames.cpp
View File

@ -22,10 +22,12 @@ public:
static bool isVariadic() { return false; }
static size_t getNumberOfArguments() { return 1; }
static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {}; }
static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
{
FunctionArgumentDescriptors mandatory_args{
{"URL", &isString<IDataType>, nullptr, "String"},
{"URL", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
};
validateFunctionArgumentTypes(func, arguments, mandatory_args);

4
src/Functions/URL/extractURLParameters.cpp
View File

@ -23,10 +23,12 @@ public:
static bool isVariadic() { return false; }
static size_t getNumberOfArguments() { return 1; }
static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {}; }
static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
{
FunctionArgumentDescriptors mandatory_args{
{"URL", &isString<IDataType>, nullptr, "String"},
{"URL", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
};
validateFunctionArgumentTypes(func, arguments, mandatory_args);

2
src/Functions/alphaTokens.cpp
View File

@ -32,6 +32,8 @@ public:
static size_t getNumberOfArguments() { return 0; }
static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {1}; }
static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
{
checkArgumentsWithOptionalMaxSubstrings(func, arguments);

4
src/Functions/array/arrayJaccardIndex.cpp
View File

@ -84,8 +84,8 @@ public:
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
FunctionArgumentDescriptors args{
{"array_1", &isArray<IDataType>, nullptr, "Array"},
{"array_2", &isArray<IDataType>, nullptr, "Array"},
{"array_1", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
{"array_2", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
};
validateFunctionArgumentTypes(*this, arguments, args);
return std::make_shared<DataTypeNumber<ResultType>>();

4
src/Functions/array/arrayRandomSample.cpp
View File

@ -36,8 +36,8 @@ public:
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
FunctionArgumentDescriptors args{
{"array", &isArray<IDataType>, nullptr, "Array"},
{"samples", &isUInt<IDataType>, isColumnConst, "const UInt*"},
{"array", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
{"samples", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isUInt), isColumnConst, "const UInt*"},
};
validateFunctionArgumentTypes(*this, arguments, args);

4
src/Functions/array/arrayShingles.cpp
View File

@ -28,8 +28,8 @@ public:
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
FunctionArgumentDescriptors args{
{"array", &isArray<IDataType>, nullptr, "Array"},
{"length", &isInteger<IDataType>, nullptr, "Integer"}
{"array", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
{"length", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isInteger), nullptr, "Integer"}
};
validateFunctionArgumentTypes(*this, arguments, args);

4
src/Functions/arrayStringConcat.cpp
View File

@ -151,12 +151,12 @@ public:
{
FunctionArgumentDescriptors mandatory_args
{
{"arr", &isArray<IDataType>, nullptr, "Array"},
{"arr", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
};
FunctionArgumentDescriptors optional_args
{
{"separator", &isString<IDataType>, isColumnConst, "const String"},
{"separator", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"},
};
validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);

4
src/Functions/castOrDefault.cpp
View File

@ -210,10 +210,10 @@ private:
FunctionArgumentDescriptors optional_args;
if constexpr (IsDataTypeDecimal<Type>)
mandatory_args.push_back({"scale", &isNativeInteger<IDataType>, &isColumnConst, "const Integer"});
mandatory_args.push_back({"scale", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), &isColumnConst, "const Integer"});
if (std::is_same_v<Type, DataTypeDateTime> || std::is_same_v<Type, DataTypeDateTime64>)
optional_args.push_back({"timezone", &isString<IDataType>, isColumnConst, "const String"});
optional_args.push_back({"timezone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"});
optional_args.push_back({"default_value", nullptr, nullptr, nullptr});

4
src/Functions/countMatches.h
View File

@ -35,8 +35,8 @@ public:
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
FunctionArgumentDescriptors args{
{"haystack", &isStringOrFixedString<IDataType>, nullptr, "String or FixedString"},
{"pattern", &isString<IDataType>, isColumnConst, "constant String"}
{"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String or FixedString"},
{"pattern", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "constant String"}
};
validateFunctionArgumentTypes(*this, arguments, args);

6
src/Functions/extractAll.cpp
View File

@ -50,11 +50,13 @@ public:
static bool isVariadic() { return false; }
static size_t getNumberOfArguments() { return 2; }
static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {1}; }
static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
{
FunctionArgumentDescriptors mandatory_args{
{"haystack", &isString<IDataType>, nullptr, "String"},
{"pattern", &isString<IDataType>, isColumnConst, "const String"}
{"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
{"pattern", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"}
};
validateFunctionArgumentTypes(func, arguments, mandatory_args);

4
src/Functions/extractAllGroups.h
View File

@ -71,8 +71,8 @@ public:
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
FunctionArgumentDescriptors args{
{"haystack", &isStringOrFixedString<IDataType>, nullptr, "const String or const FixedString"},
{"needle", &isStringOrFixedString<IDataType>, isColumnConst, "const String or const FixedString"},
{"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "const String or const FixedString"},
{"needle", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "const String or const FixedString"},
};
validateFunctionArgumentTypes(*this, arguments, args);

4
src/Functions/extractGroups.cpp
View File

@ -45,8 +45,8 @@ public:
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
FunctionArgumentDescriptors args{
{"haystack", &isStringOrFixedString<IDataType>, nullptr, "const String or const FixedString"},
{"needle", &isStringOrFixedString<IDataType>, isColumnConst, "const String or const FixedString"},
{"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "const String or const FixedString"},
{"needle", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "const String or const FixedString"},
};
validateFunctionArgumentTypes(*this, arguments, args);

2
src/Functions/formatQuery.cpp
View File

@ -54,7 +54,7 @@ public:
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
FunctionArgumentDescriptors args{
{"query", &isString<IDataType>, nullptr, "String"}
{"query", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
};
validateFunctionArgumentTypes(*this, arguments, args);

Some files were not shown because too many files have changed in this diff Show More