thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 01:22:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into tmp

Browse Source
This commit is contained in:
Alexander Kuzmenkov 2020-10-22 19:48:02 +03:00
commit 5cbf645de4
581 changed files with 32602 additions and 4303 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmake/sanitize.cmake
View File

@ -57,8 +57,8 @@ if (SANITIZE)
endif ()
elseif (SANITIZE STREQUAL "undefined")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/ubsan_suppressions.txt")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/ubsan_suppressions.txt")
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
endif()

69
debian/clickhouse-server.init vendored
View File

@ -153,82 +153,19 @@ initdb()
start()
{
[ -x $CLICKHOUSE_BINDIR/$PROGRAM ] || exit 0
local EXIT_STATUS
EXIT_STATUS=0
echo -n "Start $PROGRAM service: "
if is_running; then
echo -n "already running "
EXIT_STATUS=1
else
ulimit -n 262144
mkdir -p $CLICKHOUSE_PIDDIR
chown -R $CLICKHOUSE_USER:$CLICKHOUSE_GROUP $CLICKHOUSE_PIDDIR
initdb
if ! is_running; then
# Lock should not be held while running child process, so we release the lock. Note: obviously, there is race condition.
# But clickhouse-server has protection from simultaneous runs with same data directory.
su -s $SHELL ${CLICKHOUSE_USER} -c "$FLOCK -u 9; $CLICKHOUSE_PROGRAM_ENV exec -a \"$PROGRAM\" \"$CLICKHOUSE_BINDIR/$PROGRAM\" --daemon --pid-file=\"$CLICKHOUSE_PIDFILE\" --config-file=\"$CLICKHOUSE_CONFIG\""
EXIT_STATUS=$?
if [ $EXIT_STATUS -ne 0 ]; then
return $EXIT_STATUS
fi
fi
fi
if [ $EXIT_STATUS -eq 0 ]; then
attempts=0
while ! is_running && [ $attempts -le ${CLICKHOUSE_START_TIMEOUT:=10} ]; do
attempts=$(($attempts + 1))
sleep 1
done
if is_running; then
echo "DONE"
else
echo "UNKNOWN"
fi
else
echo "FAILED"
fi
return $EXIT_STATUS
${CLICKHOUSE_GENERIC_PROGRAM} start --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}"
}
stop()
{
#local EXIT_STATUS
EXIT_STATUS=0
if [ -f $CLICKHOUSE_PIDFILE ]; then
echo -n "Stop $PROGRAM service: "
kill -TERM $(cat "$CLICKHOUSE_PIDFILE")
if ! wait_for_done ${CLICKHOUSE_STOP_TIMEOUT}; then
EXIT_STATUS=2
echo "TIMEOUT"
else
echo "DONE"
fi
fi
return $EXIT_STATUS
${CLICKHOUSE_GENERIC_PROGRAM} stop --pid-path "${CLICKHOUSE_PIDDIR}"
}
restart()
{
check_config
if stop; then
if start; then
return 0
fi
fi
return 1
${CLICKHOUSE_GENERIC_PROGRAM} restart --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}"
}

104
debian/clickhouse-server.postinst vendored
View File

@ -2,6 +2,7 @@
set -e
# set -x
PROGRAM=clickhouse-server
CLICKHOUSE_USER=${CLICKHOUSE_USER:=clickhouse}
CLICKHOUSE_GROUP=${CLICKHOUSE_GROUP:=${CLICKHOUSE_USER}}
# Please note that we don't support paths with whitespaces. This is rather ignorant.
@ -12,6 +13,7 @@ CLICKHOUSE_BINDIR=${CLICKHOUSE_BINDIR:=/usr/bin}
CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM:=clickhouse}
EXTRACT_FROM_CONFIG=${CLICKHOUSE_GENERIC_PROGRAM}-extract-from-config
CLICKHOUSE_CONFIG=$CLICKHOUSE_CONFDIR/config.xml
CLICKHOUSE_PIDDIR=/var/run/$PROGRAM
[ -f /usr/share/debconf/confmodule ] && . /usr/share/debconf/confmodule
[ -f /etc/default/clickhouse ] && . /etc/default/clickhouse
@ -41,105 +43,5 @@ if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then
fi
fi
# Make sure the administrative user exists
if ! getent passwd ${CLICKHOUSE_USER} > /dev/null; then
if [ -n "$not_deb_os" ]; then
useradd -r -s /bin/false --home-dir /nonexistent ${CLICKHOUSE_USER} > /dev/null
else
adduser --system --disabled-login --no-create-home --home /nonexistent \
--shell /bin/false --group --gecos "ClickHouse server" ${CLICKHOUSE_USER} > /dev/null
fi
fi
# if the user was created manually, make sure the group is there as well
if ! getent group ${CLICKHOUSE_GROUP} > /dev/null; then
groupadd -r ${CLICKHOUSE_GROUP} > /dev/null
fi
# make sure user is in the correct group
if ! id -Gn ${CLICKHOUSE_USER} | grep -qw ${CLICKHOUSE_USER}; then
usermod -a -G ${CLICKHOUSE_GROUP} ${CLICKHOUSE_USER} > /dev/null
fi
# check validity of user and group
if [ "$(id -u ${CLICKHOUSE_USER})" -eq 0 ]; then
echo "The ${CLICKHOUSE_USER} system user must not have uid 0 (root).
Please fix this and reinstall this package." >&2
exit 1
fi
if [ "$(id -g ${CLICKHOUSE_GROUP})" -eq 0 ]; then
echo "The ${CLICKHOUSE_USER} system user must not have root as primary group.
Please fix this and reinstall this package." >&2
exit 1
fi
if [ -x "$CLICKHOUSE_BINDIR/$EXTRACT_FROM_CONFIG" ] && [ -f "$CLICKHOUSE_CONFIG" ]; then
if [ -z "$SHELL" ]; then
SHELL="/bin/sh"
fi
CLICKHOUSE_DATADIR_FROM_CONFIG=$(su -s $SHELL ${CLICKHOUSE_USER} -c "$CLICKHOUSE_BINDIR/$EXTRACT_FROM_CONFIG --config-file=\"$CLICKHOUSE_CONFIG\" --key=path") ||:
echo "Path to data directory in ${CLICKHOUSE_CONFIG}: ${CLICKHOUSE_DATADIR_FROM_CONFIG}"
fi
CLICKHOUSE_DATADIR_FROM_CONFIG=${CLICKHOUSE_DATADIR_FROM_CONFIG:=$CLICKHOUSE_DATADIR}
if [ ! -d ${CLICKHOUSE_DATADIR_FROM_CONFIG} ]; then
mkdir -p ${CLICKHOUSE_DATADIR_FROM_CONFIG}
chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_DATADIR_FROM_CONFIG}
chmod 700 ${CLICKHOUSE_DATADIR_FROM_CONFIG}
fi
if [ -d ${CLICKHOUSE_CONFDIR} ]; then
mkdir -p ${CLICKHOUSE_CONFDIR}/users.d
mkdir -p ${CLICKHOUSE_CONFDIR}/config.d
rm -fv ${CLICKHOUSE_CONFDIR}/*-preprocessed.xml ||:
fi
[ -e ${CLICKHOUSE_CONFDIR}/preprocessed ] || ln -s ${CLICKHOUSE_DATADIR_FROM_CONFIG}/preprocessed_configs ${CLICKHOUSE_CONFDIR}/preprocessed ||:
if [ ! -d ${CLICKHOUSE_LOGDIR} ]; then
mkdir -p ${CLICKHOUSE_LOGDIR}
chown root:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR}
# Allow everyone to read logs, root and clickhouse to read-write
chmod 775 ${CLICKHOUSE_LOGDIR}
fi
# Set net_admin capabilities to support introspection of "taskstats" performance metrics from the kernel
# and ipc_lock capabilities to allow mlock of clickhouse binary.
# 1. Check that "setcap" tool exists.
# 2. Check that an arbitrary program with installed capabilities can run.
# 3. Set the capabilities.
# The second is important for Docker and systemd-nspawn.
# When the container has no capabilities,
# but the executable file inside the container has capabilities,
# then attempt to run this file will end up with a cryptic "Operation not permitted" message.
TMPFILE=/tmp/test_setcap.sh
command -v setcap >/dev/null \
&& echo > $TMPFILE && chmod a+x $TMPFILE && $TMPFILE && setcap "cap_net_admin,cap_ipc_lock,cap_sys_nice+ep" $TMPFILE && $TMPFILE && rm $TMPFILE \
&& setcap "cap_net_admin,cap_ipc_lock,cap_sys_nice+ep" "${CLICKHOUSE_BINDIR}/${CLICKHOUSE_GENERIC_PROGRAM}" \
|| echo "Cannot set 'net_admin' or 'ipc_lock' or 'sys_nice' capability for clickhouse binary. This is optional. Taskstats accounting will be disabled. To enable taskstats accounting you may add the required capability later manually."
# Clean old dynamic compilation results
if [ -d "${CLICKHOUSE_DATADIR_FROM_CONFIG}/build" ]; then
rm -f ${CLICKHOUSE_DATADIR_FROM_CONFIG}/build/*.cpp ${CLICKHOUSE_DATADIR_FROM_CONFIG}/build/*.so ||:
fi
if [ -f /usr/share/debconf/confmodule ]; then
db_get clickhouse-server/default-password
defaultpassword="$RET"
if [ -n "$defaultpassword" ]; then
echo "<yandex><users><default><password>$defaultpassword</password></default></users></yandex>" > ${CLICKHOUSE_CONFDIR}/users.d/default-password.xml
chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_CONFDIR}/users.d/default-password.xml
chmod 600 ${CLICKHOUSE_CONFDIR}/users.d/default-password.xml
fi
# everything went well, so now let's reset the password
db_set clickhouse-server/default-password ""
# ... done with debconf here
db_stop
fi
${CLICKHOUSE_GENERIC_PROGRAM} install --user "${CLICKHOUSE_USER}" --group "${CLICKHOUSE_GROUP}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" --log-path "${CLICKHOUSE_LOGDIR}" --data-path "${CLICKHOUSE_DATADIR}"
fi

4
docker/packager/deb/Dockerfile
View File

@ -31,14 +31,10 @@ RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb \
&& chmod +x dpkg-deb \
&& cp dpkg-deb /usr/bin
ENV APACHE_PUBKEY_HASH="bba6987b63c63f710fd4ed476121c588bc3812e99659d27a855f8c4d312783ee66ad6adfce238765691b04d62fa3688f"
RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
&& wget -nv -O /tmp/arrow-keyring.deb "https://apache.bintray.com/arrow/ubuntu/apache-arrow-archive-keyring-latest-${CODENAME}.deb" \
&& echo "${APACHE_PUBKEY_HASH} /tmp/arrow-keyring.deb" | sha384sum -c \
&& dpkg -i /tmp/arrow-keyring.deb
# Libraries from OS are only needed to test the "unbundled" build (this is not used in production).
RUN apt-get update \
&& apt-get install \

2
docker/test/fasttest/run.sh
View File

@ -219,6 +219,8 @@ TESTS_TO_SKIP=(
01268_dictionary_direct_layout
01280_ssd_complex_key_dictionary
01281_group_by_limit_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently
01318_encrypt # Depends on OpenSSL
01318_decrypt # Depends on OpenSSL
01281_unsucceeded_insert_select_queries_counter
01292_create_user
01294_lazy_database_concurrent

7
docker/test/performance-comparison/README.md
View File

@ -48,12 +48,13 @@ This table shows queries that take significantly longer to process on the client
#### Unexpected Query Duration
Action required for every item -- these are errors that must be fixed.
Queries that have "short" duration (on the order of 0.1 s) can't be reliably tested in a normal way, where we perform a small (about ten) measurements for each server, because the signal-to-noise ratio is much smaller. There is a special mode for such queries that instead runs them for a fixed amount of time, normally with much higher number of measurements (up to thousands). This mode must be explicitly enabled by the test author to avoid accidental errors. It must be used only for queries that are meant to complete "immediately", such as `select count(*)`. If your query is not supposed to be "immediate", try to make it run longer, by e.g. processing more data.
A query is supposed to run longer than 0.1 second. If your query runs faster, increase the amount of processed data to bring the run time above this threshold. You can use a bigger table (e.g. `hits_100m` instead of `hits_10m`), increase a `LIMIT`, make a query single-threaded, and so on. Queries that are too fast suffer from poor stability and precision.
This table shows queries for which the "short" marking is not consistent with the actual query run time -- i.e., a query runs for a long time but is marked as short, or it runs very fast but is not marked as short.
Sometimes you want to test a query that is supposed to complete "instantaneously", i.e. in sublinear time. This might be `count(*)`, or parsing a complicated tuple. It might not be practical or even possible to increase the run time of such queries by adding more data. For such queries there is a specal comparison mode which runs them for a fixed amount of time, instead of a fixed number of iterations like we do normally. This mode is inferior to the normal mode, because the influence of noise and overhead is higher, which leads to less precise and stable results.
If your query is really supposed to complete "immediately" and can't be made to run longer, you have to mark it as "short". To do so, write `<query short="1">...` in the test file. The value of "short" attribute is evaluated as a python expression, and substitutions are performed, so you can write something like `<query short="{column1} = {column2}">select count(*) from table where {column1} > {column2}</query>`, to mark only a particular combination of variables as short.
If it is impossible to increase the run time of a query and it is supposed to complete "immediately", you have to explicitly mark this in the test. To do so, add a `short` attribute to the query tag in the test file: `<query short="1">...`. The value of the `short` attribute is evaluated as a python expression, and substitutions are performed, so you can write something like `<query short="{column1} = {column2}">select count(*) from table where {column1} > {column2}</query>`, to mark only a particular combination of variables as short.
This table shows queries for which the `short` marking is not consistent with the actual query run time -- i.e., a query runs for a normal time but is marked as `short`, or it runs faster than normal but is not marked as `short`.
#### Partial Queries
Action required for the cells marked in red.

12
docker/test/performance-comparison/report.py
View File

@ -469,12 +469,12 @@ if args.report == 'main':
columns = [
'Test', #0
'Wall clock time,&nbsp;s', #1
'Total client time,&nbsp;s', #2
'Total queries', #3
'Longest query<br>(sum for all runs),&nbsp;s', #4
'Avg wall clock time<br>(sum for all runs),&nbsp;s', #5
'Shortest query<br>(sum for all runs),&nbsp;s', #6
'Wall clock time, entire test,&nbsp;s', #1
'Total client time for measured query runs,&nbsp;s', #2
'Queries', #3
'Longest query, total for measured runs,&nbsp;s', #4
'Wall clock time per query,&nbsp;s', #5
'Shortest query, total for measured runs,&nbsp;s', #6
'', # Runs #7
]
attrs = ['' for c in columns]

3
docker/test/stateless/Dockerfile
View File

@ -33,5 +33,8 @@ RUN mkdir -p /tmp/clickhouse-odbc-tmp \
ENV TZ=Europe/Moscow
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
ENV NUM_TRIES=1
ENV MAX_RUN_TIME=0
COPY run.sh /
CMD ["/bin/bash", "/run.sh"]

15
docker/test/stateless/run.sh
View File

@ -1,6 +1,7 @@
#!/bin/bash
set -e -x
# fail on errors, verbose and export all env variables
set -e -x -a
dpkg -i package_folder/clickhouse-common-static_*.deb
dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
@ -16,9 +17,17 @@ service clickhouse-server start && sleep 5
if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then
SKIP_LIST_OPT="--use-skip-list"
fi
# We can have several additional options so we path them as array because it's
# more idiologically correct.
read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}"
clickhouse-test --testname --shard --zookeeper --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
function run_tests()
{
for i in $(seq 1 $NUM_TRIES); do
clickhouse-test --testname --shard --zookeeper --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a test_output/test_result.txt
done
}
export -f run_tests
timeout $MAX_RUN_TIME bash -c run_tests ||:

1
docs/en/introduction/adopters.md
View File

@ -88,6 +88,7 @@ toc_title: Adopters
| <a href="https://smi2.ru/" class="favicon">SMI2</a> | News | Analytics | — | — | [Blog Post in Russian, November 2017](https://habr.com/ru/company/smi2/blog/314558/) |
| <a href="https://www.splunk.com/" class="favicon">Splunk</a> | Business Analytics | Main product | — | — | [Slides in English, January 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) |
| <a href="https://www.spotify.com" class="favicon">Spotify</a> | Music | Experimentation | — | — | [Slides, July 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) |
| <a href="https://www.staffcop.ru/" class="favicon">Staffcop</a> | Information Security | Main Product | — | — | [Official website, Documentation](https://www.staffcop.ru/sce43) |
| <a href="https://www.tencent.com" class="favicon">Tencent</a> | Big Data | Data processing | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) |
| <a href="https://www.tencent.com" class="favicon">Tencent</a> | Messaging | Logging | — | — | [Talk in Chinese, November 2019](https://youtu.be/T-iVQRuw-QY?t=5050) |
| <a href="https://trafficstars.com/" class="favicon">Traffic Stars</a> | AD network | — | — | — | [Slides in Russian, May 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) |

19
docs/en/operations/settings/settings.md
View File

@ -305,6 +305,10 @@ When enabled, replace empty input fields in TSV with default values. For complex
Disabled by default.
## input_format_tsv_enum_as_number {#settings-input_format_tsv_enum_as_number}
For TSV input format switches to parsing enum values as enum ids.
## input_format_null_as_default {#settings-input-format-null-as-default}
Enables or disables using default values if input data contain `NULL`, but the data type of the corresponding column in not `Nullable(T)` (for text input formats).
@ -1161,6 +1165,10 @@ The character is interpreted as a delimiter in the CSV data. By default, the del
For CSV input format enables or disables parsing of unquoted `NULL` as literal (synonym for `\N`).
## input_format_csv_enum_as_number {#settings-input_format_csv_enum_as_number}
For CSV input format switches to parsing enum values as enum ids.
## output_format_csv_crlf_end_of_line {#settings-output-format-csv-crlf-end-of-line}
Use DOS/Windows-style line separator (CRLF) in CSV instead of Unix style (LF).
@ -1398,6 +1406,17 @@ Possible values:
Default value: 0
## allow_nondeterministic_optimize_skip_unused_shards {#allow-nondeterministic-optimize-skip-unused-shards}
Allow nondeterministic (like `rand` or `dictGet`, since later has some caveats with updates) functions in sharding key.
Possible values:
- 0 — Disallowed.
- 1 — Allowed.
Default value: 0
## optimize_skip_unused_shards_nesting {#optimize-skip-unused-shards-nesting}
Controls [`optimize_skip_unused_shards`](#optimize-skip-unused-shards) (hence still requires [`optimize_skip_unused_shards`](#optimize-skip-unused-shards)) depends on the nesting level of the distributed query (case when you have `Distributed` table that look into another `Distributed` table).

1
docs/ru/operations/system-tables/query_log.md
View File

@ -34,6 +34,7 @@ ClickHouse не удаляет данные из таблица автомати
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата начала запроса.
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала запроса.
- `query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала обработки запроса.
- `query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время начала обработки запроса с точностью до микросекунд.
- `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — длительность выполнения запроса в миллисекундах.
- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Общее количество строк, считанных из всех таблиц и табличных функций, участвующих в запросе. Включает в себя обычные подзапросы, подзапросы для `IN` и `JOIN`. Для распределенных запросов `read_rows` включает в себя общее количество строк, прочитанных на всех репликах. Каждая реплика передает собственное значение `read_rows`, а сервер-инициатор запроса суммирует все полученные и локальные значения. Объемы кэша не учитываюся.
- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Общее количество байтов, считанных из всех таблиц и табличных функций, участвующих в запросе. Включает в себя обычные подзапросы, подзапросы для `IN` и `JOIN`. Для распределенных запросов `read_bytes` включает в себя общее количество байтов, прочитанных на всех репликах. Каждая реплика передает собственное значение `read_bytes`, а сервер-инициатор запроса суммирует все полученные и локальные значения. Объемы кэша не учитываюся.

1
docs/ru/operations/system-tables/query_thread_log.md
View File

@ -16,6 +16,7 @@ ClickHouse не удаляет данные из таблицы автомати
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата завершения выполнения запроса потоком.
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — дата и время завершения выполнения запроса потоком.
- `query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала обработки запроса.
- `query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время начала обработки запроса с точностью до микросекунд.
- `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — длительность обработки запроса в миллисекундах.
- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — количество прочитанных строк.
- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — количество прочитанных байтов.

2
docs/tools/requirements.txt
View File

@ -14,7 +14,7 @@ Jinja2==2.11.2
jinja2-highlight==0.6.1
jsmin==2.2.2
livereload==2.6.2
Markdown==3.2.1
Markdown==3.3.2
MarkupSafe==1.1.1
mkdocs==1.1.2
mkdocs-htmlproofer-plugin==0.0.3

20
programs/install/Install.cpp
View File

@ -548,11 +548,27 @@ int mainEntryClickHouseInstall(int argc, char ** argv)
users_config_file.string(), users_d.string());
}
/// Set capabilities for the binary.
/** Set capabilities for the binary.
*
* 1. Check that "setcap" tool exists.
* 2. Check that an arbitrary program with installed capabilities can run.
* 3. Set the capabilities.
*
* The second is important for Docker and systemd-nspawn.
* When the container has no capabilities,
* but the executable file inside the container has capabilities,
* then attempt to run this file will end up with a cryptic "Operation not permitted" message.
*/
#if defined(__linux__)
fmt::print("Setting capabilities for clickhouse binary. This is optional.\n");
std::string command = fmt::format("command -v setcap && setcap 'cap_net_admin,cap_ipc_lock,cap_sys_nice+ep' {}", main_bin_path.string());
std::string command = fmt::format("command -v setcap >/dev/null"
" && echo > {0} && chmod a+x {0} && {0} && setcap 'cap_net_admin,cap_ipc_lock,cap_sys_nice+ep' {0} && {0} && rm {0}"
" && setcap 'cap_net_admin,cap_ipc_lock,cap_sys_nice+ep' {1}"
" || echo \"Cannot set 'net_admin' or 'ipc_lock' or 'sys_nice' capability for clickhouse binary."
" This is optional. Taskstats accounting will be disabled."
" To enable taskstats accounting you may add the required capability later manually.\"",
"/tmp/test_setcap.sh", main_bin_path.string());
fmt::print(" {}\n", command);
executeScript(command);
#endif

22
programs/server/CMakeLists.txt
View File

@ -4,7 +4,7 @@ set(CLICKHOUSE_SERVER_SOURCES
)
if (OS_LINUX)
set (LINK_CONFIG_LIB INTERFACE "-Wl,${WHOLE_ARCHIVE} $<TARGET_FILE:clickhouse_server_configs> -Wl,${NO_WHOLE_ARCHIVE}")
set (LINK_RESOURCE_LIB INTERFACE "-Wl,${WHOLE_ARCHIVE} $<TARGET_FILE:clickhouse_server_configs> -Wl,${NO_WHOLE_ARCHIVE}")
endif ()
set (CLICKHOUSE_SERVER_LINK
@ -20,7 +20,7 @@ set (CLICKHOUSE_SERVER_LINK
clickhouse_table_functions
string_utils
${LINK_CONFIG_LIB}
${LINK_RESOURCE_LIB}
PUBLIC
daemon
@ -37,20 +37,20 @@ if (OS_LINUX)
# 1. Allow to run the binary without download of any other files.
# 2. Allow to implement "sudo clickhouse install" tool.
foreach(CONFIG_FILE config users embedded)
set(CONFIG_OBJ ${CONFIG_FILE}.o)
set(CONFIG_OBJS ${CONFIG_OBJS} ${CONFIG_OBJ})
foreach(RESOURCE_FILE config.xml users.xml embedded.xml play.html)
set(RESOURCE_OBJ ${RESOURCE_FILE}.o)
set(RESOURCE_OBJS ${RESOURCE_OBJS} ${RESOURCE_OBJ})
# https://stackoverflow.com/questions/14776463/compile-and-add-an-object-file-from-a-binary-with-cmake
add_custom_command(OUTPUT ${CONFIG_OBJ}
COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${OBJCOPY_PATH} -I binary ${OBJCOPY_ARCH_OPTIONS} ${CONFIG_FILE}.xml ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_OBJ}
add_custom_command(OUTPUT ${RESOURCE_OBJ}
COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${OBJCOPY_PATH} -I binary ${OBJCOPY_ARCH_OPTIONS} ${RESOURCE_FILE} ${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ}
COMMAND ${OBJCOPY_PATH} --rename-section .data=.rodata,alloc,load,readonly,data,contents
${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_OBJ} ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_OBJ})
${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ} ${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ})
set_source_files_properties(${CONFIG_OBJ} PROPERTIES EXTERNAL_OBJECT true GENERATED true)
endforeach(CONFIG_FILE)
set_source_files_properties(${RESOURCE_OBJ} PROPERTIES EXTERNAL_OBJECT true GENERATED true)
endforeach(RESOURCE_FILE)
add_library(clickhouse_server_configs STATIC ${CONFIG_OBJS})
add_library(clickhouse_server_configs STATIC ${RESOURCE_OBJS})
set_target_properties(clickhouse_server_configs PROPERTIES LINKER_LANGUAGE C)
# whole-archive prevents symbols from being discarded for unknown reason

64
programs/server/config.xml
View File

@ -212,22 +212,10 @@
<!-- Directory with user provided files that are accessible by 'file' table function. -->
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
<!-- Sources to read users, roles, access rights, profiles of settings, quotas. -->
<user_directories>
<users_xml>
<!-- Path to configuration file with predefined users. -->
<path>users.xml</path>
</users_xml>
<local_directory>
<!-- Path to folder where users created by SQL commands are stored. -->
<path>/var/lib/clickhouse/access/</path>
</local_directory>
</user_directories>
<!-- External user directories (LDAP). -->
<!-- LDAP server definitions. -->
<ldap_servers>
<!-- List LDAP servers with their connection parameters here to later use them as authenticators for dedicated users,
who have 'ldap' authentication mechanism specified instead of 'password'.
<!-- List LDAP servers with their connection parameters here to later 1) use them as authenticators for dedicated local users,
who have 'ldap' authentication mechanism specified instead of 'password', or to 2) use them as remote user directories.
Parameters:
host - LDAP server hostname or IP, this parameter is mandatory and cannot be empty.
port - LDAP server port, default is 636 if enable_tls is set to true, 389 otherwise.
@ -246,7 +234,7 @@
tls_key_file - path to certificate key file.
tls_ca_cert_file - path to CA certificate file.
tls_ca_cert_dir - path to the directory containing CA certificates.
tls_cipher_suite - allowed cipher suite.
tls_cipher_suite - allowed cipher suite (in OpenSSL notation).
Example:
<my_ldap_server>
<host>localhost</host>
@ -265,6 +253,36 @@
-->
</ldap_servers>
<!-- Sources to read users, roles, access rights, profiles of settings, quotas. -->
<user_directories>
<users_xml>
<!-- Path to configuration file with predefined users. -->
<path>users.xml</path>
</users_xml>
<local_directory>
<!-- Path to folder where users created by SQL commands are stored. -->
<path>/var/lib/clickhouse/access/</path>
</local_directory>
<!-- To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section
with the following parameters:
server - one of LDAP server names defined in 'ldap_servers' config section above.
This parameter is mandatory and cannot be empty.
roles - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server.
If no roles are specified, user will not be able to perform any actions after authentication.
If any of the listed roles is not defined locally at the time of authentication, the authenthication attept
will fail as if the provided password was incorrect.
Example:
<ldap>
<server>my_ldap_server</server>
<roles>
<my_local_role1 />
<my_local_role2 />
</roles>
</ldap>
-->
</user_directories>
<!-- Default profile of settings. -->
<default_profile>default</default_profile>
@ -729,18 +747,22 @@
-->
<format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
<!-- Uncomment to use query masking rules.
<!-- Default query masking rules, matching lines would be replaced with something else in the logs
(both text logs and system.query_log).
name - name for the rule (optional)
regexp - RE2 compatible regular expression (mandatory)
replace - substitution string for sensitive data (optional, by default - six asterisks)
-->
<query_masking_rules>
<rule>
<name>hide SSN</name>
<regexp>\b\d{3}-\d{2}-\d{4}\b</regexp>
<replace>000-00-0000</replace>
<name>hide encrypt/decrypt arguments</name>
<regexp>((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\)</regexp>
<!-- or more secure, but also more invasive:
(aes_\w+)\s*\(.*\)
-->
<replace>\1(???)</replace>
</rule>
</query_masking_rules>
-->
<!-- Uncomment to use custom http handlers.
rules are checked from top to bottom, first match runs the handler

448
programs/server/play.html Normal file
View File

@ -0,0 +1,448 @@
<html> <!-- TODO If I write DOCTYPE HTML something changes but I don't know what. -->
<head>
<meta charset="UTF-8">
<title>ClickHouse Query</title>
<!-- Code style:
Do not use any JavaScript or CSS frameworks or preprocessors.
This HTML page should not require any build systems (node.js, npm, gulp, etc.)
This HTML page should not be minified, instead it should be reasonably minimalistic by itself.
This HTML page should not load any external resources
(CSS and JavaScript must be embedded directly to the page. No external fonts or images should be loaded).
This UI should look as lightweight, clean and fast as possible.
All UI elements must be aligned in pixel-perfect way.
There should not be any animations.
No unexpected changes in positions of elements while the page is loading.
Navigation by keyboard should work.
64-bit numbers must display correctly.
-->
<style type="text/css">
:root {
--background-color: #DDF8FF; /* Or #FFFBEF; actually many pastel colors look great for light theme. */
--element-background-color: #FFF;
--border-color: #EEE;
--shadow-color: rgba(0, 0, 0, 0.1);
--button-color: #FFAA00; /* Orange on light-cyan is especially good. */
--text-color: #000;
--button-active-color: #F00;
--button-active-text-color: #FFF;
--misc-text-color: #888;
--error-color: #FEE; /* Light-pink on light-cyan is so neat, I even want to trigger errors to see this cool combination of colors. */
--table-header-color: #F8F8F8;
--table-hover-color: #FFF8EF;
--null-color: #A88;
}
[data-theme="dark"] {
--background-color: #000;
--element-background-color: #102030;
--border-color: #111;
--shadow-color: rgba(255, 255, 255, 0.1);
--text-color: #CCC;
--button-color: #FFAA00;
--button-text-color: #000;
--button-active-color: #F00;
--button-active-text-color: #FFF;
--misc-text-color: #888;
--error-color: #400;
--table-header-color: #102020;
--table-hover-color: #003333;
--null-color: #A88;
}
html, body
{
/* Personal choice. */
font-family: Sans-Serif;
background: var(--background-color);
color: var(--text-color);
}
/* Otherwise Webkit based browsers will display ugly border on focus. */
textarea, input, button
{
outline: none;
border: none;
color: var(--text-color);
}
/* Otherwise scrollbar may appear dynamically and it will alter viewport height,
then relative heights of elements will change suddenly, and it will break overall impression. */
/* html
{
overflow-x: scroll;
}*/
div
{
width: 100%;
}
.monospace
{
/* Prefer fonts that have full hinting info. This is important for non-retina displays.
Also I personally dislike "Ubuntu" font due to the similarity of 'r' and 'г' (it looks very ignorant).
*/
font-family: Liberation Mono, DejaVu Sans Mono, MonoLisa, Consolas, Monospace;
}
.shadow
{
box-shadow: 0 0 1rem var(--shadow-color);
}
input, textarea
{
border: 1px solid var(--border-color);
/* The font must be not too small (to be inclusive) and not too large (it's less practical and make general feel of insecurity) */
font-size: 11pt;
padding: 0.25rem;
background-color: var(--element-background-color);
}
#query
{
/* Make enough space for even huge queries. */
height: 20%;
width: 100%;
}
#inputs
{
white-space: nowrap;
width: 100%;
}
#url
{
width: 70%;
}
#user
{
width: 15%;
}
#password
{
width: 15%;
}
#run_div
{
margin-top: 1rem;
}
#run
{
color: var(--button-text-color);
background-color: var(--button-color);
padding: 0.25rem 1rem;
cursor: pointer;
font-weight: bold;
font-size: 100%; /* Otherwise button element will have lower font size. */
}
#run:hover, #run:focus
{
color: var(--button-active-text-color);
background-color: var(--button-active-color);
}
#stats
{
float: right;
color: var(--misc-text-color);
}
#toggle-light, #toggle-dark
{
float: right;
padding-right: 0.5rem;
cursor: pointer;
}
.hint
{
color: var(--misc-text-color);
}
#data_div
{
margin-top: 1rem;
}
#data-table
{
border-collapse: collapse;
border-spacing: 0;
/* I need pixel-perfect alignment but not sure the following is correct, please help */
min-width: calc(100vw - 2rem);
}
/* Will be displayed when user specified custom format. */
#data-unparsed
{
background-color: var(--element-background-color);
margin-top: 0rem;
padding: 0.25rem 0.5rem;
display: none;
}
td
{
background-color: var(--element-background-color);
white-space: nowrap;
/* For wide tables any individual column will be no more than 50% of page width. */
max-width: 50vw;
/* The content is cut unless you hover. */
overflow: hidden;
padding: 0.25rem 0.5rem;
border: 1px solid var(--border-color);
white-space: pre;
}
td.right
{
text-align: right;
}
th
{
padding: 0.25rem 0.5rem;
text-align: middle;
background-color: var(--table-header-color);
border: 1px solid var(--border-color);
}
/* The row under mouse pointer is highlight for better legibility. */
tr:hover, tr:hover td
{
background-color: var(--table-hover-color);
}
tr:hover
{
box-shadow: 0 0 1rem rgba(0, 0, 0, 0.1);
}
#error
{
background: var(--error-color);
white-space: pre-wrap;
padding: 0.5rem 1rem;
display: none;
}
/* When mouse pointer is over table cell, will display full text (with wrap) instead of cut.
TODO Find a way to make it work on touch devices. */
td.left:hover
{
white-space: pre-wrap;
}
/* The style for SQL NULL */
.null
{
color: var(--null-color);
}
</style>
</head>
<body>
<div id="inputs">
<input class="monospace shadow" id="url" type="text" value="http://localhost:8123/" /><input class="monospace shadow" id="user" type="text" value="default" /><input class="monospace shadow" id="password" type="password" />
</div>
<div>
<textarea autofocus spellcheck="false" class="monospace shadow" id="query"></textarea>
</div>
<div id="run_div">
<button class="shadow" id="run">Run</button>
<span class="hint">&nbsp;(Ctrl+Enter)</span>
<span id="stats"></span>
<span id="toggle-dark">🌑</span><span id="toggle-light">🌞</span>
</div>
<div id="data_div">
<table class="monospace shadow" id="data-table"></table>
<pre class="monospace shadow" id="data-unparsed"></pre>
</div>
<p id="error" class="monospace shadow">
</p>
</body>
<script type="text/javascript">
/// Substitute the address of the server where the page is served.
if (location.protocol != 'file:') {
document.getElementById('url').value = location.origin;
}
function post()
{
/// TODO: Avoid race condition on subsequent requests when responses may come out of order.
/// TODO: Check if URL already contains query string (append parameters).
var url = document.getElementById('url').value +
/// Ask server to allow cross-domain requests.
'?add_http_cors_header=1' +
'&user=' + encodeURIComponent(document.getElementById('user').value) +
'&password=' + encodeURIComponent(document.getElementById('password').value) +
'&default_format=JSONCompact' +
/// Safety settings to prevent results that browser cannot display.
'&max_result_rows=1000&max_result_bytes=10000000&result_overflow_mode=break';
var query = document.getElementById('query').value;
var xhr = new XMLHttpRequest;
xhr.open('POST', url, true);
xhr.send(query);
xhr.onreadystatechange = function()
{
if (this.readyState === XMLHttpRequest.DONE) {
if (this.status === 200) {
var json;
try { json = JSON.parse(this.response); } catch (e) {}
if (json !== undefined && json.statistics !== undefined) {
renderResult(json);
} else {
renderUnparsedResult(this.response);
}
} else {
/// TODO: Proper rendering of network errors.
renderError(this.response);
}
} else {
//console.log(this);
}
}
}
document.getElementById('run').onclick = function()
{
post();
}
document.getElementById('query').onkeypress = function(event)
{
/// Firefox has code 13 for Enter and Chromium has code 10.
if (event.ctrlKey && (event.charCode == 13 || event.charCode == 10)) {
post();
}
}
function clear()
{
var table = document.getElementById('data-table');
while (table.firstChild) {
table.removeChild(table.lastChild);
}
document.getElementById('data-unparsed').innerText = '';
document.getElementById('data-unparsed').style.display = 'none';
document.getElementById('error').innerText = '';
document.getElementById('error').style.display = 'none';
document.getElementById('stats').innerText = '';
}
function renderResult(response)
{
//console.log(response);
clear();
var stats = document.getElementById('stats');
stats.innerText = 'Elapsed: ' + response.statistics.elapsed.toFixed(3) + " sec, read " + response.statistics.rows_read + " rows.";
var thead = document.createElement('thead');
for (var idx in response.meta) {
var th = document.createElement('th');
var name = document.createTextNode(response.meta[idx].name);
th.appendChild(name);
thead.appendChild(th);
}
/// To prevent hanging the browser, limit the number of cells in a table.
/// It's important to have the limit on number of cells, not just rows, because tables may be wide or narrow.
var max_rows = 10000 / response.meta.length;
var row_num = 0;
var tbody = document.createElement('tbody');
for (var row_idx in response.data) {
var tr = document.createElement('tr');
for (var col_idx in response.data[row_idx]) {
var td = document.createElement('td');
var cell = response.data[row_idx][col_idx];
var is_null = (cell === null);
var content = document.createTextNode(is_null ? 'ᴺᵁᴸᴸ' : cell);
td.appendChild(content);
/// TODO: Execute regexp only once for each column.
td.className = response.meta[col_idx].type.match(/^(U?Int|Decimal|Float)/) ? 'right' : 'left';
if (is_null) {
td.className += ' null';
}
tr.appendChild(td);
}
tbody.appendChild(tr);
++row_num;
if (row_num >= max_rows) {
break;
}
}
var table = document.getElementById('data-table');
table.appendChild(thead);
table.appendChild(tbody);
}
/// A function to render raw data when non-default format is specified.
function renderUnparsedResult(response)
{
clear();
var data = document.getElementById('data-unparsed')
if (response === '') {
/// TODO: Fade or remove previous result when new request will be performed.
response = 'Ok.';
}
data.innerText = response;
/// inline-block make width adjust to the size of content.
data.style.display = 'inline-block';
}
function renderError(response)
{
clear();
document.getElementById('error').innerText = response;
document.getElementById('error').style.display = 'block';
}
function setColorTheme(theme)
{
window.localStorage.setItem('theme', theme);
document.documentElement.setAttribute('data-theme', theme);
}
/// The choice of color theme is saved in browser.
var theme = window.localStorage.getItem('theme');
if (theme) {
setColorTheme(theme);
}
document.getElementById('toggle-light').onclick = function()
{
setColorTheme('light');
}
document.getElementById('toggle-dark').onclick = function()
{
setColorTheme('dark');
}
</script>
</html>

15
src/Access/AccessControlManager.cpp
View File

@ -3,6 +3,7 @@
#include <Access/MemoryAccessStorage.h>
#include <Access/UsersConfigAccessStorage.h>
#include <Access/DiskAccessStorage.h>
#include <Access/LDAPAccessStorage.h>
#include <Access/ContextAccess.h>
#include <Access/RoleCache.h>
#include <Access/RowPolicyCache.h>
@ -253,6 +254,12 @@ void AccessControlManager::addMemoryStorage(const String & storage_name_)
}
void AccessControlManager::addLDAPStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_)
{
addStorage(std::make_shared<LDAPAccessStorage>(storage_name_, this, config_, prefix_));
}
void AccessControlManager::addStoragesFromUserDirectoriesConfig(
const Poco::Util::AbstractConfiguration & config,
const String & key,
@ -275,6 +282,8 @@ void AccessControlManager::addStoragesFromUserDirectoriesConfig(
type = UsersConfigAccessStorage::STORAGE_TYPE;
else if ((type == "local") || (type == "local_directory"))
type = DiskAccessStorage::STORAGE_TYPE;
else if (type == "ldap")
type = LDAPAccessStorage::STORAGE_TYPE;
String name = config.getString(prefix + ".name", type);
@ -295,6 +304,10 @@ void AccessControlManager::addStoragesFromUserDirectoriesConfig(
bool readonly = config.getBool(prefix + ".readonly", false);
addDiskStorage(name, path, readonly);
}
else if (type == LDAPAccessStorage::STORAGE_TYPE)
{
addLDAPStorage(name, config, prefix);
}
else
throw Exception("Unknown storage type '" + type + "' at " + prefix + " in config", ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
}
@ -346,7 +359,7 @@ UUID AccessControlManager::login(const String & user_name, const String & passwo
void AccessControlManager::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config)
{
external_authenticators->setConfig(config, getLogger());
external_authenticators->setConfiguration(config, getLogger());
}

3
src/Access/AccessControlManager.h
View File

@ -82,6 +82,9 @@ public:
void addMemoryStorage();
void addMemoryStorage(const String & storage_name_);
/// Adds LDAPAccessStorage which allows querying remote LDAP server for user info.
void addLDAPStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_);
/// Adds storages from <users_directories> config.
void addStoragesFromUserDirectoriesConfig(const Poco::Util::AbstractConfiguration & config,
const String & key,

2
src/Access/ExternalAuthenticators.cpp
View File

@ -156,7 +156,7 @@ void ExternalAuthenticators::reset()
ldap_server_params.clear();
}
void ExternalAuthenticators::setConfig(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log)
void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log)
{
std::scoped_lock lock(mutex);
reset();

2
src/Access/ExternalAuthenticators.h
View File

@ -26,7 +26,7 @@ class ExternalAuthenticators
{
public:
void reset();
void setConfig(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log);
void setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log);
void setLDAPServerParams(const String & server, const LDAPServerParams & params);
LDAPServerParams getLDAPServerParams(const String & server) const;

34
src/Access/IAccessStorage.cpp
View File

@ -14,6 +14,8 @@ namespace ErrorCodes
extern const int ACCESS_ENTITY_ALREADY_EXISTS;
extern const int ACCESS_ENTITY_NOT_FOUND;
extern const int ACCESS_STORAGE_READONLY;
extern const int WRONG_PASSWORD;
extern const int IP_ADDRESS_NOT_ALLOWED;
extern const int AUTHENTICATION_FAILED;
extern const int LOGICAL_ERROR;
}
@ -418,9 +420,21 @@ UUID IAccessStorage::login(
const String & user_name,
const String & password,
const Poco::Net::IPAddress & address,
const ExternalAuthenticators & external_authenticators) const
const ExternalAuthenticators & external_authenticators,
bool replace_exception_with_cannot_authenticate) const
{
try
{
return loginImpl(user_name, password, address, external_authenticators);
}
catch (...)
{
if (!replace_exception_with_cannot_authenticate)
throw;
tryLogCurrentException(getLogger(), user_name + ": Authentication failed");
throwCannotAuthenticate(user_name);
}
}
@ -434,11 +448,16 @@ UUID IAccessStorage::loginImpl(
{
if (auto user = tryRead<User>(*id))
{
if (isPasswordCorrectImpl(*user, password, external_authenticators) && isAddressAllowedImpl(*user, address))
if (!isPasswordCorrectImpl(*user, password, external_authenticators))
throwInvalidPassword();
if (!isAddressAllowedImpl(*user, address))
throwAddressNotAllowed(address);
return *id;
}
}
throwCannotAuthenticate(user_name);
throwNotFound(EntityType::USER, user_name);
}
@ -554,6 +573,15 @@ void IAccessStorage::throwReadonlyCannotRemove(EntityType type, const String & n
ErrorCodes::ACCESS_STORAGE_READONLY);
}
void IAccessStorage::throwAddressNotAllowed(const Poco::Net::IPAddress & address)
{
throw Exception("Connections from " + address.toString() + " are not allowed", ErrorCodes::IP_ADDRESS_NOT_ALLOWED);
}
void IAccessStorage::throwInvalidPassword()
{
throw Exception("Invalid password", ErrorCodes::WRONG_PASSWORD);
}
void IAccessStorage::throwCannotAuthenticate(const String & user_name)
{

4
src/Access/IAccessStorage.h
View File

@ -144,7 +144,7 @@ public:
/// Finds an user, check its password and returns the ID of the user.
/// Throws an exception if no such user or password is incorrect.
UUID login(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const;
UUID login(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool replace_exception_with_cannot_authenticate = true) const;
/// Returns the ID of an user who has logged in (maybe on another node).
/// The function assumes that the password has been already checked somehow, so we can skip checking it now.
@ -182,6 +182,8 @@ protected:
[[noreturn]] void throwReadonlyCannotInsert(EntityType type, const String & name) const;
[[noreturn]] void throwReadonlyCannotUpdate(EntityType type, const String & name) const;
[[noreturn]] void throwReadonlyCannotRemove(EntityType type, const String & name) const;
[[noreturn]] static void throwAddressNotAllowed(const Poco::Net::IPAddress & address);
[[noreturn]] static void throwInvalidPassword();
[[noreturn]] static void throwCannotAuthenticate(const String & user_name);
using Notification = std::tuple<OnChangedHandler, UUID, AccessEntityPtr>;

313
src/Access/LDAPAccessStorage.cpp Normal file
View File

@ -0,0 +1,313 @@
#include <Access/LDAPAccessStorage.h>
#include <Access/AccessControlManager.h>
#include <Access/User.h>
#include <Access/Role.h>
#include <Common/Exception.h>
#include <common/logger_useful.h>
#include <ext/scope_guard.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <Poco/JSON/JSON.h>
#include <Poco/JSON/Object.h>
#include <Poco/JSON/Stringifier.h>
#include <boost/range/algorithm/copy.hpp>
#include <iterator>
#include <sstream>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_, AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix)
: IAccessStorage(storage_name_)
{
setConfiguration(access_control_manager_, config, prefix);
}
void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix)
{
std::scoped_lock lock(mutex);
// TODO: switch to passing config as a ConfigurationView and remove this extra prefix once a version of Poco with proper implementation is available.
const String prefix_str = (prefix.empty() ? "" : prefix + ".");
const bool has_server = config.has(prefix_str + "server");
const bool has_roles = config.has(prefix_str + "roles");
if (!has_server)
throw Exception("Missing 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS);
const auto ldap_server_cfg = config.getString(prefix_str + "server");
if (ldap_server_cfg.empty())
throw Exception("Empty 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS);
std::set<String> roles_cfg;
if (has_roles)
{
Poco::Util::AbstractConfiguration::Keys role_names;
config.keys(prefix_str + "roles", role_names);
// Currently, we only extract names of roles from the section names and assign them directly and unconditionally.
roles_cfg.insert(role_names.begin(), role_names.end());
}
access_control_manager = access_control_manager_;
ldap_server = ldap_server_cfg;
default_role_names.swap(roles_cfg);
roles_of_interest.clear();
role_change_subscription = access_control_manager->subscribeForChanges<Role>(
[this] (const UUID & id, const AccessEntityPtr & entity)
{
return this->processRoleChange(id, entity);
}
);
/// Update `roles_of_interests` with initial values.
for (const auto & role_name : default_role_names)
{
if (auto role_id = access_control_manager->find<Role>(role_name))
roles_of_interest.emplace(*role_id, role_name);
}
}
void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr & entity)
{
std::scoped_lock lock(mutex);
/// Update `roles_of_interests`.
auto role = typeid_cast<std::shared_ptr<const Role>>(entity);
bool need_to_update_users = false;
if (role && default_role_names.count(role->getName()))
{
/// If a role was created with one of the `default_role_names` or renamed to one of the `default_role_names`,
/// then set `need_to_update_users`.
need_to_update_users = roles_of_interest.insert_or_assign(id, role->getName()).second;
}
else
{
/// If a role was removed or renamed to a name which isn't contained in the `default_role_names`,
/// then set `need_to_update_users`.
need_to_update_users = roles_of_interest.erase(id) > 0;
}
/// Update users which have been created.
if (need_to_update_users)
{
auto update_func = [this] (const AccessEntityPtr & entity_) -> AccessEntityPtr
{
if (auto user = typeid_cast<std::shared_ptr<const User>>(entity_))
{
auto changed_user = typeid_cast<std::shared_ptr<User>>(user->clone());
auto & granted_roles = changed_user->granted_roles.roles;
granted_roles.clear();
boost::range::copy(roles_of_interest | boost::adaptors::map_keys, std::inserter(granted_roles, granted_roles.end()));
return changed_user;
}
return entity_;
};
memory_storage.update(memory_storage.findAll<User>(), update_func);
}
}
void LDAPAccessStorage::checkAllDefaultRoleNamesFoundNoLock() const
{
boost::container::flat_set<std::string_view> role_names_of_interest;
boost::range::copy(roles_of_interest | boost::adaptors::map_values, std::inserter(role_names_of_interest, role_names_of_interest.end()));
for (const auto & role_name : default_role_names)
{
if (!role_names_of_interest.count(role_name))
throwDefaultRoleNotFound(role_name);
}
}
const char * LDAPAccessStorage::getStorageType() const
{
return STORAGE_TYPE;
}
String LDAPAccessStorage::getStorageParamsJSON() const
{
std::scoped_lock lock(mutex);
Poco::JSON::Object params_json;
params_json.set("server", ldap_server);
params_json.set("roles", default_role_names);
std::ostringstream oss;
Poco::JSON::Stringifier::stringify(params_json, oss);
return oss.str();
}
std::optional<UUID> LDAPAccessStorage::findImpl(EntityType type, const String & name) const
{
std::scoped_lock lock(mutex);
return memory_storage.find(type, name);
}
std::vector<UUID> LDAPAccessStorage::findAllImpl(EntityType type) const
{
std::scoped_lock lock(mutex);
return memory_storage.findAll(type);
}
bool LDAPAccessStorage::existsImpl(const UUID & id) const
{
std::scoped_lock lock(mutex);
return memory_storage.exists(id);
}
AccessEntityPtr LDAPAccessStorage::readImpl(const UUID & id) const
{
std::scoped_lock lock(mutex);
return memory_storage.read(id);
}
String LDAPAccessStorage::readNameImpl(const UUID & id) const
{
std::scoped_lock lock(mutex);
return memory_storage.readName(id);
}
bool LDAPAccessStorage::canInsertImpl(const AccessEntityPtr &) const
{
return false;
}
UUID LDAPAccessStorage::insertImpl(const AccessEntityPtr & entity, bool)
{
throwReadonlyCannotInsert(entity->getType(), entity->getName());
}
void LDAPAccessStorage::removeImpl(const UUID & id)
{
std::scoped_lock lock(mutex);
auto entity = read(id);
throwReadonlyCannotRemove(entity->getType(), entity->getName());
}
void LDAPAccessStorage::updateImpl(const UUID & id, const UpdateFunc &)
{
std::scoped_lock lock(mutex);
auto entity = read(id);
throwReadonlyCannotUpdate(entity->getType(), entity->getName());
}
ext::scope_guard LDAPAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const
{
std::scoped_lock lock(mutex);
return memory_storage.subscribeForChanges(id, handler);
}
ext::scope_guard LDAPAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const
{
std::scoped_lock lock(mutex);
return memory_storage.subscribeForChanges(type, handler);
}
bool LDAPAccessStorage::hasSubscriptionImpl(const UUID & id) const
{
std::scoped_lock lock(mutex);
return memory_storage.hasSubscription(id);
}
bool LDAPAccessStorage::hasSubscriptionImpl(EntityType type) const
{
std::scoped_lock lock(mutex);
return memory_storage.hasSubscription(type);
}
UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const
{
std::scoped_lock lock(mutex);
auto id = memory_storage.find<User>(user_name);
if (id)
{
auto user = memory_storage.read<User>(*id);
if (!isPasswordCorrectImpl(*user, password, external_authenticators))
throwInvalidPassword();
if (!isAddressAllowedImpl(*user, address))
throwAddressNotAllowed(address);
return *id;
}
else
{
// User does not exist, so we create one, and will add it if authentication is successful.
auto user = std::make_shared<User>();
user->setName(user_name);
user->authentication = Authentication(Authentication::Type::LDAP_SERVER);
user->authentication.setServerName(ldap_server);
if (!isPasswordCorrectImpl(*user, password, external_authenticators))
throwInvalidPassword();
if (!isAddressAllowedImpl(*user, address))
throwAddressNotAllowed(address);
checkAllDefaultRoleNamesFoundNoLock();
auto & granted_roles = user->granted_roles.roles;
boost::range::copy(roles_of_interest | boost::adaptors::map_keys, std::inserter(granted_roles, granted_roles.end()));
return memory_storage.insert(user);
}
}
UUID LDAPAccessStorage::getIDOfLoggedUserImpl(const String & user_name) const
{
std::scoped_lock lock(mutex);
auto id = memory_storage.find<User>(user_name);
if (id)
{
return *id;
}
else
{
// User does not exist, so we create one, and add it pretending that the authentication is successful.
auto user = std::make_shared<User>();
user->setName(user_name);
user->authentication = Authentication(Authentication::Type::LDAP_SERVER);
user->authentication.setServerName(ldap_server);
checkAllDefaultRoleNamesFoundNoLock();
auto & granted_roles = user->granted_roles.roles;
boost::range::copy(roles_of_interest | boost::adaptors::map_keys, std::inserter(granted_roles, granted_roles.end()));
return memory_storage.insert(user);
}
}
void LDAPAccessStorage::throwDefaultRoleNotFound(const String & role_name)
{
throw Exception("One of the default roles, the role '" + role_name + "', is not found", IAccessEntity::TypeInfo::get(IAccessEntity::Type::ROLE).not_found_error_code);
}
}

71
src/Access/LDAPAccessStorage.h Normal file
View File

@ -0,0 +1,71 @@
#pragma once
#include <Access/MemoryAccessStorage.h>
#include <Core/Types.h>
#include <ext/scope_guard.h>
#include <map>
#include <mutex>
#include <set>
namespace Poco
{
namespace Util
{
class AbstractConfiguration;
}
}
namespace DB
{
class AccessControlManager;
/// Implementation of IAccessStorage which allows attaching users from a remote LDAP server.
/// Currently, any user name will be treated as a name of an existing remote user,
/// a user info entity will be created, with LDAP_SERVER authentication type.
class LDAPAccessStorage : public IAccessStorage
{
public:
static constexpr char STORAGE_TYPE[] = "ldap";
explicit LDAPAccessStorage(const String & storage_name_, AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix);
virtual ~LDAPAccessStorage() override = default;
public: // IAccessStorage implementations.
virtual const char * getStorageType() const override;
virtual String getStorageParamsJSON() const override;
private: // IAccessStorage implementations.
virtual std::optional<UUID> findImpl(EntityType type, const String & name) const override;
virtual std::vector<UUID> findAllImpl(EntityType type) const override;
virtual bool existsImpl(const UUID & id) const override;
virtual AccessEntityPtr readImpl(const UUID & id) const override;
virtual String readNameImpl(const UUID & id) const override;
virtual bool canInsertImpl(const AccessEntityPtr &) const override;
virtual UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override;
virtual void removeImpl(const UUID & id) override;
virtual void updateImpl(const UUID & id, const UpdateFunc & update_func) override;
virtual ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
virtual ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override;
virtual bool hasSubscriptionImpl(const UUID & id) const override;
virtual bool hasSubscriptionImpl(EntityType type) const override;
virtual UUID loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const override;
virtual UUID getIDOfLoggedUserImpl(const String & user_name) const override;
private:
void setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix);
void processRoleChange(const UUID & id, const AccessEntityPtr & entity);
void checkAllDefaultRoleNamesFoundNoLock() const;
[[noreturn]] static void throwDefaultRoleNotFound(const String & role_name);
mutable std::recursive_mutex mutex;
AccessControlManager * access_control_manager = nullptr;
String ldap_server;
std::set<String> default_role_names;
std::map<UUID, String> roles_of_interest;
ext::scope_guard role_change_subscription;
mutable MemoryAccessStorage memory_storage;
};
}

74
src/Access/LDAPClient.cpp
View File

@ -2,6 +2,8 @@
#include <Common/Exception.h>
#include <ext/scope_guard.h>
#include <mutex>
#include <cstring>
#include <sys/time.h>
@ -27,16 +29,13 @@ LDAPClient::~LDAPClient()
closeConnection();
}
void LDAPClient::openConnection()
{
const bool graceful_bind_failure = false;
diag(openConnection(graceful_bind_failure));
}
#if USE_LDAP
namespace
{
std::recursive_mutex ldap_global_mutex;
auto escapeForLDAP(const String & src)
{
String dest;
@ -63,10 +62,13 @@ namespace
return dest;
}
}
void LDAPClient::diag(const int rc)
{
std::scoped_lock lock(ldap_global_mutex);
if (rc != LDAP_SUCCESS)
{
String text;
@ -100,8 +102,10 @@ void LDAPClient::diag(const int rc)
}
}
int LDAPClient::openConnection(const bool graceful_bind_failure)
void LDAPClient::openConnection()
{
std::scoped_lock lock(ldap_global_mutex);
closeConnection();
{
@ -232,8 +236,6 @@ int LDAPClient::openConnection(const bool graceful_bind_failure)
if (params.enable_tls == LDAPServerParams::TLSEnable::YES_STARTTLS)
diag(ldap_start_tls_s(handle, nullptr, nullptr));
int rc = LDAP_OTHER;
switch (params.sasl_mechanism)
{
case LDAPServerParams::SASLMechanism::SIMPLE:
@ -244,20 +246,21 @@ int LDAPClient::openConnection(const bool graceful_bind_failure)
cred.bv_val = const_cast<char *>(params.password.c_str());
cred.bv_len = params.password.size();
rc = ldap_sasl_bind_s(handle, dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr);
if (!graceful_bind_failure)
diag(rc);
diag(ldap_sasl_bind_s(handle, dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr));
break;
}
default:
{
throw Exception("Unknown SASL mechanism", ErrorCodes::LDAP_ERROR);
}
}
return rc;
}
void LDAPClient::closeConnection() noexcept
{
std::scoped_lock lock(ldap_global_mutex);
if (!handle)
return;
@ -267,42 +270,21 @@ void LDAPClient::closeConnection() noexcept
bool LDAPSimpleAuthClient::check()
{
if (params.user.empty())
throw Exception("LDAP authentication of a user with an empty name is not allowed", ErrorCodes::BAD_ARGUMENTS);
std::scoped_lock lock(ldap_global_mutex);
if (params.user.empty())
throw Exception("LDAP authentication of a user with empty name is not allowed", ErrorCodes::BAD_ARGUMENTS);
// Silently reject authentication attempt if the password is empty as if it didn't match.
if (params.password.empty())
return false; // Silently reject authentication attempt if the password is empty as if it didn't match.
return false;
SCOPE_EXIT({ closeConnection(); });
const bool graceful_bind_failure = true;
const auto rc = openConnection(graceful_bind_failure);
// Will throw on any error, including invalid credentials.
openConnection();
bool result = false;
switch (rc)
{
case LDAP_SUCCESS:
{
result = true;
break;
}
case LDAP_INVALID_CREDENTIALS:
{
result = false;
break;
}
default:
{
result = false;
diag(rc);
break;
}
}
return result;
return true;
}
#else // USE_LDAP
@ -312,7 +294,7 @@ void LDAPClient::diag(const int)
throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME);
}
int LDAPClient::openConnection(const bool)
void LDAPClient::openConnection()
{
throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME);
}

1
src/Access/LDAPClient.h
View File

@ -32,7 +32,6 @@ public:
protected:
MAYBE_NORETURN void diag(const int rc);
MAYBE_NORETURN void openConnection();
int openConnection(const bool graceful_bind_failure = false);
void closeConnection() noexcept;
protected:

1
src/Access/LDAPParams.h
View File

@ -42,6 +42,7 @@ struct LDAPServerParams
enum class SASLMechanism
{
UNKNOWN,
SIMPLE
};

2
src/Access/MemoryAccessStorage.cpp
View File

@ -69,7 +69,7 @@ UUID MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool re
UUID id = generateRandomID();
std::lock_guard lock{mutex};
insertNoLock(generateRandomID(), new_entity, replace_if_exists, notifications);
insertNoLock(id, new_entity, replace_if_exists, notifications);
return id;
}

14
src/Access/MultipleAccessStorage.cpp
View File

@ -2,6 +2,7 @@
#include <Common/Exception.h>
#include <ext/range.h>
#include <boost/range/adaptor/map.hpp>
#include <boost/range/adaptor/reversed.hpp>
#include <boost/range/algorithm/copy.hpp>
#include <boost/range/algorithm/find.hpp>
@ -27,6 +28,15 @@ MultipleAccessStorage::MultipleAccessStorage(const String & storage_name_)
{
}
MultipleAccessStorage::~MultipleAccessStorage()
{
/// It's better to remove the storages in the reverse order because they could depend on each other somehow.
const auto storages = getStoragesPtr();
for (const auto & storage : *storages | boost::adaptors::reversed)
{
removeStorage(storage);
}
}
void MultipleAccessStorage::setStorages(const std::vector<StoragePtr> & storages)
{
@ -400,7 +410,7 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p
{
try
{
auto id = storage->login(user_name, password, address, external_authenticators);
auto id = storage->login(user_name, password, address, external_authenticators, /* replace_exception_with_cannot_authenticate = */ false);
std::lock_guard lock{mutex};
ids_cache.set(id, storage);
return id;
@ -416,7 +426,7 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p
throw;
}
}
throwCannotAuthenticate(user_name);
throwNotFound(EntityType::USER, user_name);
}

1
src/Access/MultipleAccessStorage.h
View File

@ -18,6 +18,7 @@ public:
using ConstStoragePtr = std::shared_ptr<const Storage>;
MultipleAccessStorage(const String & storage_name_ = STORAGE_TYPE);
~MultipleAccessStorage() override;
const char * getStorageType() const override { return STORAGE_TYPE; }

1
src/Access/ya.make
View File

@ -24,6 +24,7 @@ SRCS(
GrantedRoles.cpp
IAccessEntity.cpp
IAccessStorage.cpp
LDAPAccessStorage.cpp
LDAPClient.cpp
MemoryAccessStorage.cpp
MultipleAccessStorage.cpp

19
src/AggregateFunctions/AggregateFunctionMLMethod.cpp
View File

@ -143,13 +143,12 @@ void LinearModelData::updateState()
void LinearModelData::predict(
ColumnVector<Float64>::Container & container,
ColumnsWithTypeAndName & columns,
ColumnsWithTypeAndName & arguments,
size_t offset,
size_t limit,
const ColumnNumbers & arguments,
const Context & context) const
{
gradient_computer->predict(container, columns, offset, limit, arguments, weights, bias, context);
gradient_computer->predict(container, arguments, offset, limit, weights, bias, context);
}
void LinearModelData::returnWeights(IColumn & to) const
@ -449,15 +448,14 @@ void IWeightsUpdater::addToBatch(
void LogisticRegression::predict(
ColumnVector<Float64>::Container & container,
ColumnsWithTypeAndName & columns,
ColumnsWithTypeAndName & arguments,
size_t offset,
size_t limit,
const ColumnNumbers & arguments,
const std::vector<Float64> & weights,
Float64 bias,
const Context & /*context*/) const
{
size_t rows_num = columns[arguments.front()].column->size();
size_t rows_num = arguments.front().column->size();
if (offset > rows_num || offset + limit > rows_num)
throw Exception("Invalid offset and limit for LogisticRegression::predict. "
@ -468,7 +466,7 @@ void LogisticRegression::predict(
for (size_t i = 1; i < arguments.size(); ++i)
{
const ColumnWithTypeAndName & cur_col = columns[arguments[i]];
const ColumnWithTypeAndName & cur_col = arguments[i];
if (!isNativeNumber(cur_col.type))
throw Exception("Prediction arguments must have numeric type", ErrorCodes::BAD_ARGUMENTS);
@ -518,10 +516,9 @@ void LogisticRegression::compute(
void LinearRegression::predict(
ColumnVector<Float64>::Container & container,
ColumnsWithTypeAndName & columns,
ColumnsWithTypeAndName & arguments,
size_t offset,
size_t limit,
const ColumnNumbers & arguments,
const std::vector<Float64> & weights,
Float64 bias,
const Context & /*context*/) const
@ -531,7 +528,7 @@ void LinearRegression::predict(
throw Exception("In predict function number of arguments differs from the size of weights vector", ErrorCodes::LOGICAL_ERROR);
}
size_t rows_num = columns[arguments.front()].column->size();
size_t rows_num = arguments.front().column->size();
if (offset > rows_num || offset + limit > rows_num)
throw Exception("Invalid offset and limit for LogisticRegression::predict. "
@ -542,7 +539,7 @@ void LinearRegression::predict(
for (size_t i = 1; i < arguments.size(); ++i)
{
const ColumnWithTypeAndName & cur_col = columns[arguments[i]];
const ColumnWithTypeAndName & cur_col = arguments[i];
if (!isNativeNumber(cur_col.type))
throw Exception("Prediction arguments must have numeric type", ErrorCodes::BAD_ARGUMENTS);

17
src/AggregateFunctions/AggregateFunctionMLMethod.h
View File

@ -39,10 +39,9 @@ public:
virtual void predict(
ColumnVector<Float64>::Container & container,
ColumnsWithTypeAndName & columns,
ColumnsWithTypeAndName & arguments,
size_t offset,
size_t limit,
const ColumnNumbers & arguments,
const std::vector<Float64> & weights,
Float64 bias,
const Context & context) const = 0;
@ -65,10 +64,9 @@ public:
void predict(
ColumnVector<Float64>::Container & container,
ColumnsWithTypeAndName & columns,
ColumnsWithTypeAndName & arguments,
size_t offset,
size_t limit,
const ColumnNumbers & arguments,
const std::vector<Float64> & weights,
Float64 bias,
const Context & context) const override;
@ -91,10 +89,9 @@ public:
void predict(
ColumnVector<Float64>::Container & container,
ColumnsWithTypeAndName & columns,
ColumnsWithTypeAndName & arguments,
size_t offset,
size_t limit,
const ColumnNumbers & arguments,
const std::vector<Float64> & weights,
Float64 bias,
const Context & context) const override;
@ -264,10 +261,9 @@ public:
void predict(
ColumnVector<Float64>::Container & container,
ColumnsWithTypeAndName & columns,
ColumnsWithTypeAndName & arguments,
size_t offset,
size_t limit,
const ColumnNumbers & arguments,
const Context & context) const;
void returnWeights(IColumn & to) const;
@ -364,10 +360,9 @@ public:
void predictValues(
ConstAggregateDataPtr place,
IColumn & to,
ColumnsWithTypeAndName & columns,
ColumnsWithTypeAndName & arguments,
size_t offset,
size_t limit,
const ColumnNumbers & arguments,
const Context & context) const override
{
if (arguments.size() != param_num + 1)
@ -382,7 +377,7 @@ public:
throw Exception("Cast of column of predictions is incorrect. getReturnTypeToPredict must return same value as it is casted to",
ErrorCodes::LOGICAL_ERROR);
this->data(place).predict(column->getData(), columns, offset, limit, arguments, context);
this->data(place).predict(column->getData(), arguments, offset, limit, context);
}
/** This function is called if aggregate function without State modifier is selected in a query.

35
src/AggregateFunctions/AggregateFunctionRankCorrelation.h
View File

@ -21,10 +21,6 @@
#include <type_traits>
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
namespace DB
{
@ -138,23 +134,18 @@ public:
const auto & value = this->data(place).values;
size_t size = this->data(place).size_x;
if (size < 2)
{
throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS);
}
//create a copy of values not to format data
// create a copy of values not to format data
PODArrayWithStackMemory<std::pair<Float64, Float64>, 32> tmp_values;
tmp_values.resize(size);
for (size_t j = 0; j < size; ++ j)
tmp_values[j] = static_cast<std::pair<Float64, Float64>>(value[j]);
//sort x_values
// sort x_values
std::sort(std::begin(tmp_values), std::end(tmp_values), ComparePairFirst<std::greater>{});
for (size_t j = 0; j < size;)
{
//replace x_values with their ranks
// replace x_values with their ranks
size_t rank = j + 1;
size_t same = 1;
size_t cur_sum = rank;
@ -166,9 +157,9 @@ public:
{
// rank of (j + 1)th number
rank += 1;
same++;
++same;
cur_sum += rank;
j++;
++j;
}
else
break;
@ -178,16 +169,16 @@ public:
Float64 insert_rank = static_cast<Float64>(cur_sum) / same;
for (size_t i = cur_start; i <= j; ++i)
tmp_values[i].first = insert_rank;
j++;
++j;
}
//sort y_values
// sort y_values
std::sort(std::begin(tmp_values), std::end(tmp_values), ComparePairSecond<std::greater>{});
//replace y_values with their ranks
// replace y_values with their ranks
for (size_t j = 0; j < size;)
{
//replace x_values with their ranks
// replace x_values with their ranks
size_t rank = j + 1;
size_t same = 1;
size_t cur_sum = rank;
@ -199,9 +190,9 @@ public:
{
// rank of (j + 1)th number
rank += 1;
same++;
++same;
cur_sum += rank;
j++;
++j;
}
else
{
@ -213,10 +204,10 @@ public:
Float64 insert_rank = static_cast<Float64>(cur_sum) / same;
for (size_t i = cur_start; i <= j; ++i)
tmp_values[i].second = insert_rank;
j++;
++j;
}
//count d^2 sum
// count d^2 sum
Float64 answer = static_cast<Float64>(0);
for (size_t j = 0; j < size; ++ j)
answer += (tmp_values[j].first - tmp_values[j].second) * (tmp_values[j].first - tmp_values[j].second);

3
src/AggregateFunctions/IAggregateFunction.h
View File

@ -114,10 +114,9 @@ public:
virtual void predictValues(
ConstAggregateDataPtr /* place */,
IColumn & /*to*/,
ColumnsWithTypeAndName & /*block*/,
ColumnsWithTypeAndName & /*arguments*/,
size_t /*offset*/,
size_t /*limit*/,
const ColumnNumbers & /*arguments*/,
const Context & /*context*/) const
{
throw Exception("Method predictValues is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);

6
src/Columns/ColumnAggregateFunction.cpp
View File

@ -161,7 +161,7 @@ MutableColumnPtr ColumnAggregateFunction::convertToValues(MutableColumnPtr colum
return res;
}
MutableColumnPtr ColumnAggregateFunction::predictValues(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const Context & context) const
MutableColumnPtr ColumnAggregateFunction::predictValues(ColumnsWithTypeAndName & arguments, const Context & context) const
{
MutableColumnPtr res = func->getReturnTypeToPredict()->createColumn();
res->reserve(data.size());
@ -172,7 +172,7 @@ MutableColumnPtr ColumnAggregateFunction::predictValues(ColumnsWithTypeAndName &
if (data.size() == 1)
{
/// Case for const column. Predict using single model.
machine_learning_function->predictValues(data[0], *res, block, 0, block[arguments.front()].column->size(), arguments, context);
machine_learning_function->predictValues(data[0], *res, arguments, 0, arguments.front().column->size(), context);
}
else
{
@ -180,7 +180,7 @@ MutableColumnPtr ColumnAggregateFunction::predictValues(ColumnsWithTypeAndName &
size_t row_num = 0;
for (auto * val : data)
{
machine_learning_function->predictValues(val, *res, block, row_num, 1, arguments, context);
machine_learning_function->predictValues(val, *res, arguments, row_num, 1, context);
++row_num;
}
}

2
src/Columns/ColumnAggregateFunction.h
View File

@ -119,7 +119,7 @@ public:
const char * getFamilyName() const override { return "AggregateFunction"; }
TypeIndex getDataType() const override { return TypeIndex::AggregateFunction; }
MutableColumnPtr predictValues(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const Context & context) const;
MutableColumnPtr predictValues(ColumnsWithTypeAndName & arguments, const Context & context) const;
size_t size() const override
{

11
src/Columns/ColumnFunction.cpp
View File

@ -188,15 +188,10 @@ ColumnWithTypeAndName ColumnFunction::reduce() const
"arguments but " + toString(captured) + " columns were captured.", ErrorCodes::LOGICAL_ERROR);
auto columns = captured_columns;
columns.emplace_back(ColumnWithTypeAndName {nullptr, function->getReturnType(), ""});
ColumnWithTypeAndName res{nullptr, function->getResultType(), ""};
ColumnNumbers arguments(captured_columns.size());
for (size_t i = 0; i < captured_columns.size(); ++i)
arguments[i] = i;
function->execute(columns, arguments, captured_columns.size(), size_);
return columns[captured_columns.size()];
res.column = function->execute(columns, res.type, size_);
return res;
}
}

6
src/Columns/ColumnString.cpp
View File

@ -634,4 +634,10 @@ void ColumnString::protect()
getOffsets().protect();
}
void ColumnString::validate() const
{
if (!offsets.empty() && offsets.back() != chars.size())
throw Exception(ErrorCodes::LOGICAL_ERROR, "ColumnString validation failed: size mismatch (internal logical error) {} != {}", offsets.back(), chars.size());
}
}

3
src/Columns/ColumnString.h
View File

@ -267,6 +267,9 @@ public:
Offsets & getOffsets() { return offsets; }
const Offsets & getOffsets() const { return offsets; }
// Throws an exception if offsets/chars are messed up
void validate() const;
};

2
src/Common/ErrorCodes.cpp
View File

@ -510,6 +510,8 @@ namespace ErrorCodes
extern const int ROW_AND_ROWS_TOGETHER = 544;
extern const int FIRST_AND_NEXT_TOGETHER = 545;
extern const int NO_ROW_DELIMITER = 546;
extern const int INVALID_RAID_TYPE = 547;
extern const int UNKNOWN_VOLUME = 548;
extern const int KEEPER_EXCEPTION = 999;
extern const int POCO_EXCEPTION = 1000;

43
src/Common/MemoryTracker.cpp
View File

@ -30,6 +30,8 @@ namespace ProfileEvents
static constexpr size_t log_peak_memory_usage_every = 1ULL << 30;
thread_local bool MemoryTracker::BlockerInThread::is_blocked = false;
MemoryTracker total_memory_tracker(nullptr, VariableContext::Global);
@ -56,13 +58,15 @@ MemoryTracker::~MemoryTracker()
void MemoryTracker::logPeakMemoryUsage() const
{
const auto * description = description_ptr.load(std::memory_order_relaxed);
LOG_DEBUG(&Poco::Logger::get("MemoryTracker"), "Peak memory usage{}: {}.", (description ? " " + std::string(description) : ""), ReadableSize(peak));
LOG_DEBUG(&Poco::Logger::get("MemoryTracker"),
"Peak memory usage{}: {}.", (description ? " " + std::string(description) : ""), ReadableSize(peak));
}
void MemoryTracker::logMemoryUsage(Int64 current) const
{
const auto * description = description_ptr.load(std::memory_order_relaxed);
LOG_DEBUG(&Poco::Logger::get("MemoryTracker"), "Current memory usage{}: {}.", (description ? " " + std::string(description) : ""), ReadableSize(current));
LOG_DEBUG(&Poco::Logger::get("MemoryTracker"),
"Current memory usage{}: {}.", (description ? " " + std::string(description) : ""), ReadableSize(current));
}
@ -71,7 +75,7 @@ void MemoryTracker::alloc(Int64 size)
if (size < 0)
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Negative size ({}) is passed to MemoryTracker. It is a bug.", size);
if (blocker.isCancelled())
if (BlockerInThread::isBlocked())
return;
/** Using memory_order_relaxed means that if allocations are done simultaneously,
@ -86,12 +90,15 @@ void MemoryTracker::alloc(Int64 size)
Int64 current_hard_limit = hard_limit.load(std::memory_order_relaxed);
Int64 current_profiler_limit = profiler_limit.load(std::memory_order_relaxed);
/// Cap the limit to the total_memory_tracker, since it may include some drift.
/// Cap the limit to the total_memory_tracker, since it may include some drift
/// for user-level memory tracker.
///
/// And since total_memory_tracker is reset to the process resident
/// memory peridically (in AsynchronousMetrics::update()), any limit can be
/// capped to it, to avoid possible drift.
if (unlikely(current_hard_limit && will_be > current_hard_limit))
if (unlikely(current_hard_limit
&& will_be > current_hard_limit
&& level == VariableContext::User))
{
Int64 total_amount = total_memory_tracker.get();
if (amount > total_amount)
@ -104,10 +111,8 @@ void MemoryTracker::alloc(Int64 size)
std::bernoulli_distribution fault(fault_probability);
if (unlikely(fault_probability && fault(thread_local_rng)))
{
free(size);
/// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc
auto untrack_lock = blocker.cancel(); // NOLINT
BlockerInThread untrack_lock;
ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded);
std::stringstream message;
@ -118,12 +123,13 @@ void MemoryTracker::alloc(Int64 size)
<< " (attempt to allocate chunk of " << size << " bytes)"
<< ", maximum: " << formatReadableSizeWithBinarySuffix(current_hard_limit);
amount.fetch_sub(size, std::memory_order_relaxed);
throw DB::Exception(message.str(), DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED);
}
if (unlikely(current_profiler_limit && will_be > current_profiler_limit))
{
auto no_track = blocker.cancel();
BlockerInThread untrack_lock;
DB::TraceCollector::collect(DB::TraceType::Memory, StackTrace(), size);
setOrRaiseProfilerLimit((will_be + profiler_step - 1) / profiler_step * profiler_step);
}
@ -131,16 +137,14 @@ void MemoryTracker::alloc(Int64 size)
std::bernoulli_distribution sample(sample_probability);
if (unlikely(sample_probability && sample(thread_local_rng)))
{
auto no_track = blocker.cancel();
BlockerInThread untrack_lock;
DB::TraceCollector::collect(DB::TraceType::MemorySample, StackTrace(), size);
}
if (unlikely(current_hard_limit && will_be > current_hard_limit))
{
free(size);
/// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc
auto no_track = blocker.cancel(); // NOLINT
BlockerInThread untrack_lock;
ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded);
std::stringstream message;
@ -151,6 +155,7 @@ void MemoryTracker::alloc(Int64 size)
<< " (attempt to allocate chunk of " << size << " bytes)"
<< ", maximum: " << formatReadableSizeWithBinarySuffix(current_hard_limit);
amount.fetch_sub(size, std::memory_order_relaxed);
throw DB::Exception(message.str(), DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED);
}
@ -177,13 +182,13 @@ void MemoryTracker::updatePeak(Int64 will_be)
void MemoryTracker::free(Int64 size)
{
if (blocker.isCancelled())
if (BlockerInThread::isBlocked())
return;
std::bernoulli_distribution sample(sample_probability);
if (unlikely(sample_probability && sample(thread_local_rng)))
{
auto no_track = blocker.cancel();
BlockerInThread untrack_lock;
DB::TraceCollector::collect(DB::TraceType::MemorySample, StackTrace(), -size);
}
@ -298,11 +303,3 @@ namespace CurrentMemoryTracker
}
}
}
DB::SimpleActionLock getCurrentMemoryTrackerActionLock()
{
auto * memory_tracker = DB::CurrentThread::getMemoryTracker();
if (!memory_tracker)
return {};
return memory_tracker->blocker.cancel();
}

19
src/Common/MemoryTracker.h
View File

@ -3,7 +3,6 @@
#include <atomic>
#include <common/types.h>
#include <Common/CurrentMetrics.h>
#include <Common/SimpleActionBlocker.h>
#include <Common/VariableContext.h>
@ -131,8 +130,18 @@ public:
/// Prints info about peak memory consumption into log.
void logPeakMemoryUsage() const;
/// To be able to temporarily stop memory tracker
DB::SimpleActionBlocker blocker;
/// To be able to temporarily stop memory tracking from current thread.
struct BlockerInThread
{
private:
BlockerInThread(const BlockerInThread &) = delete;
BlockerInThread & operator=(const BlockerInThread &) = delete;
static thread_local bool is_blocked;
public:
BlockerInThread() { is_blocked = true; }
~BlockerInThread() { is_blocked = false; }
static bool isBlocked() { return is_blocked; }
};
};
extern MemoryTracker total_memory_tracker;
@ -145,7 +154,3 @@ namespace CurrentMemoryTracker
void realloc(Int64 old_size, Int64 new_size);
void free(Int64 size);
}
/// Holding this object will temporarily disable memory tracking.
DB::SimpleActionLock getCurrentMemoryTrackerActionLock();

2
src/Common/ThreadStatus.h
View File

@ -164,6 +164,8 @@ public:
void detachQuery(bool exit_if_already_detached = false, bool thread_exits = false);
protected:
void applyQuerySettings();
void initPerformanceCounters();
void initQueryProfiler();

14
src/Common/tests/average.cpp
View File

@ -502,8 +502,8 @@ Float NO_INLINE really_unrolled(const PODArray<UInt8> & keys, const PODArray<Flo
struct State4
{
Float sum[4] = {0, 0, 0, 0};
size_t count[4] = {0, 0, 0, 0};
Float sum[4]{};
size_t count[4]{};
template <UInt32 idx>
void add(Float value)
@ -522,13 +522,13 @@ Float NO_INLINE another_unrolled_x4(const PODArray<UInt8> & keys, const PODArray
{
State4 map[256]{};
size_t size = keys.size() & ~size_t(3);
for (size_t i = 0; i < size; i+=4)
size_t size = keys.size() / 4 * 4;
for (size_t i = 0; i < size; i += 4)
{
map[keys[i]].add<0>(values[i]);
map[keys[i+1]].add<1>(values[i]);
map[keys[i+2]].add<2>(values[i]);
map[keys[i+3]].add<3>(values[i]);
map[keys[i + 1]].add<1>(values[i]);
map[keys[i + 2]].add<2>(values[i]);
map[keys[i + 3]].add<3>(values[i]);
}
/// tail

18
src/Core/DecimalComparison.h
View File

@ -60,27 +60,17 @@ public:
using ArrayA = typename ColVecA::Container;
using ArrayB = typename ColVecB::Container;
DecimalComparison(ColumnsWithTypeAndName & data, size_t result, const ColumnWithTypeAndName & col_left, const ColumnWithTypeAndName & col_right)
{
if (!apply(data, result, col_left, col_right))
throw Exception("Wrong decimal comparison with " + col_left.type->getName() + " and " + col_right.type->getName(),
ErrorCodes::LOGICAL_ERROR);
}
static bool apply(ColumnsWithTypeAndName & data, size_t result [[maybe_unused]],
const ColumnWithTypeAndName & col_left, const ColumnWithTypeAndName & col_right)
static ColumnPtr apply(const ColumnWithTypeAndName & col_left, const ColumnWithTypeAndName & col_right)
{
if constexpr (_actual)
{
ColumnPtr c_res;
Shift shift = getScales<A, B>(col_left.type, col_right.type);
c_res = applyWithScale(col_left.column, col_right.column, shift);
if (c_res)
data[result].column = std::move(c_res);
return true;
return applyWithScale(col_left.column, col_right.column, shift);
}
return false;
else
return nullptr;
}
static bool compare(A a, B b, UInt32 scale_a, UInt32 scale_b)

4
src/Core/Settings.h
View File

@ -111,6 +111,7 @@ class IColumn;
M(UInt64, distributed_group_by_no_merge, 0, "If 1, Do not merge aggregation states from different servers for distributed query processing - in case it is for certain that there are different keys on different shards. If 2 - same as 1 but also apply ORDER BY and LIMIT stages", 0) \
M(Bool, optimize_distributed_group_by_sharding_key, false, "Optimize GROUP BY sharding_key queries (by avodiing costly aggregation on the initiator server).", 0) \
M(Bool, optimize_skip_unused_shards, false, "Assumes that data is distributed by sharding_key. Optimization to skip unused shards if SELECT query filters by sharding_key.", 0) \
M(Bool, allow_nondeterministic_optimize_skip_unused_shards, false, "Allow non-deterministic functions (includes dictGet) in sharding_key for optimize_skip_unused_shards", 0) \
M(UInt64, force_optimize_skip_unused_shards, 0, "Throw an exception if unused shards cannot be skipped (1 - throw only if the table has the sharding key, 2 - always throw.", 0) \
M(UInt64, optimize_skip_unused_shards_nesting, 0, "Same as optimize_skip_unused_shards, but accept nesting level until which it will work.", 0) \
M(UInt64, force_optimize_skip_unused_shards_nesting, 0, "Same as force_optimize_skip_unused_shards, but accept nesting level until which it will work.", 0) \
@ -153,6 +154,7 @@ class IColumn;
\
M(DistributedProductMode, distributed_product_mode, DistributedProductMode::DENY, "How are distributed subqueries performed inside IN or JOIN sections?", IMPORTANT) \
\
M(UInt64, max_concurrent_queries_for_all_users, 0, "The maximum number of concurrent requests for all users.", 0) \
M(UInt64, max_concurrent_queries_for_user, 0, "The maximum number of concurrent requests per user.", 0) \
\
M(Bool, insert_deduplicate, true, "For INSERT queries in the replicated table, specifies that deduplication of insertings blocks should be performed", 0) \
@ -412,12 +414,14 @@ class IColumn;
M(Bool, format_csv_allow_double_quotes, 1, "If it is set to true, allow strings in double quotes.", 0) \
M(Bool, output_format_csv_crlf_end_of_line, false, "If it is set true, end of line in CSV format will be \\r\\n instead of \\n.", 0) \
M(Bool, input_format_csv_unquoted_null_literal_as_null, false, "Consider unquoted NULL literal as \\N", 0) \
M(Bool, input_format_csv_enum_as_number, false, "Treat inserted enum values in CSV formats as enum indices \\N", 0) \
M(Bool, input_format_skip_unknown_fields, false, "Skip columns with unknown names from input data (it works for JSONEachRow, CSVWithNames, TSVWithNames and TSKV formats).", 0) \
M(Bool, input_format_with_names_use_header, true, "For TSVWithNames and CSVWithNames input formats this controls whether format parser is to assume that column data appear in the input exactly as they are specified in the header.", 0) \
M(Bool, input_format_import_nested_json, false, "Map nested JSON data to nested tables (it works for JSONEachRow format).", 0) \
M(Bool, optimize_aggregators_of_group_by_keys, true, "Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section", 0) \
M(Bool, input_format_defaults_for_omitted_fields, true, "For input data calculate default expressions for omitted fields (it works for JSONEachRow, CSV and TSV formats).", IMPORTANT) \
M(Bool, input_format_tsv_empty_as_default, false, "Treat empty fields in TSV input as default values.", 0) \
M(Bool, input_format_tsv_enum_as_number, false, "Treat inserted enum values in TSV formats as enum indices \\N", 0) \
M(Bool, input_format_null_as_default, false, "For text input formats initialize null fields with default values if data type of this field is not nullable", 0) \
\
M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.", 0) \

24
src/DataTypes/DataTypeEnum.cpp
View File

@ -146,12 +146,17 @@ void DataTypeEnum<Type>::serializeTextEscaped(const IColumn & column, size_t row
}
template <typename Type>
void DataTypeEnum<Type>::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
void DataTypeEnum<Type>::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
if (settings.tsv.input_format_enum_as_number)
assert_cast<ColumnType &>(column).getData().push_back(readValue(istr));
else
{
/// NOTE It would be nice to do without creating a temporary object - at least extract std::string out.
std::string field_name;
readEscapedString(field_name, istr);
assert_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(field_name)));
}
}
template <typename Type>
@ -169,11 +174,16 @@ void DataTypeEnum<Type>::deserializeTextQuoted(IColumn & column, ReadBuffer & is
}
template <typename Type>
void DataTypeEnum<Type>::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
void DataTypeEnum<Type>::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
if (settings.tsv.input_format_enum_as_number)
assert_cast<ColumnType &>(column).getData().push_back(readValue(istr));
else
{
std::string field_name;
readString(field_name, istr);
assert_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(field_name)));
}
}
template <typename Type>
@ -191,9 +201,14 @@ void DataTypeEnum<Type>::serializeTextXML(const IColumn & column, size_t row_num
template <typename Type>
void DataTypeEnum<Type>::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
{
if (!istr.eof() && *istr.position() != '"')
assert_cast<ColumnType &>(column).getData().push_back(readValue(istr));
else
{
std::string field_name;
readJSONString(field_name, istr);
assert_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(field_name)));
}
}
template <typename Type>
@ -205,9 +220,14 @@ void DataTypeEnum<Type>::serializeTextCSV(const IColumn & column, size_t row_num
template <typename Type>
void DataTypeEnum<Type>::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
if (settings.csv.input_format_enum_as_number)
assert_cast<ColumnType &>(column).getData().push_back(readValue(istr));
else
{
std::string field_name;
readCSVString(field_name, istr, settings.csv);
assert_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(field_name)));
}
}
template <typename Type>

16
src/DataTypes/DataTypeEnum.h
View File

@ -66,13 +66,18 @@ public:
TypeIndex getTypeId() const override { return sizeof(FieldType) == 1 ? TypeIndex::Enum8 : TypeIndex::Enum16; }
const StringRef & getNameForValue(const FieldType & value) const
auto findByValue(const FieldType & value) const
{
const auto it = value_to_name_map.find(value);
if (it == std::end(value_to_name_map))
throw Exception{"Unexpected value " + toString(value) + " for type " + getName(), ErrorCodes::BAD_ARGUMENTS};
return it->second;
return it;
}
const StringRef & getNameForValue(const FieldType & value) const
{
return findByValue(value)->second;
}
FieldType getValue(StringRef field_name) const
@ -84,6 +89,13 @@ public:
return it->getMapped();
}
FieldType readValue(ReadBuffer & istr) const
{
FieldType x;
readText(x, istr);
return findByValue(x)->first;
}
Field castToName(const Field & value_or_name) const override;
Field castToValue(const Field & value_or_name) const override;

4
src/Dictionaries/FileDictionarySource.cpp
View File

@ -32,7 +32,7 @@ FileDictionarySource::FileDictionarySource(
{
const String user_files_path = context.getUserFilesPath();
if (!startsWith(filepath, user_files_path))
throw Exception("File path " + filepath + " is not inside " + user_files_path, ErrorCodes::PATH_ACCESS_DENIED);
throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", filepath, user_files_path);
}
}
@ -60,7 +60,7 @@ BlockInputStreamPtr FileDictionarySource::loadAll()
std::string FileDictionarySource::toString() const
{
return "File: " + filepath + ' ' + format;
return fmt::format("File: {}, {}", filepath, format);
}

5
src/Disks/DiskDecorator.cpp
View File

@ -180,4 +180,9 @@ void DiskDecorator::sync(int fd) const
delegate->sync(fd);
}
Executor & DiskDecorator::getExecutor()
{
return delegate->getExecutor();
}
}

5
src/Disks/DiskDecorator.h
View File

@ -4,6 +4,10 @@
namespace DB
{
/** Forwards all methods to another disk.
* Methods can be overridden by descendants.
*/
class DiskDecorator : public IDisk
{
public:
@ -46,6 +50,7 @@ public:
void close(int fd) const override;
void sync(int fd) const override;
const String getType() const override { return delegate->getType(); }
Executor & getExecutor() override;
protected:
DiskPtr delegate;

7
src/Disks/DiskSelector.h
View File

@ -23,8 +23,11 @@ public:
DiskSelector(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, const Context & context);
DiskSelector(const DiskSelector & from) : disks(from.disks) { }
DiskSelectorPtr
updateFromConfig(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, const Context & context) const;
DiskSelectorPtr updateFromConfig(
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
const Context & context
) const;
/// Get disk by name
DiskPtr get(const String & name) const;

4
src/Disks/IDisk.h
View File

@ -195,10 +195,10 @@ public:
/// Invoked when Global Context is shutdown.
virtual void shutdown() { }
private:
/// Returns executor to perform asynchronous operations.
Executor & getExecutor() { return *executor; }
virtual Executor & getExecutor() { return *executor; }
private:
std::unique_ptr<Executor> executor;
};

4
src/Disks/IVolume.cpp
View File

@ -9,7 +9,7 @@ namespace DB
{
namespace ErrorCodes
{
extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
extern const int NO_ELEMENTS_IN_CONFIG;
extern const int INCONSISTENT_RESERVATIONS;
extern const int NO_RESERVATIONS_PROVIDED;
extern const int UNKNOWN_VOLUME_TYPE;
@ -51,7 +51,7 @@ IVolume::IVolume(
}
if (disks.empty())
throw Exception("Volume must contain at least one disk.", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
throw Exception("Volume must contain at least one disk", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
}
UInt64 IVolume::getMaxUnreservedFreeSpace() const

6
src/Disks/IVolume.h
View File

@ -64,6 +64,12 @@ public:
virtual DiskPtr getDisk(size_t i) const { return disks[i]; }
const Disks & getDisks() const { return disks; }
/// Returns effective value of whether merges are allowed on this volume (true) or not (false).
virtual bool areMergesAvoided() const { return false; }
/// User setting for enabling and disabling merges on volume.
virtual void setAvoidMergesUserOverride(bool /*avoid*/) {}
protected:
Disks disks;
const String name;

2
src/Disks/SingleDiskVolume.h
View File

@ -8,7 +8,7 @@ namespace DB
class SingleDiskVolume : public IVolume
{
public:
SingleDiskVolume(const String & name_, DiskPtr disk): IVolume(name_, {disk})
SingleDiskVolume(const String & name_, DiskPtr disk, size_t max_data_part_size_ = 0): IVolume(name_, {disk}, max_data_part_size_)
{
}

198
src/Disks/StoragePolicy.cpp
View File

@ -11,6 +11,13 @@
#include <Poco/File.h>
namespace
{
const auto DEFAULT_STORAGE_POLICY_NAME = "default";
const auto DEFAULT_VOLUME_NAME = "default";
const auto DEFAULT_DISK_NAME = "default";
}
namespace DB
{
@ -18,11 +25,14 @@ namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
extern const int NO_ELEMENTS_IN_CONFIG;
extern const int UNKNOWN_DISK;
extern const int UNKNOWN_POLICY;
extern const int UNKNOWN_VOLUME;
extern const int LOGICAL_ERROR;
}
StoragePolicy::StoragePolicy(
String name_,
const Poco::Util::AbstractConfiguration & config,
@ -30,44 +40,42 @@ StoragePolicy::StoragePolicy(
DiskSelectorPtr disks)
: name(std::move(name_))
{
String volumes_prefix = config_prefix + ".volumes";
if (!config.has(volumes_prefix))
throw Exception("StoragePolicy must contain at least one volume (.volumes)", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
Poco::Util::AbstractConfiguration::Keys keys;
String volumes_prefix = config_prefix + ".volumes";
if (!config.has(volumes_prefix))
{
if (name != DEFAULT_STORAGE_POLICY_NAME)
throw Exception("Storage policy " + backQuote(name) + " must contain at least one volume (.volumes)", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
}
else
{
config.keys(volumes_prefix, keys);
}
for (const auto & attr_name : keys)
{
if (!std::all_of(attr_name.begin(), attr_name.end(), isWordCharASCII))
throw Exception(
"Volume name can contain only alphanumeric and '_' (" + attr_name + ")", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
volumes.push_back(std::make_shared<VolumeJBOD>(attr_name, config, volumes_prefix + "." + attr_name, disks));
if (volumes_names.find(attr_name) != volumes_names.end())
throw Exception("Volumes names must be unique (" + attr_name + " duplicated)", ErrorCodes::UNKNOWN_POLICY);
volumes_names[attr_name] = volumes.size() - 1;
"Volume name can contain only alphanumeric and '_' in storage policy " + backQuote(name) + " (" + attr_name + ")", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
volumes.emplace_back(createVolumeFromConfig(attr_name, config, volumes_prefix + "." + attr_name, disks));
}
if (volumes.empty() && name == DEFAULT_STORAGE_POLICY_NAME)
{
auto default_volume = std::make_shared<VolumeJBOD>(DEFAULT_VOLUME_NAME, std::vector<DiskPtr>{disks->get(DEFAULT_DISK_NAME)}, 0, false);
volumes.emplace_back(std::move(default_volume));
}
if (volumes.empty())
throw Exception("StoragePolicy must contain at least one volume.", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
throw Exception("Storage policy " + backQuote(name) + " must contain at least one volume.", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
/// Check that disks are unique in Policy
std::set<String> disk_names;
for (const auto & volume : volumes)
{
for (const auto & disk : volume->getDisks())
{
if (disk_names.find(disk->getName()) != disk_names.end())
throw Exception(
"Duplicate disk '" + disk->getName() + "' in storage policy '" + name + "'", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
disk_names.insert(disk->getName());
}
}
move_factor = config.getDouble(config_prefix + ".move_factor", 0.1);
const double default_move_factor = volumes.size() > 1 ? 0.1 : 0.0;
move_factor = config.getDouble(config_prefix + ".move_factor", default_move_factor);
if (move_factor > 1)
throw Exception("Disk move factor have to be in [0., 1.] interval, but set to " + toString(move_factor), ErrorCodes::LOGICAL_ERROR);
throw Exception("Disk move factor have to be in [0., 1.] interval, but set to " + toString(move_factor) + " in storage policy " + backQuote(name), ErrorCodes::LOGICAL_ERROR);
buildVolumeIndices();
}
@ -75,16 +83,43 @@ StoragePolicy::StoragePolicy(String name_, Volumes volumes_, double move_factor_
: volumes(std::move(volumes_)), name(std::move(name_)), move_factor(move_factor_)
{
if (volumes.empty())
throw Exception("StoragePolicy must contain at least one Volume.", ErrorCodes::UNKNOWN_POLICY);
throw Exception("Storage policy " + backQuote(name) + " must contain at least one Volume.", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
if (move_factor > 1)
throw Exception("Disk move factor have to be in [0., 1.] interval, but set to " + toString(move_factor), ErrorCodes::LOGICAL_ERROR);
throw Exception("Disk move factor have to be in [0., 1.] interval, but set to " + toString(move_factor) + " in storage policy " + backQuote(name), ErrorCodes::LOGICAL_ERROR);
for (size_t i = 0; i < volumes.size(); ++i)
buildVolumeIndices();
}
StoragePolicy::StoragePolicy(const StoragePolicy & storage_policy,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
DiskSelectorPtr disks)
: StoragePolicy(storage_policy.getName(), config, config_prefix, disks)
{
for (auto & volume : volumes)
{
if (volumes_names.find(volumes[i]->getName()) != volumes_names.end())
throw Exception("Volumes names must be unique (" + volumes[i]->getName() + " duplicated).", ErrorCodes::UNKNOWN_POLICY);
volumes_names[volumes[i]->getName()] = i;
if (storage_policy.volume_index_by_volume_name.count(volume->getName()) > 0)
{
auto old_volume = storage_policy.getVolumeByName(volume->getName());
try
{
auto new_volume = updateVolumeFromConfig(old_volume, config, config_prefix + ".volumes." + volume->getName(), disks);
volume = std::move(new_volume);
}
catch (Exception & e)
{
/// Default policies are allowed to be missed in configuration.
if (e.code() != ErrorCodes::NO_ELEMENTS_IN_CONFIG || storage_policy.getName() != DEFAULT_STORAGE_POLICY_NAME)
throw;
Poco::Util::AbstractConfiguration::Keys keys;
config.keys(config_prefix, keys);
if (!keys.empty())
throw;
}
}
}
}
@ -93,20 +128,20 @@ bool StoragePolicy::isDefaultPolicy() const
{
/// Guessing if this policy is default, not 100% correct though.
if (getName() != "default")
if (getName() != DEFAULT_STORAGE_POLICY_NAME)
return false;
if (volumes.size() != 1)
return false;
if (volumes[0]->getName() != "default")
if (volumes[0]->getName() != DEFAULT_VOLUME_NAME)
return false;
const auto & disks = volumes[0]->getDisks();
if (disks.size() != 1)
return false;
if (disks[0]->getName() != "default")
if (disks[0]->getName() != DEFAULT_DISK_NAME)
return false;
return true;
@ -128,10 +163,10 @@ DiskPtr StoragePolicy::getAnyDisk() const
/// StoragePolicy must contain at least one Volume
/// Volume must contain at least one Disk
if (volumes.empty())
throw Exception("StoragePolicy has no volumes. It's a bug.", ErrorCodes::LOGICAL_ERROR);
throw Exception("Storage policy " + backQuote(name) + " has no volumes. It's a bug.", ErrorCodes::LOGICAL_ERROR);
if (volumes[0]->getDisks().empty())
throw Exception("Volume '" + volumes[0]->getName() + "' has no disks. It's a bug.", ErrorCodes::LOGICAL_ERROR);
throw Exception("Volume " + backQuote(name) + "." + backQuote(volumes[0]->getName()) + " has no disks. It's a bug.", ErrorCodes::LOGICAL_ERROR);
return volumes[0]->getDisks()[0];
}
@ -195,6 +230,24 @@ ReservationPtr StoragePolicy::makeEmptyReservationOnLargestDisk() const
}
VolumePtr StoragePolicy::getVolume(size_t index) const
{
if (index < volume_index_by_volume_name.size())
return volumes[index];
else
throw Exception("No volume with index " + std::to_string(index) + " in storage policy " + backQuote(name), ErrorCodes::UNKNOWN_VOLUME);
}
VolumePtr StoragePolicy::getVolumeByName(const String & volume_name) const
{
auto it = volume_index_by_volume_name.find(volume_name);
if (it == volume_index_by_volume_name.end())
throw Exception("No such volume " + backQuote(volume_name) + " in storage policy " + backQuote(name), ErrorCodes::UNKNOWN_VOLUME);
return getVolume(it->second);
}
void StoragePolicy::checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const
{
std::unordered_set<String> new_volume_names;
@ -204,7 +257,7 @@ void StoragePolicy::checkCompatibleWith(const StoragePolicyPtr & new_storage_pol
for (const auto & volume : getVolumes())
{
if (new_volume_names.count(volume->getName()) == 0)
throw Exception("New storage policy shall contain volumes of old one", ErrorCodes::BAD_ARGUMENTS);
throw Exception("New storage policy " + backQuote(name) + " shall contain volumes of old one", ErrorCodes::BAD_ARGUMENTS);
std::unordered_set<String> new_disk_names;
for (const auto & disk : new_storage_policy->getVolumeByName(volume->getName())->getDisks())
@ -212,21 +265,46 @@ void StoragePolicy::checkCompatibleWith(const StoragePolicyPtr & new_storage_pol
for (const auto & disk : volume->getDisks())
if (new_disk_names.count(disk->getName()) == 0)
throw Exception("New storage policy shall contain disks of old one", ErrorCodes::BAD_ARGUMENTS);
throw Exception("New storage policy " + backQuote(name) + " shall contain disks of old one", ErrorCodes::BAD_ARGUMENTS);
}
}
size_t StoragePolicy::getVolumeIndexByDisk(const DiskPtr & disk_ptr) const
{
for (size_t i = 0; i < volumes.size(); ++i)
auto it = volume_index_by_disk_name.find(disk_ptr->getName());
if (it != volume_index_by_disk_name.end())
return it->second;
else
throw Exception("No disk " + backQuote(disk_ptr->getName()) + " in policy " + backQuote(name), ErrorCodes::UNKNOWN_DISK);
}
void StoragePolicy::buildVolumeIndices()
{
for (size_t index = 0; index < volumes.size(); ++index)
{
const auto & volume = volumes[i];
const VolumePtr & volume = volumes[index];
if (volume_index_by_volume_name.find(volume->getName()) != volume_index_by_volume_name.end())
throw Exception("Volume names must be unique in storage policy "
+ backQuote(name) + " (" + backQuote(volume->getName()) + " is duplicated)"
, ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
volume_index_by_volume_name[volume->getName()] = index;
for (const auto & disk : volume->getDisks())
if (disk->getName() == disk_ptr->getName())
return i;
{
const String & disk_name = disk->getName();
if (volume_index_by_disk_name.find(disk_name) != volume_index_by_disk_name.end())
throw Exception("Disk names must be unique in storage policy "
+ backQuote(name) + " (" + backQuote(disk_name) + " is duplicated)"
, ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
volume_index_by_disk_name[disk_name] = index;
}
}
throw Exception("No disk " + disk_ptr->getName() + " in policy " + name, ErrorCodes::UNKNOWN_DISK);
}
@ -242,44 +320,40 @@ StoragePolicySelector::StoragePolicySelector(
{
if (!std::all_of(name.begin(), name.end(), isWordCharASCII))
throw Exception(
"StoragePolicy name can contain only alphanumeric and '_' (" + name + ")", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
"Storage policy name can contain only alphanumeric and '_' (" + backQuote(name) + ")", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
policies.emplace(name, std::make_shared<StoragePolicy>(name, config, config_prefix + "." + name, disks));
LOG_INFO(&Poco::Logger::get("StoragePolicySelector"), "Storage policy {} loaded", backQuote(name));
}
constexpr auto default_storage_policy_name = "default";
constexpr auto default_volume_name = "default";
constexpr auto default_disk_name = "default";
/// Add default policy if it's not specified explicetly
if (policies.find(default_storage_policy_name) == policies.end())
/// Add default policy if it isn't explicitly specified.
if (policies.find(DEFAULT_STORAGE_POLICY_NAME) == policies.end())
{
auto default_volume = std::make_shared<VolumeJBOD>(default_volume_name, std::vector<DiskPtr>{disks->get(default_disk_name)}, 0);
auto default_policy = std::make_shared<StoragePolicy>(default_storage_policy_name, Volumes{default_volume}, 0.0);
policies.emplace(default_storage_policy_name, default_policy);
auto default_policy = std::make_shared<StoragePolicy>(DEFAULT_STORAGE_POLICY_NAME, config, config_prefix + "." + DEFAULT_STORAGE_POLICY_NAME, disks);
policies.emplace(DEFAULT_STORAGE_POLICY_NAME, std::move(default_policy));
}
}
StoragePolicySelectorPtr StoragePolicySelector::updateFromConfig(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, DiskSelectorPtr disks) const
{
Poco::Util::AbstractConfiguration::Keys keys;
config.keys(config_prefix, keys);
std::shared_ptr<StoragePolicySelector> result = std::make_shared<StoragePolicySelector>(config, config_prefix, disks);
constexpr auto default_storage_policy_name = "default";
/// First pass, check.
for (const auto & [name, policy] : policies)
{
if (name != default_storage_policy_name && result->policies.count(name) == 0)
if (result->policies.count(name) == 0)
throw Exception("Storage policy " + backQuote(name) + " is missing in new configuration", ErrorCodes::BAD_ARGUMENTS);
policy->checkCompatibleWith(result->policies[name]);
}
/// Second pass, load.
for (const auto & [name, policy] : policies)
{
result->policies[name] = std::make_shared<StoragePolicy>(*policy, config, config_prefix + "." + name, disks);
}
return result;
}
@ -288,7 +362,7 @@ StoragePolicyPtr StoragePolicySelector::get(const String & name) const
{
auto it = policies.find(name);
if (it == policies.end())
throw Exception("Unknown StoragePolicy " + name, ErrorCodes::UNKNOWN_POLICY);
throw Exception("Unknown storage policy " + backQuote(name), ErrorCodes::UNKNOWN_POLICY);
return it->second;
}

25
src/Disks/StoragePolicy.h
View File

@ -14,6 +14,7 @@
#include <memory>
#include <mutex>
#include <unordered_map>
#include <unistd.h>
#include <boost/noncopyable.hpp>
#include <Poco/Util/AbstractConfiguration.h>
@ -36,6 +37,13 @@ public:
StoragePolicy(String name_, Volumes volumes_, double move_factor_);
StoragePolicy(
const StoragePolicy & storage_policy,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
DiskSelectorPtr disks
);
bool isDefaultPolicy() const;
/// Returns disks ordered by volumes priority
@ -72,16 +80,10 @@ public:
/// which should be kept with help of background moves
double getMoveFactor() const { return move_factor; }
/// Get volume by index from storage_policy
VolumePtr getVolume(size_t i) const { return (i < volumes_names.size() ? volumes[i] : VolumePtr()); }
/// Get volume by index.
VolumePtr getVolume(size_t index) const;
VolumePtr getVolumeByName(const String & volume_name) const
{
auto it = volumes_names.find(volume_name);
if (it == volumes_names.end())
return {};
return getVolume(it->second);
}
VolumePtr getVolumeByName(const String & volume_name) const;
/// Checks if storage policy can be replaced by another one.
void checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const;
@ -89,12 +91,15 @@ public:
private:
Volumes volumes;
const String name;
std::map<String, size_t> volumes_names;
std::unordered_map<String, size_t> volume_index_by_volume_name;
std::unordered_map<String, size_t> volume_index_by_disk_name;
/// move_factor from interval [0., 1.]
/// We move something if disk from this policy
/// filled more than total_size * move_factor
double move_factor = 0.1; /// by default move factor is 10%
void buildVolumeIndices();
};

31
src/Disks/VolumeJBOD.cpp
View File

@ -56,11 +56,23 @@ VolumeJBOD::VolumeJBOD(
/// Default value is 'true' due to backward compatibility.
perform_ttl_move_on_insert = config.getBool(config_prefix + ".perform_ttl_move_on_insert", true);
are_merges_avoided = config.getBool(config_prefix + ".prefer_not_to_merge", false);
}
VolumeJBOD::VolumeJBOD(const VolumeJBOD & volume_jbod,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
DiskSelectorPtr disk_selector)
: VolumeJBOD(volume_jbod.name, config, config_prefix, disk_selector)
{
are_merges_avoided_user_override = volume_jbod.are_merges_avoided_user_override.load(std::memory_order_relaxed);
last_used = volume_jbod.last_used.load(std::memory_order_relaxed);
}
DiskPtr VolumeJBOD::getDisk(size_t /* index */) const
{
size_t start_from = last_used.fetch_add(1u, std::memory_order_relaxed);
size_t start_from = last_used.fetch_add(1u, std::memory_order_acq_rel);
size_t index = start_from % disks.size();
return disks[index];
}
@ -73,7 +85,7 @@ ReservationPtr VolumeJBOD::reserve(UInt64 bytes)
if (max_data_part_size != 0 && bytes > max_data_part_size)
return {};
size_t start_from = last_used.fetch_add(1u, std::memory_order_relaxed);
size_t start_from = last_used.fetch_add(1u, std::memory_order_acq_rel);
size_t disks_num = disks.size();
for (size_t i = 0; i < disks_num; ++i)
{
@ -87,4 +99,19 @@ ReservationPtr VolumeJBOD::reserve(UInt64 bytes)
return {};
}
bool VolumeJBOD::areMergesAvoided() const
{
auto are_merges_avoided_user_override_value = are_merges_avoided_user_override.load(std::memory_order_acquire);
if (are_merges_avoided_user_override_value)
return *are_merges_avoided_user_override_value;
else
return are_merges_avoided;
}
void VolumeJBOD::setAvoidMergesUserOverride(bool avoid)
{
are_merges_avoided_user_override.store(avoid, std::memory_order_release);
}
}

33
src/Disks/VolumeJBOD.h
View File

@ -1,10 +1,19 @@
#pragma once
#include <memory>
#include <optional>
#include <Disks/IVolume.h>
namespace DB
{
class VolumeJBOD;
using VolumeJBODPtr = std::shared_ptr<VolumeJBOD>;
using VolumesJBOD = std::vector<VolumeJBODPtr>;
/**
* Implements something similar to JBOD (https://en.wikipedia.org/wiki/Non-RAID_drive_architectures#JBOD).
* When MergeTree engine wants to write part — it requests VolumeJBOD to reserve space on the next available
@ -13,8 +22,9 @@ namespace DB
class VolumeJBOD : public IVolume
{
public:
VolumeJBOD(String name_, Disks disks_, UInt64 max_data_part_size_)
VolumeJBOD(String name_, Disks disks_, UInt64 max_data_part_size_, bool are_merges_avoided_)
: IVolume(name_, disks_, max_data_part_size_)
, are_merges_avoided(are_merges_avoided_)
{
}
@ -25,6 +35,13 @@ public:
DiskSelectorPtr disk_selector
);
VolumeJBOD(
const VolumeJBOD & volume_jbod,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
DiskSelectorPtr disk_selector
);
VolumeType getType() const override { return VolumeType::JBOD; }
/// Always returns next disk (round-robin), ignores argument.
@ -38,11 +55,19 @@ public:
/// Returns valid reservation or nullptr if there is no space left on any disk.
ReservationPtr reserve(UInt64 bytes) override;
bool areMergesAvoided() const override;
void setAvoidMergesUserOverride(bool avoid) override;
/// True if parts on this volume participate in merges according to configuration.
bool are_merges_avoided = true;
private:
/// Index of last used disk.
mutable std::atomic<size_t> last_used = 0;
/// True if parts on this volume participate in merges according to START/STOP MERGES ON VOLUME.
std::atomic<std::optional<bool>> are_merges_avoided_user_override{std::nullopt};
};
using VolumeJBODPtr = std::shared_ptr<VolumeJBOD>;
using VolumesJBOD = std::vector<VolumeJBODPtr>;
}

22
src/Disks/VolumeRAID1.h
View File

@ -3,18 +3,23 @@
#include <Disks/createVolume.h>
#include <Disks/VolumeJBOD.h>
namespace DB
{
/// Volume which reserserves space on each underlying disk.
class VolumeRAID1;
using VolumeRAID1Ptr = std::shared_ptr<VolumeRAID1>;
/// Volume which reserves space on each underlying disk.
///
/// NOTE: Just interface implementation, doesn't used in codebase,
/// also not available for user.
class VolumeRAID1 : public VolumeJBOD
{
public:
VolumeRAID1(String name_, Disks disks_, UInt64 max_data_part_size_)
: VolumeJBOD(name_, disks_, max_data_part_size_)
VolumeRAID1(String name_, Disks disks_, UInt64 max_data_part_size_, bool are_merges_avoided_in_config_)
: VolumeJBOD(name_, disks_, max_data_part_size_, are_merges_avoided_in_config_)
{
}
@ -27,11 +32,18 @@ public:
{
}
VolumeRAID1(
VolumeRAID1 & volume_raid1,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
DiskSelectorPtr disk_selector)
: VolumeJBOD(volume_raid1, config, config_prefix, disk_selector)
{
}
VolumeType getType() const override { return VolumeType::RAID1; }
ReservationPtr reserve(UInt64 bytes) override;
};
using VolumeRAID1Ptr = std::shared_ptr<VolumeRAID1>;
}

33
src/Disks/createVolume.cpp
View File

@ -12,6 +12,7 @@ namespace DB
namespace ErrorCodes
{
extern const int UNKNOWN_RAID_TYPE;
extern const int INVALID_RAID_TYPE;
}
VolumePtr createVolumeFromReservation(const ReservationPtr & reservation, VolumePtr other_volume)
@ -20,12 +21,12 @@ VolumePtr createVolumeFromReservation(const ReservationPtr & reservation, Volume
{
/// Since reservation on JBOD chooses one of disks and makes reservation there, volume
/// for such type of reservation will be with one disk.
return std::make_shared<SingleDiskVolume>(other_volume->getName(), reservation->getDisk());
return std::make_shared<SingleDiskVolume>(other_volume->getName(), reservation->getDisk(), other_volume->max_data_part_size);
}
if (other_volume->getType() == VolumeType::RAID1)
{
auto volume = std::dynamic_pointer_cast<VolumeRAID1>(other_volume);
return std::make_shared<VolumeRAID1>(volume->getName(), reservation->getDisks(), volume->max_data_part_size);
return std::make_shared<VolumeRAID1>(volume->getName(), reservation->getDisks(), volume->max_data_part_size, volume->are_merges_avoided);
}
return nullptr;
}
@ -37,17 +38,31 @@ VolumePtr createVolumeFromConfig(
DiskSelectorPtr disk_selector
)
{
auto has_raid_type = config.has(config_prefix + ".raid_type");
if (!has_raid_type)
{
return std::make_shared<VolumeJBOD>(name, config, config_prefix, disk_selector);
}
String raid_type = config.getString(config_prefix + ".raid_type");
String raid_type = config.getString(config_prefix + ".raid_type", "JBOD");
if (raid_type == "JBOD")
{
return std::make_shared<VolumeJBOD>(name, config, config_prefix, disk_selector);
}
throw Exception("Unknown raid type '" + raid_type + "'", ErrorCodes::UNKNOWN_RAID_TYPE);
throw Exception("Unknown RAID type '" + raid_type + "'", ErrorCodes::UNKNOWN_RAID_TYPE);
}
VolumePtr updateVolumeFromConfig(
VolumePtr volume,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
DiskSelectorPtr & disk_selector
)
{
String raid_type = config.getString(config_prefix + ".raid_type", "JBOD");
if (raid_type == "JBOD")
{
VolumeJBODPtr volume_jbod = std::dynamic_pointer_cast<VolumeJBOD>(volume);
if (!volume_jbod)
throw Exception("Invalid RAID type '" + raid_type + "', shall be JBOD", ErrorCodes::INVALID_RAID_TYPE);
return std::make_shared<VolumeJBOD>(*volume_jbod, config, config_prefix, disk_selector);
}
throw Exception("Unknown RAID type '" + raid_type + "'", ErrorCodes::UNKNOWN_RAID_TYPE);
}
}

8
src/Disks/createVolume.h
View File

@ -6,6 +6,7 @@ namespace DB
{
VolumePtr createVolumeFromReservation(const ReservationPtr & reservation, VolumePtr other_volume);
VolumePtr createVolumeFromConfig(
String name_,
const Poco::Util::AbstractConfiguration & config,
@ -13,4 +14,11 @@ VolumePtr createVolumeFromConfig(
DiskSelectorPtr disk_selector
);
VolumePtr updateVolumeFromConfig(
VolumePtr volume,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
DiskSelectorPtr & disk_selector
);
}

2
src/Formats/FormatFactory.cpp
View File

@ -49,6 +49,7 @@ static FormatSettings getInputFormatSetting(const Settings & settings, const Con
format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes;
format_settings.csv.unquoted_null_literal_as_null = settings.input_format_csv_unquoted_null_literal_as_null;
format_settings.csv.empty_as_default = settings.input_format_defaults_for_omitted_fields;
format_settings.csv.input_format_enum_as_number = settings.input_format_csv_enum_as_number;
format_settings.null_as_default = settings.input_format_null_as_default;
format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions;
format_settings.values.deduce_templates_of_expressions = settings.input_format_values_deduce_templates_of_expressions;
@ -63,6 +64,7 @@ static FormatSettings getInputFormatSetting(const Settings & settings, const Con
format_settings.template_settings.row_format = settings.format_template_row;
format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter;
format_settings.tsv.empty_as_default = settings.input_format_tsv_empty_as_default;
format_settings.tsv.input_format_enum_as_number = settings.input_format_tsv_enum_as_number;
format_settings.schema.format_schema = settings.format_schema;
format_settings.schema.format_schema_path = context.getFormatSchemaPath();
format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER);

2
src/Formats/FormatSettings.h
View File

@ -34,6 +34,7 @@ struct FormatSettings
bool unquoted_null_literal_as_null = false;
bool empty_as_default = false;
bool crlf_end_of_line = false;
bool input_format_enum_as_number = false;
};
CSV csv;
@ -81,6 +82,7 @@ struct FormatSettings
bool empty_as_default = false;
bool crlf_end_of_line = false;
String null_representation = "\\N";
bool input_format_enum_as_number = false;
};
TSV tsv;

14
src/Functions/CustomWeekTransforms.h
View File

@ -92,7 +92,7 @@ struct ToStartOfWeekImpl
template <typename FromType, typename ToType, typename Transform>
struct Transformer
{
Transformer(Transform transform_)
explicit Transformer(Transform transform_)
: transform(std::move(transform_))
{}
@ -116,29 +116,29 @@ template <typename FromDataType, typename ToDataType>
struct CustomWeekTransformImpl
{
template <typename Transform>
static void execute(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/, Transform transform = {})
static ColumnPtr execute(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/, Transform transform = {})
{
const auto op = Transformer<typename FromDataType::FieldType, typename ToDataType::FieldType, Transform>{std::move(transform)};
UInt8 week_mode = DEFAULT_WEEK_MODE;
if (arguments.size() > 1)
{
if (const auto week_mode_column = checkAndGetColumnConst<ColumnUInt8>(columns[arguments[1]].column.get()))
if (const auto * week_mode_column = checkAndGetColumnConst<ColumnUInt8>(arguments[1].column.get()))
week_mode = week_mode_column->getValue<UInt8>();
}
const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(columns, arguments, 2, 0);
const ColumnPtr source_col = columns[arguments[0]].column;
const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(arguments, 2, 0);
const ColumnPtr source_col = arguments[0].column;
if (const auto * sources = checkAndGetColumn<typename FromDataType::ColumnType>(source_col.get()))
{
auto col_to = ToDataType::ColumnType::create();
op.vector(sources->getData(), col_to->getData(), week_mode, time_zone);
columns[result].column = std::move(col_to);
return col_to;
}
else
{
throw Exception(
"Illegal column " + columns[arguments[0]].column->getName() + " of first argument of function "
"Illegal column " + arguments[0].column->getName() + " of first argument of function "
+ Transform::name,
ErrorCodes::ILLEGAL_COLUMN);
}

12
src/Functions/DateTimeTransforms.h
View File

@ -683,25 +683,25 @@ struct Transformer
template <typename FromDataType, typename ToDataType, typename Transform>
struct DateTimeTransformImpl
{
static void execute(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/, const Transform & transform = {})
static ColumnPtr execute(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/, const Transform & transform = {})
{
using Op = Transformer<typename FromDataType::FieldType, typename ToDataType::FieldType, Transform>;
const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(columns, arguments, 1, 0);
const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(arguments, 1, 0);
const ColumnPtr source_col = columns[arguments[0]].column;
const ColumnPtr source_col = arguments[0].column;
if (const auto * sources = checkAndGetColumn<typename FromDataType::ColumnType>(source_col.get()))
{
auto mutable_result_col = columns[result].type->createColumn();
auto mutable_result_col = result_type->createColumn();
auto * col_to = assert_cast<typename ToDataType::ColumnType *>(mutable_result_col.get());
Op::vector(sources->getData(), col_to->getData(), time_zone, transform);
columns[result].column = std::move(mutable_result_col);
return mutable_result_col;
}
else
{
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of first argument of function " + Transform::name,
ErrorCodes::ILLEGAL_COLUMN);
}

16
src/Functions/FunctionBase64Conversion.h
View File

@ -91,14 +91,14 @@ public:
return std::make_shared<DataTypeString>();
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const ColumnPtr column_string = columns[arguments[0]].column;
const ColumnPtr column_string = arguments[0].column;
const ColumnString * input = checkAndGetColumn<ColumnString>(column_string.get());
if (!input)
throw Exception(
"Illegal column " + columns[arguments[0]].column->getName() + " of first argument of function " + getName(),
"Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
auto dst_column = ColumnString::create();
@ -111,9 +111,9 @@ public:
const ColumnString::Offsets & src_offsets = input->getOffsets();
auto source = input->getChars().data();
auto dst = dst_data.data();
auto dst_pos = dst;
const auto * source = input->getChars().data();
auto * dst = dst_data.data();
auto * dst_pos = dst;
size_t src_offset_prev = 0;
@ -141,7 +141,7 @@ public:
{
// during decoding character array can be partially polluted
// if fail, revert back and clean
auto savepoint = dst_pos;
auto * savepoint = dst_pos;
outlen = _tb64d(reinterpret_cast<const uint8_t *>(source), srclen, reinterpret_cast<uint8_t *>(dst_pos));
if (!outlen)
{
@ -166,7 +166,7 @@ public:
dst_data.resize(dst_pos - dst);
columns[result].column = std::move(dst_column);
return dst_column;
}
};
}

165
src/Functions/FunctionBinaryArithmetic.h
View File

@ -613,17 +613,17 @@ class FunctionBinaryArithmetic : public IFunction
}
/// Multiply aggregation state by integer constant: by merging it with itself specified number of times.
void executeAggregateMultiply(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const
ColumnPtr executeAggregateMultiply(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const
{
ColumnNumbers new_arguments = arguments;
if (WhichDataType(columns[new_arguments[1]].type).isAggregateFunction())
ColumnsWithTypeAndName new_arguments = arguments;
if (WhichDataType(new_arguments[1].type).isAggregateFunction())
std::swap(new_arguments[0], new_arguments[1]);
if (!isColumnConst(*columns[new_arguments[1]].column))
throw Exception{"Illegal column " + columns[new_arguments[1]].column->getName()
if (!isColumnConst(*new_arguments[1].column))
throw Exception{"Illegal column " + new_arguments[1].column->getName()
+ " of argument of aggregation state multiply. Should be integer constant", ErrorCodes::ILLEGAL_COLUMN};
const IColumn & agg_state_column = *columns[new_arguments[0]].column;
const IColumn & agg_state_column = *new_arguments[0].column;
bool agg_state_is_const = isColumnConst(agg_state_column);
const ColumnAggregateFunction & column = typeid_cast<const ColumnAggregateFunction &>(
agg_state_is_const ? assert_cast<const ColumnConst &>(agg_state_column).getDataColumn() : agg_state_column);
@ -647,7 +647,7 @@ class FunctionBinaryArithmetic : public IFunction
auto & vec_to = column_to->getData();
auto & vec_from = column_from->getData();
UInt64 m = typeid_cast<const ColumnConst *>(columns[new_arguments[1]].column.get())->getValue<UInt64>();
UInt64 m = typeid_cast<const ColumnConst *>(new_arguments[1].column.get())->getValue<UInt64>();
// Since we merge the function states by ourselves, we have to have an
// Arena for this. Pass it to the resulting column so that the arena
@ -674,16 +674,16 @@ class FunctionBinaryArithmetic : public IFunction
}
if (agg_state_is_const)
columns[result].column = ColumnConst::create(std::move(column_to), input_rows_count);
return ColumnConst::create(std::move(column_to), input_rows_count);
else
columns[result].column = std::move(column_to);
return column_to;
}
/// Merge two aggregation states together.
void executeAggregateAddition(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const
ColumnPtr executeAggregateAddition(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const
{
const IColumn & lhs_column = *columns[arguments[0]].column;
const IColumn & rhs_column = *columns[arguments[1]].column;
const IColumn & lhs_column = *arguments[0].column;
const IColumn & rhs_column = *arguments[1].column;
bool lhs_is_const = isColumnConst(lhs_column);
bool rhs_is_const = isColumnConst(rhs_column);
@ -707,37 +707,33 @@ class FunctionBinaryArithmetic : public IFunction
}
if (lhs_is_const && rhs_is_const)
columns[result].column = ColumnConst::create(std::move(column_to), input_rows_count);
return ColumnConst::create(std::move(column_to), input_rows_count);
else
columns[result].column = std::move(column_to);
return column_to;
}
void executeDateTimeIntervalPlusMinus(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments,
size_t result, size_t input_rows_count, const FunctionOverloadResolverPtr & function_builder) const
ColumnPtr executeDateTimeIntervalPlusMinus(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type,
size_t input_rows_count, const FunctionOverloadResolverPtr & function_builder) const
{
ColumnNumbers new_arguments = arguments;
ColumnsWithTypeAndName new_arguments = arguments;
/// Interval argument must be second.
if (WhichDataType(columns[arguments[1]].type).isDateOrDateTime())
if (WhichDataType(arguments[1].type).isDateOrDateTime())
std::swap(new_arguments[0], new_arguments[1]);
/// Change interval argument type to its representation
ColumnsWithTypeAndName new_columns = columns;
new_columns[new_arguments[1]].type = std::make_shared<DataTypeNumber<DataTypeInterval::FieldType>>();
new_arguments[1].type = std::make_shared<DataTypeNumber<DataTypeInterval::FieldType>>();
ColumnsWithTypeAndName new_arguments_with_type_and_name =
{new_columns[new_arguments[0]], new_columns[new_arguments[1]]};
auto function = function_builder->build(new_arguments_with_type_and_name);
auto function = function_builder->build(new_arguments);
function->execute(new_columns, new_arguments, result, input_rows_count);
columns[result].column = new_columns[result].column;
return function->execute(new_arguments, result_type, input_rows_count);
}
public:
static constexpr auto name = Name::name;
static FunctionPtr create(const Context & context) { return std::make_shared<FunctionBinaryArithmetic>(context); }
FunctionBinaryArithmetic(const Context & context_)
explicit FunctionBinaryArithmetic(const Context & context_)
: context(context_),
check_decimal_overflow(decimalCheckArithmeticOverflow(context))
{}
@ -790,7 +786,7 @@ public:
new_arguments[1].type = std::make_shared<DataTypeNumber<DataTypeInterval::FieldType>>();
auto function = function_builder->build(new_arguments);
return function->getReturnType();
return function->getResultType();
}
DataTypePtr type_res;
@ -851,20 +847,20 @@ public:
return type_res;
}
bool executeFixedString(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result) const
ColumnPtr executeFixedString(ColumnsWithTypeAndName & arguments) const
{
using OpImpl = FixedStringOperationImpl<Op<UInt8, UInt8>>;
auto col_left_raw = columns[arguments[0]].column.get();
auto col_right_raw = columns[arguments[1]].column.get();
if (auto col_left_const = checkAndGetColumnConst<ColumnFixedString>(col_left_raw))
const auto * col_left_raw = arguments[0].column.get();
const auto * col_right_raw = arguments[1].column.get();
if (const auto * col_left_const = checkAndGetColumnConst<ColumnFixedString>(col_left_raw))
{
if (auto col_right_const = checkAndGetColumnConst<ColumnFixedString>(col_right_raw))
if (const auto * col_right_const = checkAndGetColumnConst<ColumnFixedString>(col_right_raw))
{
auto col_left = checkAndGetColumn<ColumnFixedString>(col_left_const->getDataColumn());
auto col_right = checkAndGetColumn<ColumnFixedString>(col_right_const->getDataColumn());
const auto * col_left = checkAndGetColumn<ColumnFixedString>(col_left_const->getDataColumn());
const auto * col_right = checkAndGetColumn<ColumnFixedString>(col_right_const->getDataColumn());
if (col_left->getN() != col_right->getN())
return false;
return nullptr;
auto col_res = ColumnFixedString::create(col_left->getN());
auto & out_chars = col_res->getChars();
out_chars.resize(col_left->getN());
@ -872,25 +868,24 @@ public:
col_right->getChars().data(),
out_chars.data(),
out_chars.size());
columns[result].column = ColumnConst::create(std::move(col_res), col_left_raw->size());
return true;
return ColumnConst::create(std::move(col_res), col_left_raw->size());
}
}
bool is_left_column_const = checkAndGetColumnConst<ColumnFixedString>(col_left_raw) != nullptr;
bool is_right_column_const = checkAndGetColumnConst<ColumnFixedString>(col_right_raw) != nullptr;
auto col_left = is_left_column_const
const auto * col_left = is_left_column_const
? checkAndGetColumn<ColumnFixedString>(checkAndGetColumnConst<ColumnFixedString>(col_left_raw)->getDataColumn())
: checkAndGetColumn<ColumnFixedString>(col_left_raw);
auto col_right = is_right_column_const
const auto * col_right = is_right_column_const
? checkAndGetColumn<ColumnFixedString>(checkAndGetColumnConst<ColumnFixedString>(col_right_raw)->getDataColumn())
: checkAndGetColumn<ColumnFixedString>(col_right_raw);
if (col_left && col_right)
{
if (col_left->getN() != col_right->getN())
return false;
return nullptr;
auto col_res = ColumnFixedString::create(col_left->getN());
auto & out_chars = col_res->getChars();
@ -922,14 +917,13 @@ public:
out_chars.size(),
col_left->getN());
}
columns[result].column = std::move(col_res);
return true;
return col_res;
}
return false;
return nullptr;
}
template <typename A, typename B>
bool executeNumeric(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result [[maybe_unused]], const A & left, const B & right) const
ColumnPtr executeNumeric(ColumnsWithTypeAndName & arguments, const A & left, const B & right) const
{
using LeftDataType = std::decay_t<decltype(left)>;
using RightDataType = std::decay_t<decltype(right)>;
@ -944,8 +938,8 @@ public:
using ColVecT1 = std::conditional_t<IsDecimalNumber<T1>, ColumnDecimal<T1>, ColumnVector<T1>>;
using ColVecResult = std::conditional_t<IsDecimalNumber<ResultType>, ColumnDecimal<ResultType>, ColumnVector<ResultType>>;
auto col_left_raw = columns[arguments[0]].column.get();
auto col_right_raw = columns[arguments[1]].column.get();
const auto * col_left_raw = arguments[0].column.get();
const auto * col_right_raw = arguments[1].column.get();
auto col_left_const = checkAndGetColumnConst<ColVecT0>(col_left_raw);
auto col_right_const = checkAndGetColumnConst<ColVecT1>(col_right_raw);
@ -981,9 +975,8 @@ public:
OpImplCheck::template constantConstant<dec_a, dec_b>(const_a, const_b, scale_a, scale_b) :
OpImpl::template constantConstant<dec_a, dec_b>(const_a, const_b, scale_a, scale_b);
columns[result].column = ResultDataType(type.getPrecision(), type.getScale()).createColumnConst(
return ResultDataType(type.getPrecision(), type.getScale()).createColumnConst(
col_left_const->size(), toField(res, type.getScale()));
return true;
}
col_res = ColVecResult::create(0, type.getScale());
@ -1016,7 +1009,7 @@ public:
OpImpl::template vectorConstant<dec_a, dec_b>(col_left->getData(), const_b, vec_res, scale_a, scale_b);
}
else
return false;
return nullptr;
}
else
{
@ -1026,8 +1019,7 @@ public:
if (col_left_const && col_right_const)
{
auto res = OpImpl::constantConstant(col_left_const->template getValue<T0>(), col_right_const->template getValue<T1>());
columns[result].column = ResultDataType().createColumnConst(col_left_const->size(), toField(res));
return true;
return ResultDataType().createColumnConst(col_left_const->size(), toField(res));
}
col_res = ColVecResult::create();
@ -1047,43 +1039,40 @@ public:
OpImpl::vectorConstant(col_left->getData().data(), col_right_const->template getValue<T1>(), vec_res.data(), vec_res.size());
}
else
return false;
return nullptr;
}
columns[result].column = std::move(col_res);
return true;
return col_res;
}
return false;
return nullptr;
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
/// Special case when multiply aggregate function state
if (isAggregateMultiply(columns[arguments[0]].type, columns[arguments[1]].type))
if (isAggregateMultiply(arguments[0].type, arguments[1].type))
{
executeAggregateMultiply(columns, arguments, result, input_rows_count);
return;
return executeAggregateMultiply(arguments, result_type, input_rows_count);
}
/// Special case - addition of two aggregate functions states
if (isAggregateAddition(columns[arguments[0]].type, columns[arguments[1]].type))
if (isAggregateAddition(arguments[0].type, arguments[1].type))
{
executeAggregateAddition(columns, arguments, result, input_rows_count);
return;
return executeAggregateAddition(arguments, result_type, input_rows_count);
}
/// Special case when the function is plus or minus, one of arguments is Date/DateTime and another is Interval.
if (auto function_builder
= getFunctionForIntervalArithmetic(columns[arguments[0]].type, columns[arguments[1]].type, context))
= getFunctionForIntervalArithmetic(arguments[0].type, arguments[1].type, context))
{
executeDateTimeIntervalPlusMinus(columns, arguments, result, input_rows_count, function_builder);
return;
return executeDateTimeIntervalPlusMinus(arguments, result_type, input_rows_count, function_builder);
}
const auto & left_argument = columns[arguments[0]];
const auto & right_argument = columns[arguments[1]];
auto * left_generic = left_argument.type.get();
auto * right_generic = right_argument.type.get();
const auto & left_argument = arguments[0];
const auto & right_argument = arguments[1];
const auto * left_generic = left_argument.type.get();
const auto * right_generic = right_argument.type.get();
ColumnPtr res;
bool valid = castBothTypes(left_generic, right_generic, [&](const auto & left, const auto & right)
{
using LeftDataType = std::decay_t<decltype(left)>;
@ -1093,10 +1082,10 @@ public:
if constexpr (!Op<DataTypeFixedString, DataTypeFixedString>::allow_fixed_string)
return false;
else
return executeFixedString(columns, arguments, result);
return (res = executeFixedString(arguments)) != nullptr;
}
else
return executeNumeric(columns, arguments, result, left, right);
return (res = executeNumeric(arguments, left, right)) != nullptr;
});
if (!valid)
@ -1109,6 +1098,8 @@ public:
left_argument.name, left_argument.type->getName(),
right_argument.name, right_argument.type->getName());
}
return res;
}
#if USE_EMBEDDED_COMPILER
@ -1190,30 +1181,26 @@ public:
{
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
if (left.column && isColumnConst(*left.column) && arguments.size() == 1)
{
ColumnsWithTypeAndName columns_with_constant
= {{left.column->cloneResized(input_rows_count), left.type, left.name},
columns[arguments[0]],
columns[result]};
arguments[0]};
Base::executeImpl(columns_with_constant, {0, 1}, 2, input_rows_count);
columns[result] = columns_with_constant[2];
return Base::executeImpl(columns_with_constant, result_type, input_rows_count);
}
else if (right.column && isColumnConst(*right.column) && arguments.size() == 1)
{
ColumnsWithTypeAndName columns_with_constant
= {columns[arguments[0]],
{right.column->cloneResized(input_rows_count), right.type, right.name},
columns[result]};
= {arguments[0],
{right.column->cloneResized(input_rows_count), right.type, right.name}};
Base::executeImpl(columns_with_constant, {0, 1}, 2, input_rows_count);
columns[result] = columns_with_constant[2];
return Base::executeImpl(columns_with_constant, result_type, input_rows_count);
}
else
Base::executeImpl(columns, arguments, result, input_rows_count);
return Base::executeImpl(arguments, result_type, input_rows_count);
}
bool hasInformationAboutMonotonicity() const override
@ -1246,12 +1233,11 @@ public:
{
ColumnsWithTypeAndName columns_with_constant
= {{left.column->cloneResized(1), left.type, left.name},
{right.type->createColumnConst(1, point), right.type, right.name},
{nullptr, return_type, ""}};
{right.type->createColumnConst(1, point), right.type, right.name}};
Base::executeImpl(columns_with_constant, {0, 1}, 2, 1);
auto col = Base::executeImpl(columns_with_constant, return_type, 1);
Field point_transformed;
columns_with_constant[2].column->get(0, point_transformed);
col->get(0, point_transformed);
return point_transformed;
};
transform(left_point);
@ -1282,12 +1268,11 @@ public:
{
ColumnsWithTypeAndName columns_with_constant
= {{left.type->createColumnConst(1, point), left.type, left.name},
{right.column->cloneResized(1), right.type, right.name},
{nullptr, return_type, ""}};
{right.column->cloneResized(1), right.type, right.name}};
Base::executeImpl(columns_with_constant, {0, 1}, 2, 1);
auto col = Base::executeImpl(columns_with_constant, return_type, 1);
Field point_transformed;
columns_with_constant[2].column->get(0, point_transformed);
col->get(0, point_transformed);
return point_transformed;
};

56
src/Functions/FunctionBitTestMany.h
View File

@ -54,32 +54,35 @@ public:
return std::make_shared<DataTypeUInt8>();
}
void executeImpl(ColumnsWithTypeAndName & columns , const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
{
const auto value_col = columns[arguments.front()].column.get();
const auto * value_col = arguments.front().column.get();
if (!execute<UInt8>(columns, arguments, result, value_col)
&& !execute<UInt16>(columns, arguments, result, value_col)
&& !execute<UInt32>(columns, arguments, result, value_col)
&& !execute<UInt64>(columns, arguments, result, value_col)
&& !execute<Int8>(columns, arguments, result, value_col)
&& !execute<Int16>(columns, arguments, result, value_col)
&& !execute<Int32>(columns, arguments, result, value_col)
&& !execute<Int64>(columns, arguments, result, value_col))
ColumnPtr res;
if (!((res = execute<UInt8>(arguments, result_type, value_col))
|| (res = execute<UInt16>(arguments, result_type, value_col))
|| (res = execute<UInt32>(arguments, result_type, value_col))
|| (res = execute<UInt64>(arguments, result_type, value_col))
|| (res = execute<Int8>(arguments, result_type, value_col))
|| (res = execute<Int16>(arguments, result_type, value_col))
|| (res = execute<Int32>(arguments, result_type, value_col))
|| (res = execute<Int64>(arguments, result_type, value_col))))
throw Exception{"Illegal column " + value_col->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
return res;
}
private:
template <typename T>
bool execute(
ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, const size_t result,
ColumnPtr execute(
ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type,
const IColumn * const value_col_untyped) const
{
if (const auto value_col = checkAndGetColumn<ColumnVector<T>>(value_col_untyped))
{
const auto size = value_col->size();
bool is_const;
const auto const_mask = createConstMaskIfConst<T>(columns, arguments, is_const);
const auto const_mask = createConstMaskIfConst<T>(arguments, is_const);
const auto & val = value_col->getData();
auto out_col = ColumnVector<UInt8>::create(size);
@ -92,29 +95,28 @@ private:
}
else
{
const auto mask = createMask<T>(size, columns, arguments);
const auto mask = createMask<T>(size, arguments);
for (const auto i : ext::range(0, size))
out[i] = Impl::apply(val[i], mask[i]);
}
columns[result].column = std::move(out_col);
return true;
return out_col;
}
else if (const auto value_col_const = checkAndGetColumnConst<ColumnVector<T>>(value_col_untyped))
{
const auto size = value_col_const->size();
bool is_const;
const auto const_mask = createConstMaskIfConst<T>(columns, arguments, is_const);
const auto const_mask = createConstMaskIfConst<T>(arguments, is_const);
const auto val = value_col_const->template getValue<T>();
if (is_const)
{
columns[result].column = columns[result].type->createColumnConst(size, toField(Impl::apply(val, const_mask)));
return result_type->createColumnConst(size, toField(Impl::apply(val, const_mask)));
}
else
{
const auto mask = createMask<T>(size, columns, arguments);
const auto mask = createMask<T>(size, arguments);
auto out_col = ColumnVector<UInt8>::create(size);
auto & out = out_col->getData();
@ -122,24 +124,22 @@ private:
for (const auto i : ext::range(0, size))
out[i] = Impl::apply(val, mask[i]);
columns[result].column = std::move(out_col);
return out_col;
}
}
return true;
}
return false;
return nullptr;
}
template <typename ValueType>
ValueType createConstMaskIfConst(const ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, bool & out_is_const) const
ValueType createConstMaskIfConst(const ColumnsWithTypeAndName & arguments, bool & out_is_const) const
{
out_is_const = true;
ValueType mask = 0;
for (const auto i : ext::range(1, arguments.size()))
{
if (auto pos_col_const = checkAndGetColumnConst<ColumnVector<ValueType>>(columns[arguments[i]].column.get()))
if (auto pos_col_const = checkAndGetColumnConst<ColumnVector<ValueType>>(arguments[i].column.get()))
{
const auto pos = pos_col_const->getUInt(0);
if (pos < 8 * sizeof(ValueType))
@ -156,13 +156,13 @@ private:
}
template <typename ValueType>
PaddedPODArray<ValueType> createMask(const size_t size, const ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments) const
PaddedPODArray<ValueType> createMask(const size_t size, const ColumnsWithTypeAndName & arguments) const
{
PaddedPODArray<ValueType> mask(size, ValueType{});
for (const auto i : ext::range(1, arguments.size()))
{
const auto pos_col = columns[arguments[i]].column.get();
const auto * pos_col = arguments[i].column.get();
if (!addToMaskImpl<UInt8>(mask, pos_col)
&& !addToMaskImpl<UInt16>(mask, pos_col)

18
src/Functions/FunctionCustomWeekToSomething.h
View File

@ -96,26 +96,26 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
const IDataType * from_type = columns[arguments[0]].type.get();
const IDataType * from_type = arguments[0].type.get();
WhichDataType which(from_type);
if (which.isDate())
CustomWeekTransformImpl<DataTypeDate, ToDataType>::execute(
columns, arguments, result, input_rows_count, Transform{});
return CustomWeekTransformImpl<DataTypeDate, ToDataType>::execute(
arguments, result_type, input_rows_count, Transform{});
else if (which.isDateTime())
CustomWeekTransformImpl<DataTypeDateTime, ToDataType>::execute(
columns, arguments, result, input_rows_count, Transform{});
return CustomWeekTransformImpl<DataTypeDateTime, ToDataType>::execute(
arguments, result_type, input_rows_count, Transform{});
else if (which.isDateTime64())
{
CustomWeekTransformImpl<DataTypeDateTime64, ToDataType>::execute(
columns, arguments, result, input_rows_count,
return CustomWeekTransformImpl<DataTypeDateTime64, ToDataType>::execute(
arguments, result_type, input_rows_count,
TransformDateTime64<Transform>{assert_cast<const DataTypeDateTime64 *>(from_type)->getScale()});
}
else
throw Exception(
"Illegal type " + columns[arguments[0]].type->getName() + " of argument of function " + getName(),
"Illegal type " + arguments[0].type->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}

34
src/Functions/FunctionDateOrDateTimeAddInterval.h
View File

@ -305,7 +305,7 @@ private:
template <typename FromDataType, typename ToDataType, typename Transform>
struct DateTimeAddIntervalImpl
{
static void execute(Transform transform, ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result)
static ColumnPtr execute(Transform transform, ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type)
{
using FromValueType = typename FromDataType::FieldType;
using FromColumnType = typename FromDataType::ColumnType;
@ -313,16 +313,16 @@ struct DateTimeAddIntervalImpl
auto op = Adder<Transform>{std::move(transform)};
const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(columns, arguments, 2, 0);
const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(arguments, 2, 0);
const ColumnPtr source_col = columns[arguments[0]].column;
const ColumnPtr source_col = arguments[0].column;
auto result_col = columns[result].type->createColumn();
auto result_col = result_type->createColumn();
auto col_to = assert_cast<ToColumnType *>(result_col.get());
if (const auto * sources = checkAndGetColumn<FromColumnType>(source_col.get()))
{
const IColumn & delta_column = *columns[arguments[1]].column;
const IColumn & delta_column = *arguments[1].column;
if (const auto * delta_const_column = typeid_cast<const ColumnConst *>(&delta_column))
op.vectorConstant(sources->getData(), col_to->getData(), delta_const_column->getInt(0), time_zone);
@ -334,16 +334,16 @@ struct DateTimeAddIntervalImpl
op.constantVector(
sources_const->template getValue<FromValueType>(),
col_to->getData(),
*columns[arguments[1]].column, time_zone);
*arguments[1].column, time_zone);
}
else
{
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of first argument of function " + Transform::name,
ErrorCodes::ILLEGAL_COLUMN);
}
columns[result].column = std::move(result_col);
return result_col;
}
};
@ -463,28 +463,28 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {2}; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
{
const IDataType * from_type = columns[arguments[0]].type.get();
const IDataType * from_type = arguments[0].type.get();
WhichDataType which(from_type);
if (which.isDate())
{
DateTimeAddIntervalImpl<DataTypeDate, TransformResultDataType<DataTypeDate>, Transform>::execute(
Transform{}, columns, arguments, result);
return DateTimeAddIntervalImpl<DataTypeDate, TransformResultDataType<DataTypeDate>, Transform>::execute(
Transform{}, arguments, result_type);
}
else if (which.isDateTime())
{
DateTimeAddIntervalImpl<DataTypeDateTime, TransformResultDataType<DataTypeDateTime>, Transform>::execute(
Transform{}, columns, arguments, result);
return DateTimeAddIntervalImpl<DataTypeDateTime, TransformResultDataType<DataTypeDateTime>, Transform>::execute(
Transform{}, arguments, result_type);
}
else if (const auto * datetime64_type = assert_cast<const DataTypeDateTime64 *>(from_type))
{
DateTimeAddIntervalImpl<DataTypeDateTime64, TransformResultDataType<DataTypeDateTime64>, Transform>::execute(
Transform{datetime64_type->getScale()}, columns, arguments, result);
return DateTimeAddIntervalImpl<DataTypeDateTime64, TransformResultDataType<DataTypeDateTime64>, Transform>::execute(
Transform{datetime64_type->getScale()}, arguments, result_type);
}
else
throw Exception("Illegal type " + columns[arguments[0]].type->getName() + " of first argument of function " + getName(),
throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
};

12
src/Functions/FunctionDateOrDateTimeToSomething.h
View File

@ -95,23 +95,23 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
const IDataType * from_type = columns[arguments[0]].type.get();
const IDataType * from_type = arguments[0].type.get();
WhichDataType which(from_type);
if (which.isDate())
DateTimeTransformImpl<DataTypeDate, ToDataType, Transform>::execute(columns, arguments, result, input_rows_count);
return DateTimeTransformImpl<DataTypeDate, ToDataType, Transform>::execute(arguments, result_type, input_rows_count);
else if (which.isDateTime())
DateTimeTransformImpl<DataTypeDateTime, ToDataType, Transform>::execute(columns, arguments, result, input_rows_count);
return DateTimeTransformImpl<DataTypeDateTime, ToDataType, Transform>::execute(arguments, result_type, input_rows_count);
else if (which.isDateTime64())
{
const auto scale = static_cast<const DataTypeDateTime64 *>(from_type)->getScale();
const TransformDateTime64<Transform> transformer(scale);
DateTimeTransformImpl<DataTypeDateTime64, ToDataType, decltype(transformer)>::execute(columns, arguments, result, input_rows_count, transformer);
return DateTimeTransformImpl<DataTypeDateTime64, ToDataType, decltype(transformer)>::execute(arguments, result_type, input_rows_count, transformer);
}
else
throw Exception("Illegal type " + columns[arguments[0]].type->getName() + " of argument of function " + getName(),
throw Exception("Illegal type " + arguments[0].type->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}

4
src/Functions/FunctionFQDN.cpp
View File

@ -34,9 +34,9 @@ public:
return std::make_shared<DataTypeString>();
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers &, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName &, const DataTypePtr & result_type, size_t input_rows_count) const override
{
columns[result].column = columns[result].type->createColumnConst(
return result_type->createColumnConst(
input_rows_count, getFQDNOrHostName())->convertToFullColumnIfConst();
}
};

27
src/Functions/FunctionHelpers.cpp
View File

@ -7,7 +7,6 @@
#include <Columns/ColumnLowCardinality.h>
#include <Common/assert_cast.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeLowCardinality.h>
namespace DB
@ -51,16 +50,12 @@ Columns convertConstTupleToConstantElements(const ColumnConst & column)
}
static ColumnsWithTypeAndName createBlockWithNestedColumnsImpl(const ColumnsWithTypeAndName & columns, const std::unordered_set<size_t> & args)
ColumnsWithTypeAndName createBlockWithNestedColumns(const ColumnsWithTypeAndName & columns)
{
ColumnsWithTypeAndName res;
size_t num_columns = columns.size();
for (size_t i = 0; i < num_columns; ++i)
for (const auto & col : columns)
{
const auto & col = columns[i];
if (args.count(i) && col.type->isNullable())
if (col.type->isNullable())
{
const DataTypePtr & nested_type = static_cast<const DataTypeNullable &>(*col.type).getNestedType();
@ -88,20 +83,6 @@ static ColumnsWithTypeAndName createBlockWithNestedColumnsImpl(const ColumnsWith
return res;
}
ColumnsWithTypeAndName createBlockWithNestedColumns(const ColumnsWithTypeAndName & columns, const ColumnNumbers & args)
{
std::unordered_set<size_t> args_set(args.begin(), args.end());
return createBlockWithNestedColumnsImpl(columns, args_set);
}
ColumnsWithTypeAndName createBlockWithNestedColumns(const ColumnsWithTypeAndName & columns, const ColumnNumbers & args, size_t result)
{
std::unordered_set<size_t> args_set(args.begin(), args.end());
args_set.insert(result);
return createBlockWithNestedColumnsImpl(columns, args_set);
}
void validateArgumentType(const IFunction & func, const DataTypes & arguments,
size_t argument_index, bool (* validator_func)(const IDataType &),
const char * expected_type_description)
@ -137,7 +118,7 @@ void validateArgumentsImpl(const IFunction & func,
const auto & arg = arguments[i + argument_offset];
const auto descriptor = descriptors[i];
if (int error_code = descriptor.isValid(arg.type, arg.column); error_code != 0)
throw Exception("Illegal type of argument #" + std::to_string(i)
throw Exception("Illegal type of argument #" + std::to_string(argument_offset + i + 1) // +1 is for human-friendly 1-based indexing
+ (descriptor.argument_name ? " '" + std::string(descriptor.argument_name) + "'" : String{})
+ " of function " + func.getName()
+ (descriptor.expected_type_description ? String(", expected ") + descriptor.expected_type_description : String{})

8
src/Functions/FunctionHelpers.h
View File

@ -82,13 +82,9 @@ inline std::enable_if_t<IsDecimalNumber<T>, Field> toField(const T & x, UInt32 s
Columns convertConstTupleToConstantElements(const ColumnConst & column);
/// Returns the copy of a given columns in which each column specified in
/// the "arguments" parameter is replaced with its respective nested
/// Returns the copy of a given columns in which each column is replaced with its respective nested
/// column if it is nullable.
ColumnsWithTypeAndName createBlockWithNestedColumns(const ColumnsWithTypeAndName & columns, const ColumnNumbers & args);
/// Similar function as above. Additionally transform the result type if needed.
ColumnsWithTypeAndName createBlockWithNestedColumns(const ColumnsWithTypeAndName & columns, const ColumnNumbers & args, size_t result);
ColumnsWithTypeAndName createBlockWithNestedColumns(const ColumnsWithTypeAndName & columns);
/// Checks argument type at specified index with predicate.
/// throws if there is no argument at specified index or if predicate returns false.

8
src/Functions/FunctionJoinGet.cpp
View File

@ -17,19 +17,19 @@ namespace ErrorCodes
}
template <bool or_null>
void ExecutableFunctionJoinGet<or_null>::execute(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t)
ColumnPtr ExecutableFunctionJoinGet<or_null>::execute(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t)
{
ColumnsWithTypeAndName keys;
for (size_t i = 2; i < arguments.size(); ++i)
{
auto key = columns[arguments[i]];
auto key = arguments[i];
keys.emplace_back(std::move(key));
}
columns[result] = join->joinGet(keys, result_columns);
return join->joinGet(keys, result_columns).column;
}
template <bool or_null>
ExecutableFunctionImplPtr FunctionJoinGet<or_null>::prepare(const ColumnsWithTypeAndName &, const ColumnNumbers &, size_t) const
ExecutableFunctionImplPtr FunctionJoinGet<or_null>::prepare(const ColumnsWithTypeAndName &) const
{
return std::make_unique<ExecutableFunctionJoinGet<or_null>>(join, DB::Block{{return_type->createColumn(), return_type, attr_name}});
}

6
src/Functions/FunctionJoinGet.h
View File

@ -24,7 +24,7 @@ public:
bool useDefaultImplementationForLowCardinalityColumns() const override { return true; }
bool useDefaultImplementationForConstants() const override { return true; }
void execute(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override;
ColumnPtr execute(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) override;
String getName() const override { return name; }
@ -54,9 +54,9 @@ public:
String getName() const override { return name; }
const DataTypes & getArgumentTypes() const override { return argument_types; }
const DataTypePtr & getReturnType() const override { return return_type; }
const DataTypePtr & getResultType() const override { return return_type; }
ExecutableFunctionImplPtr prepare(const ColumnsWithTypeAndName & sample_columns, const ColumnNumbers & arguments, size_t result) const override;
ExecutableFunctionImplPtr prepare(const ColumnsWithTypeAndName &) const override;
private:
TableLockHolder table_lock;

30
src/Functions/FunctionMathBinaryFloat64.h
View File

@ -54,7 +54,7 @@ private:
}
template <typename LeftType, typename RightType>
bool executeTyped(ColumnsWithTypeAndName & columns, const size_t result, const ColumnConst * left_arg, const IColumn * right_arg) const
ColumnPtr executeTyped(const ColumnConst * left_arg, const IColumn * right_arg) const
{
if (const auto right_arg_typed = checkAndGetColumn<ColumnVector<RightType>>(right_arg))
{
@ -95,15 +95,14 @@ private:
memcpy(&dst_data[rows_size], dst_remaining, rows_remaining * sizeof(Float64));
}
columns[result].column = std::move(dst);
return true;
return dst;
}
return false;
return nullptr;
}
template <typename LeftType, typename RightType>
bool executeTyped(ColumnsWithTypeAndName & columns, const size_t result, const ColumnVector<LeftType> * left_arg, const IColumn * right_arg) const
ColumnPtr executeTyped(const ColumnVector<LeftType> * left_arg, const IColumn * right_arg) const
{
if (const auto right_arg_typed = checkAndGetColumn<ColumnVector<RightType>>(right_arg))
{
@ -157,8 +156,7 @@ private:
memcpy(&dst_data[rows_size], dst_remaining, rows_remaining * sizeof(Float64));
}
columns[result].column = std::move(dst);
return true;
return dst;
}
if (const auto right_arg_typed = checkAndGetColumnConst<ColumnVector<RightType>>(right_arg))
{
@ -200,17 +198,17 @@ private:
memcpy(&dst_data[rows_size], dst_remaining, rows_remaining * sizeof(Float64));
}
columns[result].column = std::move(dst);
return true;
return dst;
}
return false;
return nullptr;
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const ColumnWithTypeAndName & col_left = columns[arguments[0]];
const ColumnWithTypeAndName & col_right = columns[arguments[1]];
const ColumnWithTypeAndName & col_left = arguments[0];
const ColumnWithTypeAndName & col_right = arguments[1];
ColumnPtr res;
auto call = [&](const auto & types) -> bool
{
@ -224,7 +222,7 @@ private:
if (const auto left_arg_typed = checkAndGetColumn<ColVecLeft>(left_arg))
{
if (executeTyped<LeftType, RightType>(columns, result, left_arg_typed, right_arg))
if ((res = executeTyped<LeftType, RightType>(left_arg_typed, right_arg)))
return true;
throw Exception{"Illegal column " + right_arg->getName() + " of second argument of function " + getName(),
@ -232,7 +230,7 @@ private:
}
if (const auto left_arg_typed = checkAndGetColumnConst<ColVecLeft>(left_arg))
{
if (executeTyped<LeftType, RightType>(columns, result, left_arg_typed, right_arg))
if ((res = executeTyped<LeftType, RightType>(left_arg_typed, right_arg)))
return true;
throw Exception{"Illegal column " + right_arg->getName() + " of second argument of function " + getName(),
@ -248,6 +246,8 @@ private:
if (!callOnBasicTypes<true, true, false, false>(left_index, right_index, call))
throw Exception{"Illegal column " + col_left.column->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
return res;
}
};

4
src/Functions/FunctionMathConstFloat64.h
View File

@ -25,9 +25,9 @@ private:
return std::make_shared<DataTypeFloat64>();
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers &, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName &, const DataTypePtr & result_type, size_t input_rows_count) const override
{
columns[result].column = columns[result].type->createColumnConst(input_rows_count, Impl::value);
return result_type->createColumnConst(input_rows_count, Impl::value);
}
};

19
src/Functions/FunctionMathUnary.h
View File

@ -113,7 +113,7 @@ private:
}
template <typename T, typename ReturnType>
static bool execute(ColumnsWithTypeAndName & columns, const ColumnVector<T> * col, const size_t result)
static ColumnPtr execute(const ColumnVector<T> * col)
{
const auto & src_data = col->getData();
const size_t size = src_data.size();
@ -124,12 +124,11 @@ private:
executeInIterations(src_data.data(), dst_data.data(), size);
columns[result].column = std::move(dst);
return true;
return dst;
}
template <typename T, typename ReturnType>
static bool execute(ColumnsWithTypeAndName & columns, const ColumnDecimal<T> * col, const size_t result)
static ColumnPtr execute(const ColumnDecimal<T> * col)
{
const auto & src_data = col->getData();
const size_t size = src_data.size();
@ -144,15 +143,15 @@ private:
executeInIterations(dst_data.data(), dst_data.data(), size);
columns[result].column = std::move(dst);
return true;
return dst;
}
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const ColumnWithTypeAndName & col = columns[arguments[0]];
const ColumnWithTypeAndName & col = arguments[0];
ColumnPtr res;
auto call = [&](const auto & types) -> bool
{
@ -162,12 +161,14 @@ private:
using ColVecType = std::conditional_t<IsDecimalNumber<Type>, ColumnDecimal<Type>, ColumnVector<Type>>;
const auto col_vec = checkAndGetColumn<ColVecType>(col.column.get());
return execute<Type, ReturnType>(columns, col_vec, result);
return (res = execute<Type, ReturnType>(col_vec)) != nullptr;
};
if (!callOnBasicType<void, true, true, true, false>(col.type->getTypeId(), call))
throw Exception{"Illegal column " + col.column->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
return res;
}
};

34
src/Functions/FunctionNumericPredicate.h
View File

@ -46,25 +46,28 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const auto in = columns[arguments.front()].column.get();
const auto * in = arguments.front().column.get();
if ( !execute<UInt8>(columns, in, result)
&& !execute<UInt16>(columns, in, result)
&& !execute<UInt32>(columns, in, result)
&& !execute<UInt64>(columns, in, result)
&& !execute<Int8>(columns, in, result)
&& !execute<Int16>(columns, in, result)
&& !execute<Int32>(columns, in, result)
&& !execute<Int64>(columns, in, result)
&& !execute<Float32>(columns, in, result)
&& !execute<Float64>(columns, in, result))
ColumnPtr res;
if (!((res = execute<UInt8>(in))
|| (res = execute<UInt16>(in))
|| (res = execute<UInt32>(in))
|| (res = execute<UInt64>(in))
|| (res = execute<Int8>(in))
|| (res = execute<Int16>(in))
|| (res = execute<Int32>(in))
|| (res = execute<Int64>(in))
|| (res = execute<Float32>(in))
|| (res = execute<Float64>(in))))
throw Exception{"Illegal column " + in->getName() + " of first argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
return res;
}
template <typename T>
bool execute(ColumnsWithTypeAndName & columns, const IColumn * in_untyped, const size_t result) const
ColumnPtr execute(const IColumn * in_untyped) const
{
if (const auto in = checkAndGetColumn<ColumnVector<T>>(in_untyped))
{
@ -78,11 +81,10 @@ public:
for (const auto i : ext::range(0, size))
out_data[i] = Impl::execute(in_data[i]);
columns[result].column = std::move(out);
return true;
return out;
}
return false;
return nullptr;
}
};

12
src/Functions/FunctionStartsEndsWith.h
View File

@ -63,10 +63,10 @@ public:
return std::make_shared<DataTypeUInt8>();
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const IColumn * haystack_column = columns[arguments[0]].column.get();
const IColumn * needle_column = columns[arguments[1]].column.get();
const IColumn * haystack_column = arguments[0].column.get();
const IColumn * needle_column = arguments[1].column.get();
auto col_res = ColumnVector<UInt8>::create();
typename ColumnVector<UInt8>::Container & vec_res = col_res->getData();
@ -84,7 +84,7 @@ public:
else
throw Exception("Illegal combination of columns as arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN);
columns[result].column = std::move(col_res);
return col_res;
}
private:
@ -159,9 +159,9 @@ public:
#endif
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
selector.selectAndExecute(columns, arguments, result, input_rows_count);
return selector.selectAndExecute(arguments, result_type, input_rows_count);
}
static FunctionPtr create(const Context & context)

14
src/Functions/FunctionStringOrArrayToT.h
View File

@ -50,9 +50,9 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
{
const ColumnPtr column = columns[arguments[0]].column;
const ColumnPtr column = arguments[0].column;
if (const ColumnString * col = checkAndGetColumn<ColumnString>(column.get()))
{
auto col_res = ColumnVector<ResultType>::create();
@ -61,7 +61,7 @@ public:
vec_res.resize(col->size());
Impl::vector(col->getChars(), col->getOffsets(), vec_res);
columns[result].column = std::move(col_res);
return col_res;
}
else if (const ColumnFixedString * col_fixed = checkAndGetColumn<ColumnFixedString>(column.get()))
{
@ -70,7 +70,7 @@ public:
ResultType res = 0;
Impl::vectorFixedToConstant(col_fixed->getChars(), col_fixed->getN(), res);
columns[result].column = columns[result].type->createColumnConst(col_fixed->size(), toField(res));
return result_type->createColumnConst(col_fixed->size(), toField(res));
}
else
{
@ -80,7 +80,7 @@ public:
vec_res.resize(col_fixed->size());
Impl::vectorFixedToVector(col_fixed->getChars(), col_fixed->getN(), vec_res);
columns[result].column = std::move(col_res);
return col_res;
}
}
else if (const ColumnArray * col_arr = checkAndGetColumn<ColumnArray>(column.get()))
@ -91,10 +91,10 @@ public:
vec_res.resize(col_arr->size());
Impl::array(col_arr->getOffsets(), vec_res);
columns[result].column = std::move(col_res);
return col_res;
}
else
throw Exception("Illegal column " + columns[arguments[0]].column->getName() + " of argument of function " + getName(),
throw Exception("Illegal column " + arguments[0].column->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
};

18
src/Functions/FunctionStringReplace.h
View File

@ -52,17 +52,17 @@ public:
return std::make_shared<DataTypeString>();
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const ColumnPtr column_src = columns[arguments[0]].column;
const ColumnPtr column_needle = columns[arguments[1]].column;
const ColumnPtr column_replacement = columns[arguments[2]].column;
const ColumnPtr column_src = arguments[0].column;
const ColumnPtr column_needle = arguments[1].column;
const ColumnPtr column_replacement = arguments[2].column;
if (!isColumnConst(*column_needle) || !isColumnConst(*column_replacement))
throw Exception("2nd and 3rd arguments of function " + getName() + " must be constants.", ErrorCodes::ILLEGAL_COLUMN);
const IColumn * c1 = columns[arguments[1]].column.get();
const IColumn * c2 = columns[arguments[2]].column.get();
const IColumn * c1 = arguments[1].column.get();
const IColumn * c2 = arguments[2].column.get();
const ColumnConst * c1_const = typeid_cast<const ColumnConst *>(c1);
const ColumnConst * c2_const = typeid_cast<const ColumnConst *>(c2);
String needle = c1_const->getValue<String>();
@ -75,17 +75,17 @@ public:
{
auto col_res = ColumnString::create();
Impl::vector(col->getChars(), col->getOffsets(), needle, replacement, col_res->getChars(), col_res->getOffsets());
columns[result].column = std::move(col_res);
return col_res;
}
else if (const ColumnFixedString * col_fixed = checkAndGetColumn<ColumnFixedString>(column_src.get()))
{
auto col_res = ColumnString::create();
Impl::vectorFixed(col_fixed->getChars(), col_fixed->getN(), needle, replacement, col_res->getChars(), col_res->getOffsets());
columns[result].column = std::move(col_res);
return col_res;
}
else
throw Exception(
"Illegal column " + columns[arguments[0]].column->getName() + " of first argument of function " + getName(),
"Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
};

10
src/Functions/FunctionStringToString.h
View File

@ -52,24 +52,24 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const ColumnPtr column = columns[arguments[0]].column;
const ColumnPtr column = arguments[0].column;
if (const ColumnString * col = checkAndGetColumn<ColumnString>(column.get()))
{
auto col_res = ColumnString::create();
Impl::vector(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets());
columns[result].column = std::move(col_res);
return col_res;
}
else if (const ColumnFixedString * col_fixed = checkAndGetColumn<ColumnFixedString>(column.get()))
{
auto col_res = ColumnFixedString::create(col_fixed->getN());
Impl::vectorFixed(col_fixed->getChars(), col_fixed->getN(), col_res->getChars());
columns[result].column = std::move(col_res);
return col_res;
}
else
throw Exception(
"Illegal column " + columns[arguments[0]].column->getName() + " of argument of function " + getName(),
"Illegal column " + arguments[0].column->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
};

19
src/Functions/FunctionUnaryArithmetic.h
View File

@ -154,9 +154,10 @@ public:
return result;
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
bool valid = castType(columns[arguments[0]].type.get(), [&](const auto & type)
ColumnPtr result_column;
bool valid = castType(arguments[0].type.get(), [&](const auto & type)
{
using DataType = std::decay_t<decltype(type)>;
@ -164,13 +165,13 @@ public:
{
if constexpr (allow_fixed_string)
{
if (auto col = checkAndGetColumn<ColumnFixedString>(columns[arguments[0]].column.get()))
if (const auto * col = checkAndGetColumn<ColumnFixedString>(arguments[0].column.get()))
{
auto col_res = ColumnFixedString::create(col->getN());
auto & vec_res = col_res->getChars();
vec_res.resize(col->size() * col->getN());
FixedStringUnaryOperationImpl<Op<UInt8>>::vector(col->getChars(), vec_res);
columns[result].column = std::move(col_res);
result_column = std::move(col_res);
return true;
}
}
@ -180,13 +181,13 @@ public:
using T0 = typename DataType::FieldType;
if constexpr (allow_decimal)
{
if (auto col = checkAndGetColumn<ColumnDecimal<T0>>(columns[arguments[0]].column.get()))
if (auto col = checkAndGetColumn<ColumnDecimal<T0>>(arguments[0].column.get()))
{
auto col_res = ColumnDecimal<typename Op<T0>::ResultType>::create(0, type.getScale());
auto & vec_res = col_res->getData();
vec_res.resize(col->getData().size());
UnaryOperationImpl<T0, Op<T0>>::vector(col->getData(), vec_res);
columns[result].column = std::move(col_res);
result_column = std::move(col_res);
return true;
}
}
@ -194,13 +195,13 @@ public:
else
{
using T0 = typename DataType::FieldType;
if (auto col = checkAndGetColumn<ColumnVector<T0>>(columns[arguments[0]].column.get()))
if (auto col = checkAndGetColumn<ColumnVector<T0>>(arguments[0].column.get()))
{
auto col_res = ColumnVector<typename Op<T0>::ResultType>::create();
auto & vec_res = col_res->getData();
vec_res.resize(col->getData().size());
UnaryOperationImpl<T0, Op<T0>>::vector(col->getData(), vec_res);
columns[result].column = std::move(col_res);
result_column = std::move(col_res);
return true;
}
}
@ -209,6 +210,8 @@ public:
});
if (!valid)
throw Exception(getName() + "'s argument does not match the expected data type", ErrorCodes::LOGICAL_ERROR);
return result_column;
}
#if USE_EMBEDDED_COMPILER

15
src/Functions/FunctionUnixTimestamp64.h
View File

@ -65,13 +65,12 @@ public:
}
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
using SourceColumnType = typename SourceDataType::ColumnType;
using ResultColumnType = typename ResultDataType::ColumnType;
const auto & src = columns[arguments[0]];
auto & res = columns[result];
const auto & src = arguments[0];
const auto & col = *src.column;
const SourceColumnType * source_col_typed = checkAndGetColumn<SourceColumnType>(col);
@ -80,16 +79,16 @@ public:
+ std::string(SourceDataType::family_name),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
res.column = res.type->createColumn();
auto res_column = result_type->createColumn();
if (input_rows_count == 0)
return;
return res_column;
auto & result_data = assert_cast<ResultColumnType &>(res.column->assumeMutableRef()).getData();
auto & result_data = assert_cast<ResultColumnType &>(res_column->assumeMutableRef()).getData();
result_data.reserve(source_col_typed->size());
const auto & source_data = source_col_typed->getData();
const auto scale_diff = getScaleDiff(*checkAndGetDataType<SourceDataType>(src.type.get()), *checkAndGetDataType<ResultDataType>(res.type.get()));
const auto scale_diff = getScaleDiff(*checkAndGetDataType<SourceDataType>(src.type.get()), *checkAndGetDataType<ResultDataType>(result_type.get()));
if (scale_diff == 0)
{
static_assert(sizeof(typename SourceColumnType::Container::value_type) == sizeof(typename ResultColumnType::Container::value_type));
@ -114,6 +113,8 @@ public:
for (const auto & v : source_data)
result_data.push_back(static_cast<Int64>(toDestValue(v) / scale_multiplier));
}
return res_column;
}
private:

63
src/Functions/FunctionsAES.cpp Normal file
View File

@ -0,0 +1,63 @@
#include <Functions/FunctionsAES.h>
#if USE_SSL
#include <openssl/evp.h>
#include <openssl/err.h>
#include <string>
#include <cassert>
namespace DB
{
namespace ErrorCodes
{
extern const int OPENSSL_ERROR;
}
}
namespace OpenSSLDetails
{
void onError(std::string error_message)
{
error_message += ". OpenSSL error code: " + std::to_string(ERR_get_error());
throw DB::Exception(error_message, DB::ErrorCodes::OPENSSL_ERROR);
}
StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array<char, EVP_MAX_KEY_LENGTH> & folded_key)
{
assert(cipher_key_size <= EVP_MAX_KEY_LENGTH);
memcpy(folded_key.data(), key.data, cipher_key_size);
for (size_t i = cipher_key_size; i < key.size; ++i)
{
folded_key[i % cipher_key_size] ^= key.data[i];
}
return StringRef(folded_key.data(), cipher_key_size);
}
const EVP_CIPHER * getCipherByName(const StringRef & cipher_name)
{
const auto * evp_cipher = EVP_get_cipherbyname(cipher_name.data);
if (evp_cipher == nullptr)
{
// For some reasons following ciphers can't be found by name.
if (cipher_name == "aes-128-cfb128")
evp_cipher = EVP_aes_128_cfb128();
else if (cipher_name == "aes-192-cfb128")
evp_cipher = EVP_aes_192_cfb128();
else if (cipher_name == "aes-256-cfb128")
evp_cipher = EVP_aes_256_cfb128();
}
// NOTE: cipher obtained not via EVP_CIPHER_fetch() would cause extra work on each context reset
// with EVP_CIPHER_CTX_reset() or EVP_EncryptInit_ex(), but using EVP_CIPHER_fetch()
// causes data race, so we stick to the slower but safer alternative here.
return evp_cipher;
}
}
#endif

693
src/Functions/FunctionsAES.h Normal file
View File

@ -0,0 +1,693 @@
#pragma once
#if !defined(ARCADIA_BUILD)
# include <Common/config.h>
#endif
#if USE_SSL
#include <DataTypes/DataTypeString.h>
#include <Columns/ColumnString.h>
#include <Functions/IFunction.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionHelpers.h>
#include <fmt/format.h>
#include <openssl/evp.h>
#include <openssl/engine.h>
#include <string_view>
#include <functional>
#include <initializer_list>
#include <string.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
}
namespace OpenSSLDetails
{
[[noreturn]] void onError(std::string error_message);
StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array<char, EVP_MAX_KEY_LENGTH> & folded_key);
const EVP_CIPHER * getCipherByName(const StringRef & name);
enum class CompatibilityMode
{
MySQL,
OpenSSL
};
enum class CipherMode
{
MySQLCompatibility, // with key folding
OpenSSLCompatibility, // just as regular openssl's enc application does (AEAD modes, like GCM and CCM are not supported)
RFC5116_AEAD_AES_GCM // AEAD GCM with custom IV length and tag (HMAC) appended to the ciphertext, see https://tools.ietf.org/html/rfc5116#section-5.1
};
template <CipherMode mode>
struct KeyHolder
{
inline StringRef setKey(size_t cipher_key_size, const StringRef & key) const
{
if (key.size != cipher_key_size)
throw DB::Exception(fmt::format("Invalid key size: {} expected {}", key.size, cipher_key_size),
DB::ErrorCodes::BAD_ARGUMENTS);
return key;
}
};
template <>
struct KeyHolder<CipherMode::MySQLCompatibility>
{
inline StringRef setKey(size_t cipher_key_size, const StringRef & key)
{
if (key.size < cipher_key_size)
throw DB::Exception(fmt::format("Invalid key size: {} expected {}", key.size, cipher_key_size),
DB::ErrorCodes::BAD_ARGUMENTS);
// MySQL does something fancy with the keys that are too long,
// ruining compatibility with OpenSSL and not improving security.
// But we have to do the same to be compatitable with MySQL.
// see https://github.com/mysql/mysql-server/blob/8.0/router/src/harness/src/my_aes_openssl.cc#L71
// (my_aes_create_key function)
return foldEncryptionKeyInMySQLCompatitableMode(cipher_key_size, key, folded_key);
}
~KeyHolder()
{
OPENSSL_cleanse(folded_key.data(), folded_key.size());
}
private:
std::array<char, EVP_MAX_KEY_LENGTH> folded_key;
};
template <CompatibilityMode compatibility_mode>
inline void validateCipherMode(const EVP_CIPHER * evp_cipher)
{
if constexpr (compatibility_mode == CompatibilityMode::MySQL)
{
switch (EVP_CIPHER_mode(evp_cipher))
{
case EVP_CIPH_ECB_MODE: [[fallthrough]];
case EVP_CIPH_CBC_MODE: [[fallthrough]];
case EVP_CIPH_CFB_MODE: [[fallthrough]];
case EVP_CIPH_OFB_MODE:
return;
}
}
else if constexpr (compatibility_mode == CompatibilityMode::OpenSSL)
{
switch (EVP_CIPHER_mode(evp_cipher))
{
case EVP_CIPH_ECB_MODE: [[fallthrough]];
case EVP_CIPH_CBC_MODE: [[fallthrough]];
case EVP_CIPH_CFB_MODE: [[fallthrough]];
case EVP_CIPH_OFB_MODE: [[fallthrough]];
case EVP_CIPH_CTR_MODE: [[fallthrough]];
case EVP_CIPH_GCM_MODE:
return;
}
}
throw DB::Exception("Unsupported cipher mode " + std::string(EVP_CIPHER_name(evp_cipher)), DB::ErrorCodes::BAD_ARGUMENTS);
}
template <CipherMode mode>
inline void validateIV(const StringRef & iv_value, const size_t cipher_iv_size)
{
// In MySQL mode we don't care if IV is longer than expected, only if shorter.
if ((mode == CipherMode::MySQLCompatibility && iv_value.size != 0 && iv_value.size < cipher_iv_size)
|| (mode == CipherMode::OpenSSLCompatibility && iv_value.size != 0 && iv_value.size != cipher_iv_size))
throw DB::Exception(fmt::format("Invalid IV size: {} expected {}", iv_value.size, cipher_iv_size),
DB::ErrorCodes::BAD_ARGUMENTS);
}
}
namespace DB
{
template <typename Impl>
class FunctionEncrypt : public IFunction
{
public:
static constexpr OpenSSLDetails::CompatibilityMode compatibility_mode = Impl::compatibility_mode;
static constexpr auto name = Impl::name;
static FunctionPtr create(const Context &) { return std::make_shared<FunctionEncrypt>(); }
private:
using CipherMode = OpenSSLDetails::CipherMode;
String getName() const override { return name; }
bool isVariadic() const override { return true; }
size_t getNumberOfArguments() const override { return 0; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
auto optional_args = FunctionArgumentDescriptors{
{"IV", isStringOrFixedString, nullptr, "Initialization vector binary string"},
};
if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::OpenSSL)
{
optional_args.emplace_back(FunctionArgumentDescriptor{
"AAD", isStringOrFixedString, nullptr, "Additional authenticated data binary string for GCM mode"
});
}
validateFunctionArgumentTypes(*this, arguments,
FunctionArgumentDescriptors{
{"mode", isStringOrFixedString, isColumnConst, "encryption mode string"},
{"input", nullptr, nullptr, "plaintext"},
{"key", isStringOrFixedString, nullptr, "encryption key binary string"},
},
optional_args
);
return std::make_shared<DataTypeString>();
}
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
using namespace OpenSSLDetails;
const auto mode = arguments[0].column->getDataAt(0);
if (mode.size == 0 || !std::string_view(mode).starts_with("aes-"))
throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS);
const auto * evp_cipher = getCipherByName(mode);
if (evp_cipher == nullptr)
throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS);
const auto cipher_mode = EVP_CIPHER_mode(evp_cipher);
const auto input_column = arguments[1].column;
const auto key_column = arguments[2].column;
OpenSSLDetails::validateCipherMode<compatibility_mode>(evp_cipher);
ColumnPtr result_column;
if (arguments.size() <= 3)
result_column = doEncrypt(evp_cipher, input_rows_count, input_column, key_column, nullptr, nullptr);
else
{
const auto iv_column = arguments[3].column;
if (compatibility_mode != OpenSSLDetails::CompatibilityMode::MySQL && EVP_CIPHER_iv_length(evp_cipher) == 0)
throw Exception(mode.toString() + " does not support IV", ErrorCodes::BAD_ARGUMENTS);
if (arguments.size() <= 4)
{
result_column = doEncrypt(evp_cipher, input_rows_count, input_column, key_column, iv_column, nullptr);
}
else
{
if (cipher_mode != EVP_CIPH_GCM_MODE)
throw Exception("AAD can be only set for GCM-mode", ErrorCodes::BAD_ARGUMENTS);
const auto aad_column = arguments[4].column;
result_column = doEncrypt(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column);
}
}
return result_column;
}
template <typename InputColumnType, typename KeyColumnType, typename IvColumnType, typename AadColumnType>
static ColumnPtr doEncrypt(const EVP_CIPHER * evp_cipher,
size_t input_rows_count,
const InputColumnType & input_column,
const KeyColumnType & key_column,
const IvColumnType & iv_column,
const AadColumnType & aad_column)
{
if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::MySQL)
{
return doEncryptImpl<CipherMode::MySQLCompatibility>(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column);
}
else
{
if (EVP_CIPHER_mode(evp_cipher) == EVP_CIPH_GCM_MODE)
{
return doEncryptImpl<CipherMode::RFC5116_AEAD_AES_GCM>(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column);
}
else
{
return doEncryptImpl<CipherMode::OpenSSLCompatibility>(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column);
}
}
return nullptr;
}
template <CipherMode mode, typename InputColumnType, typename KeyColumnType, typename IvColumnType, typename AadColumnType>
static ColumnPtr doEncryptImpl(const EVP_CIPHER * evp_cipher,
size_t input_rows_count,
const InputColumnType & input_column,
const KeyColumnType & key_column,
[[maybe_unused]] const IvColumnType & iv_column,
[[maybe_unused]] const AadColumnType & aad_column)
{
using namespace OpenSSLDetails;
auto evp_ctx_ptr = std::unique_ptr<EVP_CIPHER_CTX, decltype(&::EVP_CIPHER_CTX_free)>(EVP_CIPHER_CTX_new(), &EVP_CIPHER_CTX_free);
auto * evp_ctx = evp_ctx_ptr.get();
const auto block_size = static_cast<size_t>(EVP_CIPHER_block_size(evp_cipher));
const auto key_size = static_cast<size_t>(EVP_CIPHER_key_length(evp_cipher));
[[maybe_unused]] const auto iv_size = static_cast<size_t>(EVP_CIPHER_iv_length(evp_cipher));
const auto tag_size = 16; // https://tools.ietf.org/html/rfc5116#section-5.1
auto encrypted_result_column = ColumnString::create();
auto & encrypted_result_column_data = encrypted_result_column->getChars();
auto & encrypted_result_column_offsets = encrypted_result_column->getOffsets();
{
size_t resulting_size = 0;
// for modes with block_size > 1, plaintext is padded up to a block_size,
// which may result in allocating to much for block_size = 1.
// That may lead later to reading unallocated data from underlying PaddedPODArray
// due to assumption that it is safe to read up to 15 bytes past end.
const auto pad_to_next_block = block_size == 1 ? 0 : 1;
for (size_t r = 0; r < input_rows_count; ++r)
{
resulting_size += (input_column->getDataAt(r).size / block_size + pad_to_next_block) * block_size + 1;
if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM)
resulting_size += tag_size;
}
#if defined(MEMORY_SANITIZER)
encrypted_result_column_data.resize_fill(resulting_size, 0xFF);
#else
encrypted_result_column_data.resize(resulting_size);
#endif
}
auto * encrypted = encrypted_result_column_data.data();
KeyHolder<mode> key_holder;
for (size_t r = 0; r < input_rows_count; ++r)
{
const auto key_value = key_holder.setKey(key_size, key_column->getDataAt(r));
auto iv_value = StringRef{};
if constexpr (!std::is_same_v<std::nullptr_t, std::decay_t<IvColumnType>>)
{
iv_value = iv_column->getDataAt(r);
}
const auto input_value = input_column->getDataAt(r);
auto aad_value = StringRef{};
if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM && !std::is_same_v<std::nullptr_t, std::decay_t<AadColumnType>>)
{
aad_value = aad_column->getDataAt(r);
}
if constexpr (mode != CipherMode::MySQLCompatibility)
{
// in GCM mode IV can be of arbitrary size (>0), IV is optional for other modes.
if (mode == CipherMode::RFC5116_AEAD_AES_GCM && iv_value.size == 0)
{
throw Exception("Invalid IV size " + std::to_string(iv_value.size) + " != expected size " + std::to_string(iv_size),
DB::ErrorCodes::BAD_ARGUMENTS);
}
if (mode != CipherMode::RFC5116_AEAD_AES_GCM && key_value.size != key_size)
{
throw Exception("Invalid key size " + std::to_string(key_value.size) + " != expected size " + std::to_string(key_size),
DB::ErrorCodes::BAD_ARGUMENTS);
}
}
// Avoid extra work on empty ciphertext/plaintext for some ciphers
if (!(input_value.size == 0 && block_size == 1 && mode != CipherMode::RFC5116_AEAD_AES_GCM))
{
// 1: Init CTX
if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM)
{
// 1.a.1: Init CTX with custom IV length and optionally with AAD
if (EVP_EncryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr) != 1)
onError("Failed to initialize encryption context with cipher");
if (EVP_CIPHER_CTX_ctrl(evp_ctx, EVP_CTRL_AEAD_SET_IVLEN, iv_value.size, nullptr) != 1)
onError("Failed to set custom IV length to " + std::to_string(iv_value.size));
if (EVP_EncryptInit_ex(evp_ctx, nullptr, nullptr,
reinterpret_cast<const unsigned char*>(key_value.data),
reinterpret_cast<const unsigned char*>(iv_value.data)) != 1)
onError("Failed to set key and IV");
// 1.a.2 Set AAD
if constexpr (!std::is_same_v<std::nullptr_t, std::decay_t<AadColumnType>>)
{
const auto aad_data = aad_column->getDataAt(r);
int tmp_len = 0;
if (aad_data.size != 0 && EVP_EncryptUpdate(evp_ctx, nullptr, &tmp_len,
reinterpret_cast<const unsigned char *>(aad_data.data), aad_data.size) != 1)
onError("Failed to set AAD data");
}
}
else
{
// 1.b: Init CTX
validateIV<mode>(iv_value, iv_size);
if (EVP_EncryptInit_ex(evp_ctx, evp_cipher, nullptr,
reinterpret_cast<const unsigned char*>(key_value.data),
reinterpret_cast<const unsigned char*>(iv_value.data)) != 1)
onError("Failed to initialize cipher context");
}
int output_len = 0;
// 2: Feed the data to the cipher
if (EVP_EncryptUpdate(evp_ctx,
reinterpret_cast<unsigned char*>(encrypted), &output_len,
reinterpret_cast<const unsigned char*>(input_value.data), static_cast<int>(input_value.size)) != 1)
onError("Failed to encrypt");
encrypted += output_len;
// 3: retrieve encrypted data (ciphertext)
if (EVP_EncryptFinal_ex(evp_ctx,
reinterpret_cast<unsigned char*>(encrypted), &output_len) != 1)
onError("Failed to fetch ciphertext");
encrypted += output_len;
// 4: optionally retrieve a tag and append it to the ciphertext (RFC5116):
// https://tools.ietf.org/html/rfc5116#section-5.1
if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM)
{
if (EVP_CIPHER_CTX_ctrl(evp_ctx, EVP_CTRL_AEAD_GET_TAG, tag_size, encrypted) != 1)
onError("Failed to retrieve GCM tag");
encrypted += tag_size;
}
}
*encrypted = '\0';
++encrypted;
encrypted_result_column_offsets.push_back(encrypted - encrypted_result_column_data.data());
}
// in case of block size of 1, we overestimate buffer required for encrypted data, fix it up.
if (!encrypted_result_column_offsets.empty() && encrypted_result_column_data.size() > encrypted_result_column_offsets.back())
{
encrypted_result_column_data.resize(encrypted_result_column_offsets.back());
}
encrypted_result_column->validate();
return encrypted_result_column;
}
};
/// AES_decrypt(string, key, block_mode[, init_vector])
template <typename Impl>
class FunctionDecrypt : public IFunction
{
public:
static constexpr OpenSSLDetails::CompatibilityMode compatibility_mode = Impl::compatibility_mode;
static constexpr auto name = Impl::name;
static FunctionPtr create(const Context &) { return std::make_shared<FunctionDecrypt>(); }
private:
using CipherMode = OpenSSLDetails::CipherMode;
String getName() const override { return name; }
bool isVariadic() const override { return true; }
size_t getNumberOfArguments() const override { return 0; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
auto optional_args = FunctionArgumentDescriptors{
{"IV", isStringOrFixedString, nullptr, "Initialization vector binary string"},
};
if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::OpenSSL)
{
optional_args.emplace_back(FunctionArgumentDescriptor{
"AAD", isStringOrFixedString, nullptr, "Additional authenticated data binary string for GCM mode"
});
}
validateFunctionArgumentTypes(*this, arguments,
FunctionArgumentDescriptors{
{"mode", isStringOrFixedString, isColumnConst, "decryption mode string"},
{"input", nullptr, nullptr, "ciphertext"},
{"key", isStringOrFixedString, nullptr, "decryption key binary string"},
},
optional_args
);
return std::make_shared<DataTypeString>();
}
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
using namespace OpenSSLDetails;
const auto mode = arguments[0].column->getDataAt(0);
if (mode.size == 0 || !std::string_view(mode).starts_with("aes-"))
throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS);
const auto * evp_cipher = getCipherByName(mode);
if (evp_cipher == nullptr)
throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS);
OpenSSLDetails::validateCipherMode<compatibility_mode>(evp_cipher);
const auto input_column = arguments[1].column;
const auto key_column = arguments[2].column;
ColumnPtr result_column;
if (arguments.size() <= 3)
result_column = doDecrypt(evp_cipher, input_rows_count, input_column, key_column, nullptr, nullptr);
else
{
const auto iv_column = arguments[3].column;
if (compatibility_mode != OpenSSLDetails::CompatibilityMode::MySQL && EVP_CIPHER_iv_length(evp_cipher) == 0)
throw Exception(mode.toString() + " does not support IV", ErrorCodes::BAD_ARGUMENTS);
if (arguments.size() <= 4)
{
result_column = doDecrypt(evp_cipher, input_rows_count, input_column, key_column, iv_column, nullptr);
}
else
{
if (EVP_CIPHER_mode(evp_cipher) != EVP_CIPH_GCM_MODE)
throw Exception("AAD can be only set for GCM-mode", ErrorCodes::BAD_ARGUMENTS);
const auto aad_column = arguments[4].column;
result_column = doDecrypt(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column);
}
}
return result_column;
}
template <typename InputColumnType, typename KeyColumnType, typename IvColumnType, typename AadColumnType>
static ColumnPtr doDecrypt(const EVP_CIPHER * evp_cipher,
size_t input_rows_count,
const InputColumnType & input_column,
const KeyColumnType & key_column,
const IvColumnType & iv_column,
const AadColumnType & aad_column)
{
if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::MySQL)
{
return doDecryptImpl<CipherMode::MySQLCompatibility>(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column);
}
else
{
const auto cipher_mode = EVP_CIPHER_mode(evp_cipher);
if (cipher_mode == EVP_CIPH_GCM_MODE)
{
return doDecryptImpl<CipherMode::RFC5116_AEAD_AES_GCM>(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column);
}
else
{
return doDecryptImpl<CipherMode::OpenSSLCompatibility>(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column);
}
}
return nullptr;
}
template <CipherMode mode, typename InputColumnType, typename KeyColumnType, typename IvColumnType, typename AadColumnType>
static ColumnPtr doDecryptImpl(const EVP_CIPHER * evp_cipher,
size_t input_rows_count,
const InputColumnType & input_column,
const KeyColumnType & key_column,
[[maybe_unused]] const IvColumnType & iv_column,
[[maybe_unused]] const AadColumnType & aad_column)
{
using namespace OpenSSLDetails;
auto evp_ctx_ptr = std::unique_ptr<EVP_CIPHER_CTX, decltype(&::EVP_CIPHER_CTX_free)>(EVP_CIPHER_CTX_new(), &EVP_CIPHER_CTX_free);
auto * evp_ctx = evp_ctx_ptr.get();
[[maybe_unused]] const auto block_size = static_cast<size_t>(EVP_CIPHER_block_size(evp_cipher));
[[maybe_unused]] const auto iv_size = static_cast<size_t>(EVP_CIPHER_iv_length(evp_cipher));
const auto key_size = static_cast<size_t>(EVP_CIPHER_key_length(evp_cipher));
const auto tag_size = 16; // https://tools.ietf.org/html/rfc5116#section-5.1
auto decrypted_result_column = ColumnString::create();
auto & decrypted_result_column_data = decrypted_result_column->getChars();
auto & decrypted_result_column_offsets = decrypted_result_column->getOffsets();
{
size_t resulting_size = 0;
for (size_t r = 0; r < input_rows_count; ++r)
{
resulting_size += input_column->getDataAt(r).size + 1;
if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM)
resulting_size -= tag_size;
}
#if defined(MEMORY_SANITIZER)
// Pre-fill result column with values to prevent MSAN from dropping dead on
// aes-X-ecb mode with "WARNING: MemorySanitizer: use-of-uninitialized-value".
// This is most likely to be caused by the underlying assembler implementation:
// see crypto/aes/aesni-x86_64.s, function aesni_ecb_encrypt
// which msan seems to fail instrument correctly.
decrypted_result_column_data.resize_fill(resulting_size, 0xFF);
#else
decrypted_result_column_data.resize(resulting_size);
#endif
}
auto * decrypted = decrypted_result_column_data.data();
KeyHolder<mode> key_holder;
for (size_t r = 0; r < input_rows_count; ++r)
{
// 0: prepare key if required
auto key_value = key_holder.setKey(key_size, key_column->getDataAt(r));
auto iv_value = StringRef{};
if constexpr (!std::is_same_v<std::nullptr_t, std::decay_t<IvColumnType>>)
{
iv_value = iv_column->getDataAt(r);
}
auto input_value = input_column->getDataAt(r);
if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM)
{
// empty plaintext results in empty ciphertext + tag, means there should be atleast tag_size bytes.
if (input_value.size < tag_size)
throw Exception(fmt::format("Encrypted data is too short: only {} bytes, "
"should contain at least {} bytes of a tag.",
input_value.size, block_size, tag_size), ErrorCodes::BAD_ARGUMENTS);
input_value.size -= tag_size;
}
if constexpr (mode != CipherMode::MySQLCompatibility)
{
// in GCM mode IV can be of arbitrary size (>0), for other modes IV is optional.
if (mode == CipherMode::RFC5116_AEAD_AES_GCM && iv_value.size == 0)
{
throw Exception("Invalid IV size " + std::to_string(iv_value.size) + " != expected size " + std::to_string(iv_size),
DB::ErrorCodes::BAD_ARGUMENTS);
}
if (key_value.size != key_size)
{
throw Exception("Invalid key size " + std::to_string(key_value.size) + " != expected size " + std::to_string(key_size),
DB::ErrorCodes::BAD_ARGUMENTS);
}
}
// Avoid extra work on empty ciphertext/plaintext for some ciphers
if (!(input_value.size == 0 && block_size == 1 && mode != CipherMode::RFC5116_AEAD_AES_GCM))
{
// 1: Init CTX
if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM)
{
if (EVP_DecryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr) != 1)
onError("Failed to initialize cipher context 1");
// 1.a.1 : Set custom IV length
if (EVP_CIPHER_CTX_ctrl(evp_ctx, EVP_CTRL_AEAD_SET_IVLEN, iv_value.size, nullptr) != 1)
onError("Failed to set custom IV length to " + std::to_string(iv_value.size));
// 1.a.1 : Init CTX with key and IV
if (EVP_DecryptInit_ex(evp_ctx, nullptr, nullptr,
reinterpret_cast<const unsigned char*>(key_value.data),
reinterpret_cast<const unsigned char*>(iv_value.data)) != 1)
onError("Failed to set key and IV");
// 1.a.2: Set AAD if present
if constexpr (!std::is_same_v<std::nullptr_t, std::decay_t<AadColumnType>>)
{
const auto aad_data = aad_column->getDataAt(r);
int tmp_len = 0;
if (aad_data.size != 0 && EVP_DecryptUpdate(evp_ctx, nullptr, &tmp_len,
reinterpret_cast<const unsigned char *>(aad_data.data), aad_data.size) != 1)
onError("Failed to sed AAD data");
}
}
else
{
// 1.b: Init CTX
validateIV<mode>(iv_value, iv_size);
if (EVP_DecryptInit_ex(evp_ctx, evp_cipher, nullptr,
reinterpret_cast<const unsigned char*>(key_value.data),
reinterpret_cast<const unsigned char*>(iv_value.data)) != 1)
onError("Failed to initialize cipher context");
}
// 2: Feed the data to the cipher
int output_len = 0;
if (EVP_DecryptUpdate(evp_ctx,
reinterpret_cast<unsigned char*>(decrypted), &output_len,
reinterpret_cast<const unsigned char*>(input_value.data), static_cast<int>(input_value.size)) != 1)
onError("Failed to decrypt");
decrypted += output_len;
// 3: optionally get tag from the ciphertext (RFC5116) and feed it to the context
if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM)
{
void * tag = const_cast<void *>(reinterpret_cast<const void *>(input_value.data + input_value.size));
if (EVP_CIPHER_CTX_ctrl(evp_ctx, EVP_CTRL_AEAD_SET_TAG, tag_size, tag) != 1)
onError("Failed to set tag");
}
// 4: retrieve encrypted data (ciphertext)
if (EVP_DecryptFinal_ex(evp_ctx,
reinterpret_cast<unsigned char*>(decrypted), &output_len) != 1)
onError("Failed to decrypt");
decrypted += output_len;
}
*decrypted = '\0';
++decrypted;
decrypted_result_column_offsets.push_back(decrypted - decrypted_result_column_data.data());
}
// in case we overestimate buffer required for decrypted data, fix it up.
if (!decrypted_result_column_offsets.empty() && decrypted_result_column_data.size() > decrypted_result_column_offsets.back())
{
decrypted_result_column_data.resize(decrypted_result_column_offsets.back());
}
decrypted_result_column->validate();
return decrypted_result_column;
}
};
}
#endif

183
src/Functions/FunctionsBitmap.h
View File

@ -104,7 +104,7 @@ public:
if (arguments[0]->onlyNull())
return arguments[0];
auto array_type = typeid_cast<const DataTypeArray *>(arguments[0].get());
const auto * array_type = typeid_cast<const DataTypeArray *>(arguments[0].get());
if (!array_type)
throw Exception(
"First argument for function " + getName() + " must be an array but it has type " + arguments[0]->getName() + ".",
@ -122,23 +122,23 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /* input_rows_count */) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /* input_rows_count */) const override
{
const IDataType * from_type = columns[arguments[0]].type.get();
auto array_type = typeid_cast<const DataTypeArray *>(from_type);
auto nested_type = array_type->getNestedType();
const IDataType * from_type = arguments[0].type.get();
const auto * array_type = typeid_cast<const DataTypeArray *>(from_type);
const auto & nested_type = array_type->getNestedType();
DataTypes argument_types = {nested_type};
WhichDataType which(nested_type);
if (which.isUInt8())
executeBitmapData<UInt8>(columns, argument_types, arguments, result);
return executeBitmapData<UInt8>(argument_types, arguments);
else if (which.isUInt16())
executeBitmapData<UInt16>(columns, argument_types, arguments, result);
return executeBitmapData<UInt16>(argument_types, arguments);
else if (which.isUInt32())
executeBitmapData<UInt32>(columns, argument_types, arguments, result);
return executeBitmapData<UInt32>(argument_types, arguments);
else if (which.isUInt64())
executeBitmapData<UInt64>(columns, argument_types, arguments, result);
return executeBitmapData<UInt64>(argument_types, arguments);
else
throw Exception(
"Unexpected type " + from_type->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
@ -146,11 +146,11 @@ public:
private:
template <typename T>
void executeBitmapData(ColumnsWithTypeAndName & columns, DataTypes & argument_types, const ColumnNumbers & arguments, size_t result) const
ColumnPtr executeBitmapData(DataTypes & argument_types, ColumnsWithTypeAndName & arguments) const
{
// input data
const ColumnArray * array = typeid_cast<const ColumnArray *>(columns[arguments[0]].column.get());
ColumnPtr mapped = array->getDataPtr();
const ColumnArray * array = typeid_cast<const ColumnArray *>(arguments[0].column.get());
const ColumnPtr & mapped = array->getDataPtr();
const ColumnArray::Offsets & offsets = array->getOffsets();
const ColumnVector<T> * column = checkAndGetColumn<ColumnVector<T>>(&*mapped);
const typename ColumnVector<T>::Container & input_data = column->getData();
@ -174,7 +174,7 @@ private:
bitmap_data.rbs.add(input_data[pos]);
}
}
columns[result].column = std::move(col_to);
return col_to;
}
};
@ -207,32 +207,32 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
// input data
const auto & return_type = columns[result].type;
const auto & return_type = result_type;
auto res_ptr = return_type->createColumn();
ColumnArray & res = assert_cast<ColumnArray &>(*res_ptr);
IColumn & res_data = res.getData();
ColumnArray::Offsets & res_offsets = res.getOffsets();
const IDataType * from_type = columns[arguments[0]].type.get();
const IDataType * from_type = arguments[0].type.get();
const DataTypeAggregateFunction * aggr_type = typeid_cast<const DataTypeAggregateFunction *>(from_type);
WhichDataType which(aggr_type->getArgumentsDataTypes()[0]);
if (which.isUInt8())
executeIntType<UInt8>(columns, arguments, input_rows_count, res_data, res_offsets);
executeIntType<UInt8>(arguments, input_rows_count, res_data, res_offsets);
else if (which.isUInt16())
executeIntType<UInt16>(columns, arguments, input_rows_count, res_data, res_offsets);
executeIntType<UInt16>(arguments, input_rows_count, res_data, res_offsets);
else if (which.isUInt32())
executeIntType<UInt32>(columns, arguments, input_rows_count, res_data, res_offsets);
executeIntType<UInt32>(arguments, input_rows_count, res_data, res_offsets);
else if (which.isUInt64())
executeIntType<UInt64>(columns, arguments, input_rows_count, res_data, res_offsets);
executeIntType<UInt64>(arguments, input_rows_count, res_data, res_offsets);
else
throw Exception(
"Unexpected type " + from_type->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
columns[result].column = std::move(res_ptr);
return res_ptr;
}
private:
@ -240,11 +240,11 @@ private:
template <typename T>
void executeIntType(
ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t input_rows_count, IColumn & res_data_col, ColumnArray::Offsets & res_offsets)
ColumnsWithTypeAndName & arguments, size_t input_rows_count, IColumn & res_data_col, ColumnArray::Offsets & res_offsets)
const
{
const ColumnAggregateFunction * column
= typeid_cast<const ColumnAggregateFunction *>(columns[arguments[0]].column.get());
= typeid_cast<const ColumnAggregateFunction *>(arguments[0].column.get());
PaddedPODArray<T> & res_data = typeid_cast<ColumnVector<T> &>(res_data_col).getData();
ColumnArray::Offset res_offset = 0;
@ -282,13 +282,13 @@ public:
"First argument for function " + getName() + " must be a bitmap but it has type " + arguments[0]->getName() + ".",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
auto arg_type1 = typeid_cast<const DataTypeNumber<UInt32> *>(arguments[1].get());
const auto * arg_type1 = typeid_cast<const DataTypeNumber<UInt32> *>(arguments[1].get());
if (!(arg_type1))
throw Exception(
"Second argument for function " + getName() + " must be UInt32 but it has type " + arguments[1]->getName() + ".",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
auto arg_type2 = typeid_cast<const DataTypeNumber<UInt32> *>(arguments[1].get());
const auto * arg_type2 = typeid_cast<const DataTypeNumber<UInt32> *>(arguments[1].get());
if (!(arg_type2))
throw Exception(
"Third argument for function " + getName() + " must be UInt32 but it has type " + arguments[2]->getName() + ".",
@ -299,19 +299,19 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const IDataType * from_type = columns[arguments[0]].type.get();
const IDataType * from_type = arguments[0].type.get();
const DataTypeAggregateFunction * aggr_type = typeid_cast<const DataTypeAggregateFunction *>(from_type);
WhichDataType which(aggr_type->getArgumentsDataTypes()[0]);
if (which.isUInt8())
executeIntType<UInt8>(columns, arguments, result, input_rows_count);
return executeIntType<UInt8>(arguments, input_rows_count);
else if (which.isUInt16())
executeIntType<UInt16>(columns, arguments, result, input_rows_count);
return executeIntType<UInt16>(arguments, input_rows_count);
else if (which.isUInt32())
executeIntType<UInt32>(columns, arguments, result, input_rows_count);
return executeIntType<UInt32>(arguments, input_rows_count);
else if (which.isUInt64())
executeIntType<UInt64>(columns, arguments, result, input_rows_count);
return executeIntType<UInt64>(arguments, input_rows_count);
else
throw Exception(
"Unexpected type " + from_type->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
@ -321,9 +321,7 @@ private:
using ToType = UInt64;
template <typename T>
void executeIntType(
ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count)
const
ColumnPtr executeIntType(ColumnsWithTypeAndName & arguments, size_t input_rows_count) const
{
const IColumn * column_ptrs[3];
bool is_column_const[3];
@ -333,7 +331,7 @@ private:
for (size_t i = 0; i < 3; ++i)
{
column_ptrs[i] = columns[arguments[i]].column.get();
column_ptrs[i] = arguments[i].column.get();
is_column_const[i] = isColumnConst(*column_ptrs[i]);
}
if (is_column_const[0])
@ -367,7 +365,7 @@ private:
= *reinterpret_cast<AggregateFunctionGroupBitmapData<T> *>(col_to->getData()[i]);
Impl::apply(bitmap_data_0, range_start, range_end, bitmap_data_2);
}
columns[result].column = std::move(col_to);
return col_to;
}
};
@ -435,19 +433,19 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const IDataType * from_type = columns[arguments[0]].type.get();
const IDataType * from_type = arguments[0].type.get();
const DataTypeAggregateFunction * aggr_type = typeid_cast<const DataTypeAggregateFunction *>(from_type);
WhichDataType which(aggr_type->getArgumentsDataTypes()[0]);
if (which.isUInt8())
executeIntType<UInt8>(columns, arguments, result, input_rows_count);
return executeIntType<UInt8>(arguments, input_rows_count);
else if (which.isUInt16())
executeIntType<UInt16>(columns, arguments, result, input_rows_count);
return executeIntType<UInt16>(arguments, input_rows_count);
else if (which.isUInt32())
executeIntType<UInt32>(columns, arguments, result, input_rows_count);
return executeIntType<UInt32>(arguments, input_rows_count);
else if (which.isUInt64())
executeIntType<UInt64>(columns, arguments, result, input_rows_count);
return executeIntType<UInt64>(arguments, input_rows_count);
else
throw Exception(
"Unexpected type " + from_type->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
@ -457,8 +455,7 @@ private:
using ToType = UInt64;
template <typename T>
void executeIntType(
ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const
ColumnPtr executeIntType(ColumnsWithTypeAndName & arguments, size_t input_rows_count) const
{
const IColumn * column_ptrs[3];
bool is_column_const[3];
@ -468,7 +465,7 @@ private:
for (size_t i = 0; i < 3; ++i)
{
column_ptrs[i] = columns[arguments[i]].column.get();
column_ptrs[i] = arguments[i].column.get();
is_column_const[i] = isColumnConst(*column_ptrs[i]);
}
if (is_column_const[0])
@ -485,7 +482,7 @@ private:
array = typeid_cast<const ColumnArray*>(typeid_cast<const ColumnConst*>(column_ptrs[1])->getDataColumnPtr().get());
else
{
array = typeid_cast<const ColumnArray *>(columns[arguments[1]].column.get());
array = typeid_cast<const ColumnArray *>(arguments[1].column.get());
}
const ColumnArray::Offsets & from_offsets = array->getOffsets();
const ColumnVector<UInt32>::Container & from_container = typeid_cast<const ColumnVector<UInt32> *>(&array->getData())->getData();
@ -493,7 +490,7 @@ private:
if (is_column_const[2])
array = typeid_cast<const ColumnArray*>(typeid_cast<const ColumnConst*>(column_ptrs[2])->getDataColumnPtr().get());
else
array = typeid_cast<const ColumnArray *>(columns[arguments[2]].column.get());
array = typeid_cast<const ColumnArray *>(arguments[2].column.get());
const ColumnArray::Offsets & to_offsets = array->getOffsets();
const ColumnVector<UInt32>::Container & to_container = typeid_cast<const ColumnVector<UInt32> *>(&array->getData())->getData();
@ -538,7 +535,7 @@ private:
bitmap_data_2.rbs.merge(bitmap_data_0.rbs);
bitmap_data_2.rbs.rb_replace(&from_container[from_start], &to_container[to_start], from_end - from_start);
}
columns[result].column = std::move(col_to);
return col_to;
}
};
@ -558,7 +555,7 @@ public:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
auto bitmap_type = typeid_cast<const DataTypeAggregateFunction *>(arguments[0].get());
const auto * bitmap_type = typeid_cast<const DataTypeAggregateFunction *>(arguments[0].get());
if (!(bitmap_type && bitmap_type->getFunctionName() == AggregateFunctionGroupBitmapData<UInt32>::name()))
throw Exception(
"First argument for function " + getName() + " must be a bitmap but it has type " + arguments[0]->getName() + ".",
@ -568,27 +565,27 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
auto col_to = ColumnVector<ToType>::create(input_rows_count);
typename ColumnVector<ToType>::Container & vec_to = col_to->getData();
const IDataType * from_type = columns[arguments[0]].type.get();
const IDataType * from_type = arguments[0].type.get();
const DataTypeAggregateFunction * aggr_type = typeid_cast<const DataTypeAggregateFunction *>(from_type);
WhichDataType which(aggr_type->getArgumentsDataTypes()[0]);
if (which.isUInt8())
executeIntType<UInt8>(columns, arguments, input_rows_count, vec_to);
executeIntType<UInt8>(arguments, input_rows_count, vec_to);
else if (which.isUInt16())
executeIntType<UInt16>(columns, arguments, input_rows_count, vec_to);
executeIntType<UInt16>(arguments, input_rows_count, vec_to);
else if (which.isUInt32())
executeIntType<UInt32>(columns, arguments, input_rows_count, vec_to);
executeIntType<UInt32>(arguments, input_rows_count, vec_to);
else if (which.isUInt64())
executeIntType<UInt64>(columns, arguments, input_rows_count, vec_to);
executeIntType<UInt64>(arguments, input_rows_count, vec_to);
else
throw Exception(
"Unexpected type " + from_type->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
columns[result].column = std::move(col_to);
return col_to;
}
private:
@ -596,10 +593,10 @@ private:
template <typename T>
void executeIntType(
ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t input_rows_count, typename ColumnVector<ToType>::Container & vec_to) const
ColumnsWithTypeAndName & arguments, size_t input_rows_count, typename ColumnVector<ToType>::Container & vec_to) const
{
const ColumnAggregateFunction * column
= typeid_cast<const ColumnAggregateFunction *>(columns[arguments[0]].column.get());
= typeid_cast<const ColumnAggregateFunction *>(arguments[0].column.get());
for (size_t i = 0; i < input_rows_count; ++i)
{
const AggregateFunctionGroupBitmapData<T> & bitmap_data
@ -722,12 +719,12 @@ public:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
auto bitmap_type0 = typeid_cast<const DataTypeAggregateFunction *>(arguments[0].get());
const auto * bitmap_type0 = typeid_cast<const DataTypeAggregateFunction *>(arguments[0].get());
if (!(bitmap_type0 && bitmap_type0->getFunctionName() == AggregateFunctionGroupBitmapData<UInt32>::name()))
throw Exception(
"First argument for function " + getName() + " must be a bitmap but it has type " + arguments[0]->getName() + ".",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
auto arg_type1 = typeid_cast<const DataTypeNumber<UInt32> *>(arguments[1].get());
const auto * arg_type1 = typeid_cast<const DataTypeNumber<UInt32> *>(arguments[1].get());
if (!(arg_type1))
throw Exception(
"Second argument for function " + getName() + " must be UInt32 but it has type " + arguments[1]->getName() + ".",
@ -738,33 +735,33 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
auto col_to = ColumnVector<UInt8>::create(input_rows_count);
typename ColumnVector<UInt8>::Container & vec_to = col_to->getData();
const IDataType * from_type = columns[arguments[0]].type.get();
const IDataType * from_type = arguments[0].type.get();
const DataTypeAggregateFunction * aggr_type = typeid_cast<const DataTypeAggregateFunction *>(from_type);
WhichDataType which(aggr_type->getArgumentsDataTypes()[0]);
if (which.isUInt8())
executeIntType<UInt8>(columns, arguments, input_rows_count, vec_to);
executeIntType<UInt8>(arguments, input_rows_count, vec_to);
else if (which.isUInt16())
executeIntType<UInt16>(columns, arguments, input_rows_count, vec_to);
executeIntType<UInt16>(arguments, input_rows_count, vec_to);
else if (which.isUInt32())
executeIntType<UInt32>(columns, arguments, input_rows_count, vec_to);
executeIntType<UInt32>(arguments, input_rows_count, vec_to);
else if (which.isUInt64())
executeIntType<UInt64>(columns, arguments, input_rows_count, vec_to);
executeIntType<UInt64>(arguments, input_rows_count, vec_to);
else
throw Exception(
"Unexpected type " + from_type->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
columns[result].column = std::move(col_to);
return col_to;
}
private:
template <typename T>
void executeIntType(
ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t input_rows_count, typename ColumnVector<UInt8>::Container & vec_to) const
ColumnsWithTypeAndName & arguments, size_t input_rows_count, typename ColumnVector<UInt8>::Container & vec_to) const
{
const IColumn * column_ptrs[2];
bool is_column_const[2];
@ -773,7 +770,7 @@ private:
for (size_t i = 0; i < 2; ++i)
{
column_ptrs[i] = columns[arguments[i]].column.get();
column_ptrs[i] = arguments[i].column.get();
is_column_const[i] = isColumnConst(*column_ptrs[i]);
}
if (is_column_const[0])
@ -812,13 +809,13 @@ public:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
auto bitmap_type0 = typeid_cast<const DataTypeAggregateFunction *>(arguments[0].get());
const auto * bitmap_type0 = typeid_cast<const DataTypeAggregateFunction *>(arguments[0].get());
if (!(bitmap_type0 && bitmap_type0->getFunctionName() == AggregateFunctionGroupBitmapData<UInt32>::name()))
throw Exception(
"First argument for function " + getName() + " must be a bitmap but it has type " + arguments[0]->getName() + ".",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
auto bitmap_type1 = typeid_cast<const DataTypeAggregateFunction *>(arguments[1].get());
const auto * bitmap_type1 = typeid_cast<const DataTypeAggregateFunction *>(arguments[1].get());
if (!(bitmap_type1 && bitmap_type1->getFunctionName() == AggregateFunctionGroupBitmapData<UInt32>::name()))
throw Exception(
"Second argument for function " + getName() + " must be a bitmap but it has type " + arguments[1]->getName() + ".",
@ -835,46 +832,46 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
auto col_to = ColumnVector<ToType>::create(input_rows_count);
typename ColumnVector<ToType>::Container & vec_to = col_to->getData();
const IDataType * from_type = columns[arguments[0]].type.get();
const IDataType * from_type = arguments[0].type.get();
const DataTypeAggregateFunction * aggr_type = typeid_cast<const DataTypeAggregateFunction *>(from_type);
WhichDataType which(aggr_type->getArgumentsDataTypes()[0]);
if (which.isUInt8())
executeIntType<UInt8>(columns, arguments, input_rows_count, vec_to);
executeIntType<UInt8>(arguments, input_rows_count, vec_to);
else if (which.isUInt16())
executeIntType<UInt16>(columns, arguments, input_rows_count, vec_to);
executeIntType<UInt16>(arguments, input_rows_count, vec_to);
else if (which.isUInt32())
executeIntType<UInt32>(columns, arguments, input_rows_count, vec_to);
executeIntType<UInt32>(arguments, input_rows_count, vec_to);
else if (which.isUInt64())
executeIntType<UInt64>(columns, arguments, input_rows_count, vec_to);
executeIntType<UInt64>(arguments, input_rows_count, vec_to);
else
throw Exception(
"Unexpected type " + from_type->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
columns[result].column = std::move(col_to);
return col_to;
}
private:
template <typename T>
void executeIntType(
ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t input_rows_count, typename ColumnVector<ToType>::Container & vec_to) const
ColumnsWithTypeAndName & arguments, size_t input_rows_count, typename ColumnVector<ToType>::Container & vec_to) const
{
const ColumnAggregateFunction * column_ptrs[2];
bool is_column_const[2];
for (size_t i = 0; i < 2; ++i)
{
if (auto argument_column_const = checkAndGetColumn<ColumnConst>(columns[arguments[i]].column.get()))
if (const auto * argument_column_const = checkAndGetColumn<ColumnConst>(arguments[i].column.get()))
{
column_ptrs[i] = typeid_cast<const ColumnAggregateFunction*>(argument_column_const->getDataColumnPtr().get());
is_column_const[i] = true;
}
else
{
column_ptrs[i] = typeid_cast<const ColumnAggregateFunction*>(columns[arguments[i]].column.get());
column_ptrs[i] = typeid_cast<const ColumnAggregateFunction*>(arguments[i].column.get());
is_column_const[i] = false;
}
}
@ -947,13 +944,13 @@ public:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
auto bitmap_type0 = typeid_cast<const DataTypeAggregateFunction *>(arguments[0].get());
const auto * bitmap_type0 = typeid_cast<const DataTypeAggregateFunction *>(arguments[0].get());
if (!(bitmap_type0 && bitmap_type0->getFunctionName() == AggregateFunctionGroupBitmapData<UInt32>::name()))
throw Exception(
"First argument for function " + getName() + " must be a bitmap but it has type " + arguments[0]->getName() + ".",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
auto bitmap_type1 = typeid_cast<const DataTypeAggregateFunction *>(arguments[1].get());
const auto * bitmap_type1 = typeid_cast<const DataTypeAggregateFunction *>(arguments[1].get());
if (!(bitmap_type1 && bitmap_type1->getFunctionName() == AggregateFunctionGroupBitmapData<UInt32>::name()))
throw Exception(
"Second argument for function " + getName() + " must be a bitmap but it has type " + arguments[1]->getName() + ".",
@ -970,19 +967,19 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const IDataType * from_type = columns[arguments[0]].type.get();
const IDataType * from_type = arguments[0].type.get();
const DataTypeAggregateFunction * aggr_type = typeid_cast<const DataTypeAggregateFunction *>(from_type);
WhichDataType which(aggr_type->getArgumentsDataTypes()[0]);
if (which.isUInt8())
executeBitmapData<UInt8>(columns, arguments, result, input_rows_count);
return executeBitmapData<UInt8>(arguments, input_rows_count);
else if (which.isUInt16())
executeBitmapData<UInt16>(columns, arguments, result, input_rows_count);
return executeBitmapData<UInt16>(arguments, input_rows_count);
else if (which.isUInt32())
executeBitmapData<UInt32>(columns, arguments, result, input_rows_count);
return executeBitmapData<UInt32>(arguments, input_rows_count);
else if (which.isUInt64())
executeBitmapData<UInt64>(columns, arguments, result, input_rows_count);
return executeBitmapData<UInt64>(arguments, input_rows_count);
else
throw Exception(
"Unexpected type " + from_type->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
@ -990,20 +987,20 @@ public:
private:
template <typename T>
void executeBitmapData(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const
ColumnPtr executeBitmapData(ColumnsWithTypeAndName & arguments, size_t input_rows_count) const
{
const ColumnAggregateFunction * column_ptrs[2];
bool is_column_const[2];
for (size_t i = 0; i < 2; ++i)
{
if (auto argument_column_const = typeid_cast<const ColumnConst *>(columns[arguments[i]].column.get()))
if (const auto * argument_column_const = typeid_cast<const ColumnConst *>(arguments[i].column.get()))
{
column_ptrs[i] = typeid_cast<const ColumnAggregateFunction *>(argument_column_const->getDataColumnPtr().get());
is_column_const[i] = true;
}
else
{
column_ptrs[i] = typeid_cast<const ColumnAggregateFunction *>(columns[arguments[i]].column.get());
column_ptrs[i] = typeid_cast<const ColumnAggregateFunction *>(arguments[i].column.get());
is_column_const[i] = false;
}
}
@ -1026,7 +1023,7 @@ private:
= *reinterpret_cast<const AggregateFunctionGroupBitmapData<T> *>(data_ptr_1);
Impl<T>::apply(bitmap_data_1, bitmap_data_2);
}
columns[result].column = std::move(col_to);
return col_to;
}
};

261
src/Functions/FunctionsCoding.h
View File

@ -76,7 +76,7 @@ public:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
const auto ptr = checkAndGetDataType<DataTypeFixedString>(arguments[0].get());
const auto * ptr = checkAndGetDataType<DataTypeFixedString>(arguments[0].get());
if (!ptr || ptr->getN() != IPV6_BINARY_LENGTH)
throw Exception("Illegal type " + arguments[0]->getName() +
" of argument of function " + getName() +
@ -88,12 +88,12 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const auto & col_type_name = columns[arguments[0]];
const auto & col_type_name = arguments[0];
const ColumnPtr & column = col_type_name.column;
if (const auto col_in = checkAndGetColumn<ColumnFixedString>(column.get()))
if (const auto * col_in = checkAndGetColumn<ColumnFixedString>(column.get()))
{
if (col_in->getN() != IPV6_BINARY_LENGTH)
throw Exception("Illegal type " + col_type_name.type->getName() +
@ -112,8 +112,8 @@ public:
vec_res.resize(size * (IPV6_MAX_TEXT_LENGTH + 1));
offsets_res.resize(size);
auto begin = reinterpret_cast<char *>(vec_res.data());
auto pos = begin;
auto * begin = reinterpret_cast<char *>(vec_res.data());
auto * pos = begin;
for (size_t offset = 0, i = 0; offset < vec_in.size(); offset += IPV6_BINARY_LENGTH, ++i)
{
@ -123,10 +123,10 @@ public:
vec_res.resize(pos - begin);
columns[result].column = std::move(col_res);
return col_res;
}
else
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
@ -145,7 +145,7 @@ public:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
const auto ptr = checkAndGetDataType<DataTypeFixedString>(arguments[0].get());
const auto * ptr = checkAndGetDataType<DataTypeFixedString>(arguments[0].get());
if (!ptr || ptr->getN() != IPV6_BINARY_LENGTH)
throw Exception("Illegal type " + arguments[0]->getName() +
" of argument 1 of function " + getName() +
@ -168,17 +168,17 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const auto & col_type_name = columns[arguments[0]];
const auto & col_type_name = arguments[0];
const ColumnPtr & column = col_type_name.column;
const auto & col_ipv6_zeroed_tail_bytes_type = columns[arguments[1]];
const auto & col_ipv6_zeroed_tail_bytes_type = arguments[1];
const auto & col_ipv6_zeroed_tail_bytes = col_ipv6_zeroed_tail_bytes_type.column;
const auto & col_ipv4_zeroed_tail_bytes_type = columns[arguments[2]];
const auto & col_ipv4_zeroed_tail_bytes_type = arguments[2];
const auto & col_ipv4_zeroed_tail_bytes = col_ipv4_zeroed_tail_bytes_type.column;
if (const auto col_in = checkAndGetColumn<ColumnFixedString>(column.get()))
if (const auto * col_in = checkAndGetColumn<ColumnFixedString>(column.get()))
{
if (col_in->getN() != IPV6_BINARY_LENGTH)
throw Exception("Illegal type " + col_type_name.type->getName() +
@ -187,7 +187,7 @@ public:
", expected FixedString(" + toString(IPV6_BINARY_LENGTH) + ")",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
const auto ipv6_zeroed_tail_bytes = checkAndGetColumnConst<ColumnVector<UInt8>>(col_ipv6_zeroed_tail_bytes.get());
const auto * ipv6_zeroed_tail_bytes = checkAndGetColumnConst<ColumnVector<UInt8>>(col_ipv6_zeroed_tail_bytes.get());
if (!ipv6_zeroed_tail_bytes)
throw Exception("Illegal type " + col_ipv6_zeroed_tail_bytes_type.type->getName() +
" of argument 2 of function " + getName(),
@ -199,7 +199,7 @@ public:
" of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
const auto ipv4_zeroed_tail_bytes = checkAndGetColumnConst<ColumnVector<UInt8>>(col_ipv4_zeroed_tail_bytes.get());
const auto * ipv4_zeroed_tail_bytes = checkAndGetColumnConst<ColumnVector<UInt8>>(col_ipv4_zeroed_tail_bytes.get());
if (!ipv4_zeroed_tail_bytes)
throw Exception("Illegal type " + col_ipv4_zeroed_tail_bytes_type.type->getName() +
" of argument 3 of function " + getName(),
@ -221,12 +221,12 @@ public:
vec_res.resize(size * (IPV6_MAX_TEXT_LENGTH + 1));
offsets_res.resize(size);
auto begin = reinterpret_cast<char *>(vec_res.data());
auto pos = begin;
auto * begin = reinterpret_cast<char *>(vec_res.data());
auto * pos = begin;
for (size_t offset = 0, i = 0; offset < vec_in.size(); offset += IPV6_BINARY_LENGTH, ++i)
{
const auto address = &vec_in[offset];
const auto * address = &vec_in[offset];
UInt8 zeroed_tail_bytes_count = isIPv4Mapped(address) ? ipv4_zeroed_tail_bytes_count : ipv6_zeroed_tail_bytes_count;
cutAddress(reinterpret_cast<const unsigned char *>(address), pos, zeroed_tail_bytes_count);
offsets_res[i] = pos - begin;
@ -234,22 +234,22 @@ public:
vec_res.resize(pos - begin);
columns[result].column = std::move(col_res);
return col_res;
}
else
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
private:
bool isIPv4Mapped(const UInt8 * address) const
static bool isIPv4Mapped(const UInt8 * address)
{
return (unalignedLoad<UInt64>(address) == 0) &&
((unalignedLoad<UInt64>(address + 8) & 0x00000000FFFFFFFFull) == 0x00000000FFFF0000ull);
}
void cutAddress(const unsigned char * address, char *& dst, UInt8 zeroed_tail_bytes_count) const
static void cutAddress(const unsigned char * address, char *& dst, UInt8 zeroed_tail_bytes_count)
{
formatIPv6(address, dst, zeroed_tail_bytes_count);
}
@ -277,11 +277,11 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const ColumnPtr & column = columns[arguments[0]].column;
const ColumnPtr & column = arguments[0].column;
if (const auto col_in = checkAndGetColumn<ColumnString>(column.get()))
if (const auto * col_in = checkAndGetColumn<ColumnString>(column.get()))
{
auto col_res = ColumnFixedString::create(IPV6_BINARY_LENGTH);
@ -301,10 +301,10 @@ public:
src_offset = offsets_src[i];
}
columns[result].column = std::move(col_res);
return col_res;
}
else
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
@ -339,9 +339,9 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const ColumnPtr & column = columns[arguments[0]].column;
const ColumnPtr & column = arguments[0].column;
if (const ColumnUInt32 * col = typeid_cast<const ColumnUInt32 *>(column.get()))
{
@ -365,10 +365,10 @@ public:
vec_res.resize(pos - begin);
columns[result].column = std::move(col_res);
return col_res;
}
else
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
@ -407,9 +407,9 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const ColumnPtr & column = columns[arguments[0]].column;
const ColumnPtr & column = arguments[0].column;
if (const ColumnString * col = checkAndGetColumn<ColumnString>(column.get()))
{
@ -428,10 +428,10 @@ public:
prev_offset = offsets_src[i];
}
columns[result].column = std::move(col_res);
return col_res;
}
else
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
@ -460,12 +460,12 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const auto & col_type_name = columns[arguments[0]];
const auto & col_type_name = arguments[0];
const ColumnPtr & column = col_type_name.column;
if (const auto col_in = typeid_cast<const ColumnUInt32 *>(column.get()))
if (const auto * col_in = typeid_cast<const ColumnUInt32 *>(column.get()))
{
auto col_res = ColumnFixedString::create(IPV6_BINARY_LENGTH);
@ -477,16 +477,16 @@ public:
for (size_t out_offset = 0, i = 0; out_offset < vec_res.size(); out_offset += IPV6_BINARY_LENGTH, ++i)
mapIPv4ToIPv6(vec_in[i], &vec_res[out_offset]);
columns[result].column = std::move(col_res);
return col_res;
}
else
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
private:
void mapIPv4ToIPv6(UInt32 in, UInt8 * buf) const
static void mapIPv4ToIPv6(UInt32 in, UInt8 * buf)
{
unalignedStore<UInt64>(buf, 0);
unalignedStore<UInt64>(buf + 8, 0x00000000FFFF0000ull | (static_cast<UInt64>(ntohl(in)) << 32));
@ -578,9 +578,9 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const ColumnPtr & column = columns[arguments[0]].column;
const ColumnPtr & column = arguments[0].column;
if (const ColumnUInt64 * col = typeid_cast<const ColumnUInt64 *>(column.get()))
{
@ -602,10 +602,10 @@ public:
offsets_res[i] = current_offset;
}
columns[result].column = std::move(col_res);
return col_res;
}
else
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
@ -688,9 +688,9 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const ColumnPtr & column = columns[arguments[0]].column;
const ColumnPtr & column = arguments[0].column;
if (const ColumnString * col = checkAndGetColumn<ColumnString>(column.get()))
{
@ -716,10 +716,10 @@ public:
prev_offset = current_offset;
}
columns[result].column = std::move(col_res);
return col_res;
}
else
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
@ -743,7 +743,7 @@ public:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
const auto ptr = checkAndGetDataType<DataTypeFixedString>(arguments[0].get());
const auto * ptr = checkAndGetDataType<DataTypeFixedString>(arguments[0].get());
if (!ptr || ptr->getN() != uuid_bytes_length)
throw Exception("Illegal type " + arguments[0]->getName() +
" of argument of function " + getName() +
@ -755,12 +755,12 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const ColumnWithTypeAndName & col_type_name = columns[arguments[0]];
const ColumnWithTypeAndName & col_type_name = arguments[0];
const ColumnPtr & column = col_type_name.column;
if (const auto col_in = checkAndGetColumn<ColumnFixedString>(column.get()))
if (const auto * col_in = checkAndGetColumn<ColumnFixedString>(column.get()))
{
if (col_in->getN() != uuid_bytes_length)
throw Exception("Illegal type " + col_type_name.type->getName() +
@ -792,10 +792,10 @@ public:
offsets_res[i] = dst_offset;
}
columns[result].column = std::move(col_res);
return col_res;
}
else
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
@ -844,7 +844,7 @@ public:
/// String or FixedString(36)
if (!isString(arguments[0]))
{
const auto ptr = checkAndGetDataType<DataTypeFixedString>(arguments[0].get());
const auto * ptr = checkAndGetDataType<DataTypeFixedString>(arguments[0].get());
if (!ptr || ptr->getN() != uuid_text_length)
throw Exception("Illegal type " + arguments[0]->getName() +
" of argument of function " + getName() +
@ -857,12 +857,12 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const ColumnWithTypeAndName & col_type_name = columns[arguments[0]];
const ColumnWithTypeAndName & col_type_name = arguments[0];
const ColumnPtr & column = col_type_name.column;
if (const auto col_in = checkAndGetColumn<ColumnString>(column.get()))
if (const auto * col_in = checkAndGetColumn<ColumnString>(column.get()))
{
const auto & vec_in = col_in->getChars();
const auto & offsets_in = col_in->getOffsets();
@ -891,9 +891,9 @@ public:
src_offset += string_size;
}
columns[result].column = std::move(col_res);
return col_res;
}
else if (const auto col_in_fixed = checkAndGetColumn<ColumnFixedString>(column.get()))
else if (const auto * col_in_fixed = checkAndGetColumn<ColumnFixedString>(column.get()))
{
if (col_in_fixed->getN() != uuid_text_length)
throw Exception("Illegal type " + col_type_name.type->getName() +
@ -920,10 +920,10 @@ public:
dst_offset += uuid_bytes_length;
}
columns[result].column = std::move(col_res);
return col_res;
}
else
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN);
}
};
@ -1083,7 +1083,7 @@ public:
}
void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out) const
static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out)
{
while (pos < end)
{
@ -1095,7 +1095,7 @@ public:
++out;
}
bool tryExecuteString(const IColumn * col, ColumnPtr & col_res) const
static bool tryExecuteString(const IColumn * col, ColumnPtr & col_res)
{
const ColumnString * col_str_in = checkAndGetColumn<ColumnString>(col);
@ -1139,7 +1139,7 @@ public:
}
}
bool tryExecuteFixedString(const IColumn * col, ColumnPtr & col_res) const
static bool tryExecuteFixedString(const IColumn * col, ColumnPtr & col_res)
{
const ColumnFixedString * col_fstr_in = checkAndGetColumn<ColumnFixedString>(col);
@ -1187,10 +1187,10 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const IColumn * column = columns[arguments[0]].column.get();
ColumnPtr & res_column = columns[result].column;
const IColumn * column = arguments[0].column.get();
ColumnPtr res_column;
if (tryExecuteUInt<UInt8>(column, res_column) ||
tryExecuteUInt<UInt16>(column, res_column) ||
@ -1203,9 +1203,9 @@ public:
tryExecuteDecimal<Decimal32>(column, res_column) ||
tryExecuteDecimal<Decimal64>(column, res_column) ||
tryExecuteDecimal<Decimal128>(column, res_column))
return;
return res_column;
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
@ -1235,7 +1235,7 @@ public:
return std::make_shared<DataTypeString>();
}
void unhexOne(const char * pos, const char * end, char *& out) const
static void unhexOne(const char * pos, const char * end, char *& out)
{
if ((end - pos) & 1)
{
@ -1255,9 +1255,9 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const ColumnPtr & column = columns[arguments[0]].column;
const ColumnPtr & column = arguments[0].column;
if (const ColumnString * col = checkAndGetColumn<ColumnString>(column.get()))
{
@ -1290,11 +1290,11 @@ public:
out_vec.resize(pos - begin);
columns[result].column = std::move(col_res);
return col_res;
}
else
{
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
@ -1335,7 +1335,7 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
auto col_str = ColumnString::create();
ColumnString::Chars & out_vec = col_str->getChars();
@ -1355,7 +1355,7 @@ public:
for (size_t idx = 0; idx < arguments.size(); ++idx)
{
//partial const column
columns_holder[idx] = columns[arguments[idx]].column->convertToFullColumnIfConst();
columns_holder[idx] = arguments[idx].column->convertToFullColumnIfConst();
const IColumn * column = columns_holder[idx].get();
if (!(executeNumber<UInt8>(*column, out_vec, idx, input_rows_count, size_per_row)
@ -1369,12 +1369,12 @@ public:
|| executeNumber<Float32>(*column, out_vec, idx, input_rows_count, size_per_row)
|| executeNumber<Float64>(*column, out_vec, idx, input_rows_count, size_per_row)))
{
throw Exception{"Illegal column " + columns[arguments[idx]].column->getName()
throw Exception{"Illegal column " + arguments[idx].column->getName()
+ " of first argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
}
}
columns[result].column = std::move(col_str);
return col_str;
}
private:
@ -1461,10 +1461,10 @@ public:
}
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const IColumn * in_column = columns[arguments[0]].column.get();
ColumnPtr & out_column = columns[result].column;
const IColumn * in_column = arguments[0].column.get();
ColumnPtr out_column;
if (tryExecute<UInt8>(in_column, out_column) ||
tryExecute<UInt16>(in_column, out_column) ||
@ -1474,9 +1474,9 @@ public:
tryExecute<Int16>(in_column, out_column) ||
tryExecute<Int32>(in_column, out_column) ||
tryExecute<Int64>(in_column, out_column))
return;
return out_column;
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of first argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
@ -1506,7 +1506,7 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
bool tryExecuteString(const IColumn * col, ColumnPtr & col_res) const
static bool tryExecuteString(const IColumn * col, ColumnPtr & col_res)
{
const ColumnString * col_str_in = checkAndGetColumn<ColumnString>(col);
@ -1553,7 +1553,7 @@ public:
}
}
bool tryExecuteFixedString(const IColumn * col, ColumnPtr & col_res) const
static bool tryExecuteFixedString(const IColumn * col, ColumnPtr & col_res)
{
const ColumnFixedString * col_fstr_in = checkAndGetColumn<ColumnFixedString>(col);
@ -1599,15 +1599,15 @@ public:
}
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const IColumn * column = columns[arguments[0]].column.get();
ColumnPtr & res_column = columns[result].column;
const IColumn * column = arguments[0].column.get();
ColumnPtr res_column;
if (tryExecuteFixedString(column, res_column) || tryExecuteString(column, res_column))
return;
return res_column;
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
@ -1648,7 +1648,7 @@ public:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
const auto first_argument = checkAndGetDataType<DataTypeFixedString>(arguments[0].get());
const auto * first_argument = checkAndGetDataType<DataTypeFixedString>(arguments[0].get());
if (!first_argument || first_argument->getN() != IPV6_BINARY_LENGTH)
throw Exception("Illegal type " + arguments[0]->getName() +
" of first argument of function " + getName() +
@ -1659,7 +1659,7 @@ public:
if (!isUInt8(second_argument))
throw Exception{"Illegal type " + second_argument->getName()
+ " of second argument of function " + getName()
+ ", expected numeric type.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+ ", expected UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
DataTypePtr element = DataTypeFactory::instance().get("IPv6");
return std::make_shared<DataTypeTuple>(DataTypes{element, element});
@ -1668,38 +1668,38 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const auto & col_type_name_ip = columns[arguments[0]];
const auto & col_type_name_ip = arguments[0];
const ColumnPtr & column_ip = col_type_name_ip.column;
const auto col_ip_in = checkAndGetColumn<ColumnFixedString>(column_ip.get());
const auto * col_const_ip_in = checkAndGetColumnConst<ColumnFixedString>(column_ip.get());
const auto * col_ip_in = checkAndGetColumn<ColumnFixedString>(column_ip.get());
if (!col_ip_in)
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
if (!col_ip_in && !col_const_ip_in)
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
if (col_ip_in->getN() != IPV6_BINARY_LENGTH)
if ((col_const_ip_in && col_const_ip_in->getValue<String>().size() != IPV6_BINARY_LENGTH) ||
(col_ip_in && col_ip_in->getN() != IPV6_BINARY_LENGTH))
throw Exception("Illegal type " + col_type_name_ip.type->getName() +
" of column " + col_ip_in->getName() +
" of column " + column_ip->getName() +
" argument of function " + getName() +
", expected FixedString(" + toString(IPV6_BINARY_LENGTH) + ")",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
const auto & col_type_name_cidr = columns[arguments[1]];
const auto & col_type_name_cidr = arguments[1];
const ColumnPtr & column_cidr = col_type_name_cidr.column;
const auto col_const_cidr_in = checkAndGetColumnConst<ColumnUInt8>(column_cidr.get());
const auto col_cidr_in = checkAndGetColumn<ColumnUInt8>(column_cidr.get());
const auto * col_const_cidr_in = checkAndGetColumnConst<ColumnUInt8>(column_cidr.get());
const auto * col_cidr_in = checkAndGetColumn<ColumnUInt8>(column_cidr.get());
if (!col_const_cidr_in && !col_cidr_in)
throw Exception("Illegal column " + columns[arguments[1]].column->getName()
throw Exception("Illegal column " + arguments[1].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
const auto & vec_in = col_ip_in->getChars();
auto col_res_lower_range = ColumnFixedString::create(IPV6_BINARY_LENGTH);
auto col_res_upper_range = ColumnFixedString::create(IPV6_BINARY_LENGTH);
@ -1711,17 +1711,27 @@ public:
static constexpr UInt8 max_cidr_mask = IPV6_BINARY_LENGTH * 8;
const String col_const_ip_str = col_const_ip_in ? col_const_ip_in->getValue<String>() : "";
const UInt8 * col_const_ip_value = col_const_ip_in ? reinterpret_cast<const UInt8 *>(col_const_ip_str.c_str()) : nullptr;
for (size_t offset = 0; offset < input_rows_count; ++offset)
{
const size_t offset_ipv6 = offset * IPV6_BINARY_LENGTH;
const UInt8 * ip = col_const_ip_in
? col_const_ip_value
: &col_ip_in->getChars()[offset_ipv6];
UInt8 cidr = col_const_cidr_in
? col_const_cidr_in->getValue<UInt8>()
: col_cidr_in->getData()[offset];
cidr = std::min(cidr, max_cidr_mask);
applyCIDRMask(&vec_in[offset_ipv6], &vec_res_lower_range[offset_ipv6], &vec_res_upper_range[offset_ipv6], cidr);
applyCIDRMask(ip, &vec_res_lower_range[offset_ipv6], &vec_res_upper_range[offset_ipv6], cidr);
}
columns[result].column = ColumnTuple::create(Columns{std::move(col_res_lower_range), std::move(col_res_upper_range)});
return ColumnTuple::create(Columns{std::move(col_res_lower_range), std::move(col_res_upper_range)});
}
};
@ -1763,7 +1773,7 @@ public:
if (!isUInt8(second_argument))
throw Exception{"Illegal type " + second_argument->getName()
+ " of second argument of function " + getName()
+ ", expected numeric type.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+ ", expected UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
DataTypePtr element = DataTypeFactory::instance().get("IPv4");
return std::make_shared<DataTypeTuple>(DataTypes{element, element});
@ -1772,30 +1782,29 @@ public:
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const auto & col_type_name_ip = columns[arguments[0]];
const auto & col_type_name_ip = arguments[0];
const ColumnPtr & column_ip = col_type_name_ip.column;
const auto col_ip_in = checkAndGetColumn<ColumnUInt32>(column_ip.get());
if (!col_ip_in)
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
const auto * col_const_ip_in = checkAndGetColumnConst<ColumnUInt32>(column_ip.get());
const auto * col_ip_in = checkAndGetColumn<ColumnUInt32>(column_ip.get());
if (!col_const_ip_in && !col_ip_in)
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
const auto & col_type_name_cidr = columns[arguments[1]];
const auto & col_type_name_cidr = arguments[1];
const ColumnPtr & column_cidr = col_type_name_cidr.column;
const auto col_const_cidr_in = checkAndGetColumnConst<ColumnUInt8>(column_cidr.get());
const auto col_cidr_in = checkAndGetColumn<ColumnUInt8>(column_cidr.get());
const auto * col_const_cidr_in = checkAndGetColumnConst<ColumnUInt8>(column_cidr.get());
const auto * col_cidr_in = checkAndGetColumn<ColumnUInt8>(column_cidr.get());
if (!col_const_cidr_in && !col_cidr_in)
throw Exception("Illegal column " + columns[arguments[1]].column->getName()
throw Exception("Illegal column " + arguments[1].column->getName()
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
const auto & vec_in = col_ip_in->getData();
auto col_res_lower_range = ColumnUInt32::create();
auto col_res_upper_range = ColumnUInt32::create();
@ -1807,14 +1816,18 @@ public:
for (size_t i = 0; i < input_rows_count; ++i)
{
UInt32 ip = col_const_ip_in
? col_const_ip_in->getValue<UInt32>()
: col_ip_in->getData()[i];
UInt8 cidr = col_const_cidr_in
? col_const_cidr_in->getValue<UInt8>()
: col_cidr_in->getData()[i];
std::tie(vec_res_lower_range[i], vec_res_upper_range[i]) = applyCIDRMask(vec_in[i], cidr);
std::tie(vec_res_lower_range[i], vec_res_upper_range[i]) = applyCIDRMask(ip, cidr);
}
columns[result].column = ColumnTuple::create(Columns{std::move(col_res_lower_range), std::move(col_res_upper_range)});
return ColumnTuple::create(Columns{std::move(col_res_lower_range), std::move(col_res_upper_range)});
}
};

316
src/Functions/FunctionsComparison.h
View File

@ -561,7 +561,7 @@ public:
static constexpr auto name = Name::name;
static FunctionPtr create(const Context & context) { return std::make_shared<FunctionComparison>(context); }
FunctionComparison(const Context & context_)
explicit FunctionComparison(const Context & context_)
: context(context_),
check_decimal_overflow(decimalCheckComparisonOverflow(context))
{}
@ -571,7 +571,7 @@ private:
bool check_decimal_overflow = true;
template <typename T0, typename T1>
bool executeNumRightType(ColumnsWithTypeAndName & columns, size_t result, const ColumnVector<T0> * col_left, const IColumn * col_right_untyped) const
ColumnPtr executeNumRightType(const ColumnVector<T0> * col_left, const IColumn * col_right_untyped) const
{
if (const ColumnVector<T1> * col_right = checkAndGetColumn<ColumnVector<T1>>(col_right_untyped))
{
@ -581,8 +581,7 @@ private:
vec_res.resize(col_left->getData().size());
NumComparisonImpl<T0, T1, Op<T0, T1>>::vectorVector(col_left->getData(), col_right->getData(), vec_res);
columns[result].column = std::move(col_res);
return true;
return col_res;
}
else if (auto col_right_const = checkAndGetColumnConst<ColumnVector<T1>>(col_right_untyped))
{
@ -592,15 +591,14 @@ private:
vec_res.resize(col_left->size());
NumComparisonImpl<T0, T1, Op<T0, T1>>::vectorConstant(col_left->getData(), col_right_const->template getValue<T1>(), vec_res);
columns[result].column = std::move(col_res);
return true;
return col_res;
}
return false;
return nullptr;
}
template <typename T0, typename T1>
bool executeNumConstRightType(ColumnsWithTypeAndName & columns, size_t result, const ColumnConst * col_left, const IColumn * col_right_untyped) const
ColumnPtr executeNumConstRightType(const ColumnConst * col_left, const IColumn * col_right_untyped) const
{
if (const ColumnVector<T1> * col_right = checkAndGetColumn<ColumnVector<T1>>(col_right_untyped))
{
@ -610,41 +608,40 @@ private:
vec_res.resize(col_left->size());
NumComparisonImpl<T0, T1, Op<T0, T1>>::constantVector(col_left->template getValue<T0>(), col_right->getData(), vec_res);
columns[result].column = std::move(col_res);
return true;
return col_res;
}
else if (auto col_right_const = checkAndGetColumnConst<ColumnVector<T1>>(col_right_untyped))
{
UInt8 res = 0;
NumComparisonImpl<T0, T1, Op<T0, T1>>::constantConstant(col_left->template getValue<T0>(), col_right_const->template getValue<T1>(), res);
columns[result].column = DataTypeUInt8().createColumnConst(col_left->size(), toField(res));
return true;
return DataTypeUInt8().createColumnConst(col_left->size(), toField(res));
}
return false;
return nullptr;
}
template <typename T0>
bool executeNumLeftType(ColumnsWithTypeAndName & columns, size_t result, const IColumn * col_left_untyped, const IColumn * col_right_untyped) const
ColumnPtr executeNumLeftType(const IColumn * col_left_untyped, const IColumn * col_right_untyped) const
{
ColumnPtr res = nullptr;
if (const ColumnVector<T0> * col_left = checkAndGetColumn<ColumnVector<T0>>(col_left_untyped))
{
if ( executeNumRightType<T0, UInt8>(columns, result, col_left, col_right_untyped)
|| executeNumRightType<T0, UInt16>(columns, result, col_left, col_right_untyped)
|| executeNumRightType<T0, UInt32>(columns, result, col_left, col_right_untyped)
|| executeNumRightType<T0, UInt64>(columns, result, col_left, col_right_untyped)
|| executeNumRightType<T0, UInt128>(columns, result, col_left, col_right_untyped)
|| executeNumRightType<T0, UInt256>(columns, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int8>(columns, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int16>(columns, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int32>(columns, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int64>(columns, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int128>(columns, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int256>(columns, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Float32>(columns, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Float64>(columns, result, col_left, col_right_untyped))
return true;
if ( (res = executeNumRightType<T0, UInt8>(col_left, col_right_untyped))
|| (res = executeNumRightType<T0, UInt16>(col_left, col_right_untyped))
|| (res = executeNumRightType<T0, UInt32>(col_left, col_right_untyped))
|| (res = executeNumRightType<T0, UInt64>(col_left, col_right_untyped))
|| (res = executeNumRightType<T0, UInt128>(col_left, col_right_untyped))
|| (res = executeNumRightType<T0, UInt256>(col_left, col_right_untyped))
|| (res = executeNumRightType<T0, Int8>(col_left, col_right_untyped))
|| (res = executeNumRightType<T0, Int16>(col_left, col_right_untyped))
|| (res = executeNumRightType<T0, Int32>(col_left, col_right_untyped))
|| (res = executeNumRightType<T0, Int64>(col_left, col_right_untyped))
|| (res = executeNumRightType<T0, Int128>(col_left, col_right_untyped))
|| (res = executeNumRightType<T0, Int256>(col_left, col_right_untyped))
|| (res = executeNumRightType<T0, Float32>(col_left, col_right_untyped))
|| (res = executeNumRightType<T0, Float64>(col_left, col_right_untyped)))
return res;
else
throw Exception("Illegal column " + col_right_untyped->getName()
+ " of second argument of function " + getName(),
@ -652,34 +649,35 @@ private:
}
else if (auto col_left_const = checkAndGetColumnConst<ColumnVector<T0>>(col_left_untyped))
{
if ( executeNumConstRightType<T0, UInt8>(columns, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, UInt16>(columns, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, UInt32>(columns, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, UInt64>(columns, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, UInt128>(columns, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, UInt256>(columns, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int8>(columns, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int16>(columns, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int32>(columns, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int64>(columns, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int128>(columns, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int256>(columns, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Float32>(columns, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Float64>(columns, result, col_left_const, col_right_untyped))
return true;
if ( (res = executeNumConstRightType<T0, UInt8>(col_left_const, col_right_untyped))
|| (res = executeNumConstRightType<T0, UInt16>(col_left_const, col_right_untyped))
|| (res = executeNumConstRightType<T0, UInt32>(col_left_const, col_right_untyped))
|| (res = executeNumConstRightType<T0, UInt64>(col_left_const, col_right_untyped))
|| (res = executeNumConstRightType<T0, UInt128>(col_left_const, col_right_untyped))
|| (res = executeNumConstRightType<T0, UInt256>(col_left_const, col_right_untyped))
|| (res = executeNumConstRightType<T0, Int8>(col_left_const, col_right_untyped))
|| (res = executeNumConstRightType<T0, Int16>(col_left_const, col_right_untyped))
|| (res = executeNumConstRightType<T0, Int32>(col_left_const, col_right_untyped))
|| (res = executeNumConstRightType<T0, Int64>(col_left_const, col_right_untyped))
|| (res = executeNumConstRightType<T0, Int128>(col_left_const, col_right_untyped))
|| (res = executeNumConstRightType<T0, Int256>(col_left_const, col_right_untyped))
|| (res = executeNumConstRightType<T0, Float32>(col_left_const, col_right_untyped))
|| (res = executeNumConstRightType<T0, Float64>(col_left_const, col_right_untyped)))
return res;
else
throw Exception("Illegal column " + col_right_untyped->getName()
+ " of second argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
return false;
return nullptr;
}
void executeDecimal(ColumnsWithTypeAndName & columns, size_t result, const ColumnWithTypeAndName & col_left, const ColumnWithTypeAndName & col_right) const
ColumnPtr executeDecimal(const ColumnWithTypeAndName & col_left, const ColumnWithTypeAndName & col_right) const
{
TypeIndex left_number = col_left.type->getTypeId();
TypeIndex right_number = col_right.type->getTypeId();
ColumnPtr res;
auto call = [&](const auto & types) -> bool
{
@ -688,18 +686,19 @@ private:
using RightDataType = typename Types::RightType;
if (check_decimal_overflow)
DecimalComparison<LeftDataType, RightDataType, Op, true>(columns, result, col_left, col_right);
return (res = DecimalComparison<LeftDataType, RightDataType, Op, true>::apply(col_left, col_right)) != nullptr;
else
DecimalComparison<LeftDataType, RightDataType, Op, false>(columns, result, col_left, col_right);
return true;
return (res = DecimalComparison<LeftDataType, RightDataType, Op, false>::apply(col_left, col_right)) != nullptr;
};
if (!callOnBasicTypes<true, false, true, true>(left_number, right_number, call))
throw Exception("Wrong call for " + getName() + " with " + col_left.type->getName() + " and " + col_right.type->getName(),
ErrorCodes::LOGICAL_ERROR);
return res;
}
bool executeString(ColumnsWithTypeAndName & columns, size_t result, const IColumn * c0, const IColumn * c1) const
ColumnPtr executeString(const IColumn * c0, const IColumn * c1) const
{
const ColumnString * c0_string = checkAndGetColumn<ColumnString>(c0);
const ColumnString * c1_string = checkAndGetColumn<ColumnString>(c1);
@ -710,7 +709,7 @@ private:
const ColumnConst * c1_const = checkAndGetColumnConstStringOrFixedString(c1);
if (!((c0_string || c0_fixed_string || c0_const) && (c1_string || c1_fixed_string || c1_const)))
return false;
return nullptr;
const ColumnString::Chars * c0_const_chars = nullptr;
const ColumnString::Chars * c1_const_chars = nullptr;
@ -759,12 +758,11 @@ private:
if (c0_const && c1_const)
{
auto res = executeString(columns, result, &c0_const->getDataColumn(), &c1_const->getDataColumn());
auto res = executeString(&c0_const->getDataColumn(), &c1_const->getDataColumn());
if (!res)
return false;
return nullptr;
columns[result].column = ColumnConst::create(columns[result].column, c0_const->size());
return true;
return ColumnConst::create(res, c0_const->size());
}
else
{
@ -818,13 +816,12 @@ private:
+ " of arguments of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
columns[result].column = std::move(c_res);
return true;
return c_res;
}
}
bool executeWithConstString(
ColumnsWithTypeAndName & columns, size_t result, const IColumn * col_left_untyped, const IColumn * col_right_untyped,
ColumnPtr executeWithConstString(
const DataTypePtr & result_type, const IColumn * col_left_untyped, const IColumn * col_right_untyped,
const DataTypePtr & left_type, const DataTypePtr & right_type, size_t input_rows_count) const
{
/// To compare something with const string, we cast constant to appropriate type and compare as usual.
@ -835,7 +832,7 @@ private:
const ColumnConst * right_const = checkAndGetColumnConstStringOrFixedString(col_right_untyped);
if (!left_const && !right_const)
return false;
return nullptr;
const IDataType * type_string = left_const ? left_type.get() : right_type.get();
const DataTypePtr & type_to_compare = !left_const ? left_type : right_type;
@ -846,28 +843,24 @@ private:
/// If not possible to convert, comparison with =, <, >, <=, >= yields to false and comparison with != yields to true.
if (converted.isNull())
{
columns[result].column = DataTypeUInt8().createColumnConst(input_rows_count, IsOperation<Op>::not_equals);
return DataTypeUInt8().createColumnConst(input_rows_count, IsOperation<Op>::not_equals);
}
else
{
auto column_converted = type_to_compare->createColumnConst(input_rows_count, converted);
ColumnsWithTypeAndName tmp_columns_columns
ColumnsWithTypeAndName tmp_columns
{
{ left_const ? column_converted : col_left_untyped->getPtr(), type_to_compare, "" },
{ !left_const ? column_converted : col_right_untyped->getPtr(), type_to_compare, "" },
columns[result]
};
executeImpl(tmp_columns_columns, {0, 1}, 2, input_rows_count);
columns[result].column = std::move(tmp_columns_columns[2].column);
return executeImpl(tmp_columns, result_type, input_rows_count);
}
}
return true;
}
void executeTuple(ColumnsWithTypeAndName & columns, size_t result, const ColumnWithTypeAndName & c0, const ColumnWithTypeAndName & c1,
ColumnPtr executeTuple(
const DataTypePtr & result_type, const ColumnWithTypeAndName & c0, const ColumnWithTypeAndName & c1,
size_t input_rows_count) const
{
/** We will lexicographically compare the tuples. This is done as follows:
@ -892,12 +885,8 @@ private:
if (tuple_size != typeid_cast<const DataTypeTuple &>(*c1.type).getElements().size())
throw Exception("Cannot compare tuples of different sizes.", ErrorCodes::BAD_ARGUMENTS);
auto & res = columns[result];
if (res.type->onlyNull())
{
res.column = res.type->createColumnConstWithDefaultValue(input_rows_count);
return;
}
if (result_type->onlyNull())
return result_type->createColumnConstWithDefaultValue(input_rows_count);
ColumnsWithTypeAndName x(tuple_size);
ColumnsWithTypeAndName y(tuple_size);
@ -927,18 +916,16 @@ private:
y[i].column = y_columns[i];
}
executeTupleImpl(columns, result, x, y, tuple_size, input_rows_count);
return executeTupleImpl(x, y, tuple_size, input_rows_count);
}
void executeTupleImpl(ColumnsWithTypeAndName & columns, size_t result, const ColumnsWithTypeAndName & x,
ColumnPtr executeTupleImpl(const ColumnsWithTypeAndName & x,
const ColumnsWithTypeAndName & y, size_t tuple_size,
size_t input_rows_count) const;
void executeTupleEqualityImpl(
ColumnPtr executeTupleEqualityImpl(
std::shared_ptr<IFunctionOverloadResolver> func_compare,
std::shared_ptr<IFunctionOverloadResolver> func_convolution,
ColumnsWithTypeAndName & columns,
size_t result,
const ColumnsWithTypeAndName & x,
const ColumnsWithTypeAndName & y,
size_t tuple_size,
@ -947,84 +934,70 @@ private:
if (0 == tuple_size)
throw Exception("Comparison of zero-sized tuples is not implemented.", ErrorCodes::NOT_IMPLEMENTED);
ColumnsWithTypeAndName convolution_types(tuple_size);
ColumnsWithTypeAndName convolution_columns(tuple_size);
ColumnsWithTypeAndName tmp_columns(2);
ColumnsWithTypeAndName tmp_columns;
for (size_t i = 0; i < tuple_size; ++i)
{
tmp_columns.emplace_back(x[i]);
tmp_columns.emplace_back(y[i]);
tmp_columns[0] = x[i];
tmp_columns[1] = y[i];
auto impl = func_compare->build({x[i], y[i]});
convolution_types[i].type = impl->getReturnType();
auto impl = func_compare->build(tmp_columns);
convolution_columns[i].type = impl->getResultType();
/// Comparison of the elements.
tmp_columns.emplace_back(ColumnWithTypeAndName{ nullptr, impl->getReturnType(), "" });
impl->execute(tmp_columns, {i * 3, i * 3 + 1}, i * 3 + 2, input_rows_count);
convolution_columns[i].column = impl->execute(tmp_columns, impl->getResultType(), input_rows_count);
}
if (tuple_size == 1)
{
/// Do not call AND for single-element tuple.
columns[result].column = tmp_columns[2].column;
return;
return convolution_columns[0].column;
}
/// Logical convolution.
ColumnNumbers convolution_args(tuple_size);
for (size_t i = 0; i < tuple_size; ++i)
convolution_args[i] = i * 3 + 2;
auto impl = func_convolution->build(convolution_types);
tmp_columns.emplace_back(ColumnWithTypeAndName{ nullptr, impl->getReturnType(), "" });
impl->execute(tmp_columns, convolution_args, tuple_size * 3, input_rows_count);
columns[result].column = tmp_columns[tuple_size * 3].column;
auto impl = func_convolution->build(convolution_columns);
return impl->execute(convolution_columns, impl->getResultType(), input_rows_count);
}
void executeTupleLessGreaterImpl(
ColumnPtr executeTupleLessGreaterImpl(
std::shared_ptr<IFunctionOverloadResolver> func_compare_head,
std::shared_ptr<IFunctionOverloadResolver> func_compare_tail,
std::shared_ptr<IFunctionOverloadResolver> func_and,
std::shared_ptr<IFunctionOverloadResolver> func_or,
std::shared_ptr<IFunctionOverloadResolver> func_equals,
ColumnsWithTypeAndName & columns,
size_t result,
const ColumnsWithTypeAndName & x,
const ColumnsWithTypeAndName & y,
size_t tuple_size,
size_t input_rows_count) const
{
ColumnsWithTypeAndName tmp_columns;
ColumnsWithTypeAndName less_columns(tuple_size);
ColumnsWithTypeAndName equal_columns(tuple_size - 1);
ColumnsWithTypeAndName tmp_columns(2);
/// Pairwise comparison of the inequality of all elements; on the equality of all elements except the last.
/// (x[i], y[i], x[i] < y[i], x[i] == y[i])
for (size_t i = 0; i < tuple_size; ++i)
{
tmp_columns.emplace_back(x[i]);
tmp_columns.emplace_back(y[i]);
tmp_columns.emplace_back(ColumnWithTypeAndName()); // pos == i * 4 + 2
tmp_columns[0] = x[i];
tmp_columns[1] = y[i];
if (i + 1 != tuple_size)
{
auto impl_head = func_compare_head->build({x[i], y[i]});
tmp_columns[i * 4 + 2].type = impl_head->getReturnType();
impl_head->execute(tmp_columns, {i * 4, i * 4 + 1}, i * 4 + 2, input_rows_count);
auto impl_head = func_compare_head->build(tmp_columns);
less_columns[i].type = impl_head->getResultType();
less_columns[i].column = impl_head->execute(tmp_columns, less_columns[i].type, input_rows_count);
tmp_columns.emplace_back(ColumnWithTypeAndName()); // i * 4 + 3
auto impl_equals = func_equals->build({x[i], y[i]});
tmp_columns[i * 4 + 3].type = impl_equals->getReturnType();
impl_equals->execute(tmp_columns, {i * 4, i * 4 + 1}, i * 4 + 3, input_rows_count);
auto impl_equals = func_equals->build(tmp_columns);
equal_columns[i].type = impl_equals->getResultType();
equal_columns[i].column = impl_equals->execute(tmp_columns, equal_columns[i].type, input_rows_count);
}
else
{
auto impl_tail = func_compare_tail->build({x[i], y[i]});
tmp_columns[i * 4 + 2].type = impl_tail->getReturnType();
impl_tail->execute(tmp_columns, {i * 4, i * 4 + 1}, i * 4 + 2, input_rows_count);
auto impl_tail = func_compare_tail->build(tmp_columns);
less_columns[i].type = impl_tail->getResultType();
less_columns[i].column = impl_tail->execute(tmp_columns, less_columns[i].type, input_rows_count);
}
}
@ -1035,38 +1008,28 @@ private:
/// for (int i = tuple_size - 2; i >= 0; --i)
/// res = (res && `x == y`[i]) || `x < y`[i];
size_t i = tuple_size - 1;
tmp_columns[0] = less_columns[i];
while (i > 0)
{
--i;
size_t and_lhs_pos = tmp_columns.size() - 1; // res
size_t and_rhs_pos = i * 4 + 3; // `x == y`[i]
tmp_columns.emplace_back(ColumnWithTypeAndName());
tmp_columns[1] = equal_columns[i];
auto func_and_adaptor = func_and->build(tmp_columns);
ColumnsWithTypeAndName and_args = {{ nullptr, tmp_columns[and_lhs_pos].type, "" },
{ nullptr, tmp_columns[and_rhs_pos].type, "" }};
tmp_columns[0].column = func_and_adaptor->execute(tmp_columns, func_and_adaptor->getResultType(), input_rows_count);
tmp_columns[0].type = func_and_adaptor->getResultType();
auto func_and_adaptor = func_and->build(and_args);
tmp_columns[tmp_columns.size() - 1].type = func_and_adaptor->getReturnType();
func_and_adaptor->execute(tmp_columns, {and_lhs_pos, and_rhs_pos}, tmp_columns.size() - 1, input_rows_count);
size_t or_lhs_pos = tmp_columns.size() - 1; // (res && `x == y`[i])
size_t or_rhs_pos = i * 4 + 2; // `x < y`[i]
tmp_columns.emplace_back(ColumnWithTypeAndName());
ColumnsWithTypeAndName or_args = {{ nullptr, tmp_columns[or_lhs_pos].type, "" },
{ nullptr, tmp_columns[or_rhs_pos].type, "" }};
auto func_or_adaptor = func_or->build(or_args);
tmp_columns[tmp_columns.size() - 1].type = func_or_adaptor->getReturnType();
func_or_adaptor->execute(tmp_columns, {or_lhs_pos, or_rhs_pos}, tmp_columns.size() - 1, input_rows_count);
tmp_columns[1] = less_columns[i];
auto func_or_adaptor = func_or->build(tmp_columns);
tmp_columns[0].column = func_or_adaptor->execute(tmp_columns, func_or_adaptor->getResultType(), input_rows_count);
tmp_columns[tmp_columns.size() - 1].type = func_or_adaptor->getResultType();
}
columns[result].column = tmp_columns[tmp_columns.size() - 1].column;
return tmp_columns[0].column;
}
void executeGenericIdenticalTypes(ColumnsWithTypeAndName & columns, size_t result, const IColumn * c0, const IColumn * c1) const
ColumnPtr executeGenericIdenticalTypes(const IColumn * c0, const IColumn * c1) const
{
bool c0_const = isColumnConst(*c0);
bool c1_const = isColumnConst(*c1);
@ -1075,7 +1038,7 @@ private:
{
UInt8 res = 0;
GenericComparisonImpl<Op<int, int>>::constantConstant(*c0, *c1, res);
columns[result].column = DataTypeUInt8().createColumnConst(c0->size(), toField(res));
return DataTypeUInt8().createColumnConst(c0->size(), toField(res));
}
else
{
@ -1090,18 +1053,18 @@ private:
else
GenericComparisonImpl<Op<int, int>>::vectorVector(*c0, *c1, vec_res);
columns[result].column = std::move(c_res);
return c_res;
}
}
void executeGeneric(ColumnsWithTypeAndName & columns, size_t result, const ColumnWithTypeAndName & c0, const ColumnWithTypeAndName & c1) const
ColumnPtr executeGeneric(const ColumnWithTypeAndName & c0, const ColumnWithTypeAndName & c1) const
{
DataTypePtr common_type = getLeastSupertype({c0.type, c1.type});
ColumnPtr c0_converted = castColumn(c0, common_type);
ColumnPtr c1_converted = castColumn(c1, common_type);
executeGenericIdenticalTypes(columns, result, c0_converted.get(), c1_converted.get());
return executeGenericIdenticalTypes(c0_converted.get(), c1_converted.get());
}
public:
@ -1157,7 +1120,7 @@ public:
{
ColumnsWithTypeAndName args = {{nullptr, left_tuple->getElements()[i], ""},
{nullptr, right_tuple->getElements()[i], ""}};
auto element_type = adaptor.build(args)->getReturnType();
auto element_type = adaptor.build(args)->getResultType();
has_nullable = has_nullable || element_type->isNullable();
has_null = has_null || element_type->onlyNull();
}
@ -1173,10 +1136,10 @@ public:
return std::make_shared<DataTypeUInt8>();
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
const auto & col_with_type_and_name_left = columns[arguments[0]];
const auto & col_with_type_and_name_right = columns[arguments[1]];
const auto & col_with_type_and_name_left = arguments[0];
const auto & col_with_type_and_name_right = arguments[1];
const IColumn * col_left_untyped = col_with_type_and_name_left.column.get();
const IColumn * col_right_untyped = col_with_type_and_name_right.column.get();
@ -1194,13 +1157,11 @@ public:
|| IsOperation<Op>::less_or_equals
|| IsOperation<Op>::greater_or_equals)
{
columns[result].column = DataTypeUInt8().createColumnConst(input_rows_count, 1u);
return;
return DataTypeUInt8().createColumnConst(input_rows_count, 1u);
}
else
{
columns[result].column = DataTypeUInt8().createColumnConst(input_rows_count, 0u);
return;
return DataTypeUInt8().createColumnConst(input_rows_count, 0u);
}
}
@ -1216,39 +1177,44 @@ public:
bool date_and_datetime = (which_left.idx != which_right.idx) &&
which_left.isDateOrDateTime() && which_right.isDateOrDateTime();
ColumnPtr res;
if (left_is_num && right_is_num && !date_and_datetime)
{
if (!(executeNumLeftType<UInt8>(columns, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<UInt16>(columns, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<UInt32>(columns, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<UInt64>(columns, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<UInt128>(columns, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<UInt256>(columns, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int8>(columns, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int16>(columns, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int32>(columns, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int64>(columns, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int128>(columns, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int256>(columns, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Float32>(columns, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Float64>(columns, result, col_left_untyped, col_right_untyped)))
if (!((res = executeNumLeftType<UInt8>(col_left_untyped, col_right_untyped))
|| (res = executeNumLeftType<UInt16>(col_left_untyped, col_right_untyped))
|| (res = executeNumLeftType<UInt32>(col_left_untyped, col_right_untyped))
|| (res = executeNumLeftType<UInt64>(col_left_untyped, col_right_untyped))
|| (res = executeNumLeftType<UInt128>(col_left_untyped, col_right_untyped))
|| (res = executeNumLeftType<UInt256>(col_left_untyped, col_right_untyped))
|| (res = executeNumLeftType<Int8>(col_left_untyped, col_right_untyped))
|| (res = executeNumLeftType<Int16>(col_left_untyped, col_right_untyped))
|| (res = executeNumLeftType<Int32>(col_left_untyped, col_right_untyped))
|| (res = executeNumLeftType<Int64>(col_left_untyped, col_right_untyped))
|| (res = executeNumLeftType<Int128>(col_left_untyped, col_right_untyped))
|| (res = executeNumLeftType<Int256>(col_left_untyped, col_right_untyped))
|| (res = executeNumLeftType<Float32>(col_left_untyped, col_right_untyped))
|| (res = executeNumLeftType<Float64>(col_left_untyped, col_right_untyped))))
throw Exception("Illegal column " + col_left_untyped->getName()
+ " of first argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
return res;
}
else if (checkAndGetDataType<DataTypeTuple>(left_type.get())
&& checkAndGetDataType<DataTypeTuple>(right_type.get()))
{
executeTuple(columns, result, col_with_type_and_name_left, col_with_type_and_name_right, input_rows_count);
return executeTuple(result_type, col_with_type_and_name_left, col_with_type_and_name_right, input_rows_count);
}
else if (left_is_string && right_is_string && executeString(columns, result, col_left_untyped, col_right_untyped))
else if (left_is_string && right_is_string && (res = executeString(col_left_untyped, col_right_untyped)))
{
return res;
}
else if (executeWithConstString(
columns, result, col_left_untyped, col_right_untyped,
else if ((res = executeWithConstString(
result_type, col_left_untyped, col_right_untyped,
left_type, right_type,
input_rows_count))
input_rows_count)))
{
return res;
}
else if (isColumnedAsDecimal(left_type) || isColumnedAsDecimal(right_type))
{
@ -1257,15 +1223,15 @@ public:
throw Exception("No operation " + getName() + " between " + left_type->getName() + " and " + right_type->getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
executeDecimal(columns, result, col_with_type_and_name_left, col_with_type_and_name_right);
return executeDecimal(col_with_type_and_name_left, col_with_type_and_name_right);
}
else if (left_type->equals(*right_type))
{
executeGenericIdenticalTypes(columns, result, col_left_untyped, col_right_untyped);
return executeGenericIdenticalTypes(col_left_untyped, col_right_untyped);
}
else
{
executeGeneric(columns, result, col_with_type_and_name_left, col_with_type_and_name_right);
return executeGeneric(col_with_type_and_name_left, col_with_type_and_name_right);
}
}

16
src/Functions/FunctionsConsistentHashing.h
View File

@ -65,10 +65,10 @@ public:
return {1};
}
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
if (isColumnConst(*columns[arguments[1]].column))
executeConstBuckets(columns, arguments, result);
if (isColumnConst(*arguments[1].column))
return executeConstBuckets(arguments);
else
throw Exception(
"The second argument of function " + getName() + " (number of buckets) must be constant", ErrorCodes::BAD_ARGUMENTS);
@ -93,9 +93,9 @@ private:
return static_cast<BucketsType>(buckets);
}
void executeConstBuckets(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result) const
ColumnPtr executeConstBuckets(ColumnsWithTypeAndName & arguments) const
{
Field buckets_field = (*columns[arguments[1]].column)[0];
Field buckets_field = (*arguments[1].column)[0];
BucketsType num_buckets;
if (buckets_field.getType() == Field::Types::Int64)
@ -106,8 +106,8 @@ private:
throw Exception("Illegal type " + String(buckets_field.getTypeName()) + " of the second argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
const auto & hash_col = columns[arguments[0]].column;
const IDataType * hash_type = columns[arguments[0]].type.get();
const auto & hash_col = arguments[0].column;
const IDataType * hash_type = arguments[0].type.get();
auto res_col = ColumnVector<ResultType>::create();
WhichDataType which(hash_type);
@ -132,7 +132,7 @@ private:
throw Exception("Illegal type " + hash_type->getName() + " of the first argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
columns[result].column = std::move(res_col);
return res_col;
}
template <typename CurrentHashType>

452
src/Functions/FunctionsConversion.h
View File

File diff suppressed because it is too large Load Diff

62
src/Functions/FunctionsEmbeddedDictionaries.h
View File

@ -183,17 +183,17 @@ public:
bool isDeterministic() const override { return false; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
/// The dictionary key that defines the "point of view".
std::string dict_key;
if (arguments.size() == 2)
{
const ColumnConst * key_col = checkAndGetColumnConst<ColumnString>(columns[arguments[1]].column.get());
const ColumnConst * key_col = checkAndGetColumnConst<ColumnString>(arguments[1].column.get());
if (!key_col)
throw Exception("Illegal column " + columns[arguments[1]].column->getName()
throw Exception("Illegal column " + arguments[1].column->getName()
+ " of second ('point of view') argument of function " + name
+ ". Must be constant string.",
ErrorCodes::ILLEGAL_COLUMN);
@ -203,7 +203,7 @@ public:
const typename DictGetter::Dst & dict = DictGetter::get(*owned_dict, dict_key);
if (const ColumnVector<T> * col_from = checkAndGetColumn<ColumnVector<T>>(columns[arguments[0]].column.get()))
if (const ColumnVector<T> * col_from = checkAndGetColumn<ColumnVector<T>>(arguments[0].column.get()))
{
auto col_to = ColumnVector<T>::create();
@ -215,10 +215,10 @@ public:
for (size_t i = 0; i < size; ++i)
vec_to[i] = Transform::apply(vec_from[i], dict);
columns[result].column = std::move(col_to);
return col_to;
}
else
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of first argument of function " + name,
ErrorCodes::ILLEGAL_COLUMN);
}
@ -279,17 +279,17 @@ public:
bool isDeterministic() const override { return false; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
/// The dictionary key that defines the "point of view".
std::string dict_key;
if (arguments.size() == 3)
{
const ColumnConst * key_col = checkAndGetColumnConst<ColumnString>(columns[arguments[2]].column.get());
const ColumnConst * key_col = checkAndGetColumnConst<ColumnString>(arguments[2].column.get());
if (!key_col)
throw Exception("Illegal column " + columns[arguments[2]].column->getName()
throw Exception("Illegal column " + arguments[2].column->getName()
+ " of third ('point of view') argument of function " + name
+ ". Must be constant string.",
ErrorCodes::ILLEGAL_COLUMN);
@ -299,10 +299,10 @@ public:
const typename DictGetter::Dst & dict = DictGetter::get(*owned_dict, dict_key);
const ColumnVector<T> * col_vec1 = checkAndGetColumn<ColumnVector<T>>(columns[arguments[0]].column.get());
const ColumnVector<T> * col_vec2 = checkAndGetColumn<ColumnVector<T>>(columns[arguments[1]].column.get());
const ColumnConst * col_const1 = checkAndGetColumnConst<ColumnVector<T>>(columns[arguments[0]].column.get());
const ColumnConst * col_const2 = checkAndGetColumnConst<ColumnVector<T>>(columns[arguments[1]].column.get());
const ColumnVector<T> * col_vec1 = checkAndGetColumn<ColumnVector<T>>(arguments[0].column.get());
const ColumnVector<T> * col_vec2 = checkAndGetColumn<ColumnVector<T>>(arguments[1].column.get());
const ColumnConst * col_const1 = checkAndGetColumnConst<ColumnVector<T>>(arguments[0].column.get());
const ColumnConst * col_const2 = checkAndGetColumnConst<ColumnVector<T>>(arguments[1].column.get());
if (col_vec1 && col_vec2)
{
@ -317,7 +317,7 @@ public:
for (size_t i = 0; i < size; ++i)
vec_to[i] = Transform::apply(vec_from1[i], vec_from2[i], dict);
columns[result].column = std::move(col_to);
return col_to;
}
else if (col_vec1 && col_const2)
{
@ -332,7 +332,7 @@ public:
for (size_t i = 0; i < size; ++i)
vec_to[i] = Transform::apply(vec_from1[i], const_from2, dict);
columns[result].column = std::move(col_to);
return col_to;
}
else if (col_const1 && col_vec2)
{
@ -347,16 +347,16 @@ public:
for (size_t i = 0; i < size; ++i)
vec_to[i] = Transform::apply(const_from1, vec_from2[i], dict);
columns[result].column = std::move(col_to);
return col_to;
}
else if (col_const1 && col_const2)
{
columns[result].column = DataTypeUInt8().createColumnConst(col_const1->size(),
return DataTypeUInt8().createColumnConst(col_const1->size(),
toField(Transform::apply(col_const1->template getValue<T>(), col_const2->template getValue<T>(), dict)));
}
else
throw Exception("Illegal columns " + columns[arguments[0]].column->getName()
+ " and " + columns[arguments[1]].column->getName()
throw Exception("Illegal columns " + arguments[0].column->getName()
+ " and " + arguments[1].column->getName()
+ " of arguments of function " + name,
ErrorCodes::ILLEGAL_COLUMN);
}
@ -415,17 +415,17 @@ public:
bool isDeterministic() const override { return false; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
/// The dictionary key that defines the "point of view".
std::string dict_key;
if (arguments.size() == 2)
{
const ColumnConst * key_col = checkAndGetColumnConst<ColumnString>(columns[arguments[1]].column.get());
const ColumnConst * key_col = checkAndGetColumnConst<ColumnString>(arguments[1].column.get());
if (!key_col)
throw Exception("Illegal column " + columns[arguments[1]].column->getName()
throw Exception("Illegal column " + arguments[1].column->getName()
+ " of second ('point of view') argument of function " + name
+ ". Must be constant string.",
ErrorCodes::ILLEGAL_COLUMN);
@ -435,7 +435,7 @@ public:
const typename DictGetter::Dst & dict = DictGetter::get(*owned_dict, dict_key);
if (const ColumnVector<T> * col_from = checkAndGetColumn<ColumnVector<T>>(columns[arguments[0]].column.get()))
if (const ColumnVector<T> * col_from = checkAndGetColumn<ColumnVector<T>>(arguments[0].column.get()))
{
auto col_values = ColumnVector<T>::create();
auto col_offsets = ColumnArray::ColumnOffsets::create();
@ -459,10 +459,10 @@ public:
res_offsets[i] = res_values.size();
}
columns[result].column = ColumnArray::create(std::move(col_values), std::move(col_offsets));
return ColumnArray::create(std::move(col_values), std::move(col_offsets));
}
else
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of first argument of function " + name,
ErrorCodes::ILLEGAL_COLUMN);
}
@ -620,24 +620,24 @@ public:
bool isDeterministic() const override { return false; }
void executeImpl(ColumnsWithTypeAndName & columns, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
RegionsNames::Language language = RegionsNames::Language::ru;
/// If the result language is specified
if (arguments.size() == 2)
{
if (const ColumnConst * col_language = checkAndGetColumnConst<ColumnString>(columns[arguments[1]].column.get()))
if (const ColumnConst * col_language = checkAndGetColumnConst<ColumnString>(arguments[1].column.get()))
language = RegionsNames::getLanguageEnum(col_language->getValue<String>());
else
throw Exception("Illegal column " + columns[arguments[1]].column->getName()
throw Exception("Illegal column " + arguments[1].column->getName()
+ " of the second argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
const RegionsNames & dict = *owned_dict;
if (const ColumnUInt32 * col_from = typeid_cast<const ColumnUInt32 *>(columns[arguments[0]].column.get()))
if (const ColumnUInt32 * col_from = typeid_cast<const ColumnUInt32 *>(arguments[0].column.get()))
{
auto col_to = ColumnString::create();
@ -649,10 +649,10 @@ public:
col_to->insertDataWithTerminatingZero(name_ref.data, name_ref.size + 1);
}
columns[result].column = std::move(col_to);
return col_to;
}
else
throw Exception("Illegal column " + columns[arguments[0]].column->getName()
throw Exception("Illegal column " + arguments[0].column->getName()
+ " of the first argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}

632
src/Functions/FunctionsExternalDictionaries.h
View File

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More