mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-19 16:20:50 +00:00
Merge branch 'master' into variant_inference
This commit is contained in:
Kruglov Pavel
GitHub
commit
abf25c7c5c
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdlib>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
|
||||
@ -108,6 +108,14 @@ struct make_unsigned // NOLINT(readability-identifier-naming)
|
||||
using type = std::make_unsigned_t<T>;
|
||||
};
|
||||
|
||||
template <> struct make_unsigned<Int8> { using type = UInt8; };
|
||||
template <> struct make_unsigned<UInt8> { using type = UInt8; };
|
||||
template <> struct make_unsigned<Int16> { using type = UInt16; };
|
||||
template <> struct make_unsigned<UInt16> { using type = UInt16; };
|
||||
template <> struct make_unsigned<Int32> { using type = UInt32; };
|
||||
template <> struct make_unsigned<UInt32> { using type = UInt32; };
|
||||
template <> struct make_unsigned<Int64> { using type = UInt64; };
|
||||
template <> struct make_unsigned<UInt64> { using type = UInt64; };
|
||||
template <> struct make_unsigned<Int128> { using type = UInt128; };
|
||||
template <> struct make_unsigned<UInt128> { using type = UInt128; };
|
||||
template <> struct make_unsigned<Int256> { using type = UInt256; };
|
||||
@ -121,6 +129,14 @@ struct make_signed // NOLINT(readability-identifier-naming)
|
||||
using type = std::make_signed_t<T>;
|
||||
};
|
||||
|
||||
template <> struct make_signed<Int8> { using type = Int8; };
|
||||
template <> struct make_signed<UInt8> { using type = Int8; };
|
||||
template <> struct make_signed<Int16> { using type = Int16; };
|
||||
template <> struct make_signed<UInt16> { using type = Int16; };
|
||||
template <> struct make_signed<Int32> { using type = Int32; };
|
||||
template <> struct make_signed<UInt32> { using type = Int32; };
|
||||
template <> struct make_signed<Int64> { using type = Int64; };
|
||||
template <> struct make_signed<UInt64> { using type = Int64; };
|
||||
template <> struct make_signed<Int128> { using type = Int128; };
|
||||
template <> struct make_signed<UInt128> { using type = Int128; };
|
||||
template <> struct make_signed<Int256> { using type = Int256; };
|
||||
|
||||
9
base/base/isSharedPtrUnique.h
Normal file
9
base/base/isSharedPtrUnique.h
Normal file
@ -0,0 +1,9 @@
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
|
||||
template <typename T>
|
||||
bool isSharedPtrUnique(const std::shared_ptr<T> & ptr)
|
||||
{
|
||||
return ptr.use_count() == 1;
|
||||
}
|
||||
@ -232,7 +232,7 @@ void Foundation_API format(
|
||||
const Any & value10);
|
||||
|
||||
|
||||
void Foundation_API format(std::string & result, const std::string & fmt, const std::vector<Any> & values);
|
||||
void Foundation_API formatVector(std::string & result, const std::string & fmt, const std::vector<Any> & values);
|
||||
/// Supports a variable number of arguments and is used by
|
||||
/// all other variants of format().
|
||||
|
||||
|
||||
@ -21,6 +21,8 @@
|
||||
#include "Poco/AtomicCounter.h"
|
||||
#include "Poco/Foundation.h"
|
||||
|
||||
#include <atomic>
|
||||
|
||||
|
||||
namespace Poco
|
||||
{
|
||||
|
||||
@ -51,8 +51,8 @@ namespace
|
||||
}
|
||||
if (width != 0) str.width(width);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
void parsePrec(std::ostream& str, std::string::const_iterator& itFmt, const std::string::const_iterator& endFmt)
|
||||
{
|
||||
if (itFmt != endFmt && *itFmt == '.')
|
||||
@ -67,7 +67,7 @@ namespace
|
||||
if (prec >= 0) str.precision(prec);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
char parseMod(std::string::const_iterator& itFmt, const std::string::const_iterator& endFmt)
|
||||
{
|
||||
char mod = 0;
|
||||
@ -77,13 +77,13 @@ namespace
|
||||
{
|
||||
case 'l':
|
||||
case 'h':
|
||||
case 'L':
|
||||
case 'L':
|
||||
case '?': mod = *itFmt++; break;
|
||||
}
|
||||
}
|
||||
return mod;
|
||||
}
|
||||
|
||||
|
||||
std::size_t parseIndex(std::string::const_iterator& itFmt, const std::string::const_iterator& endFmt)
|
||||
{
|
||||
int index = 0;
|
||||
@ -110,8 +110,8 @@ namespace
|
||||
case 'f': str << std::fixed; break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
void writeAnyInt(std::ostream& str, const Any& any)
|
||||
{
|
||||
if (any.type() == typeid(char))
|
||||
@ -201,7 +201,7 @@ namespace
|
||||
str << RefAnyCast<std::string>(*itVal++);
|
||||
break;
|
||||
case 'z':
|
||||
str << AnyCast<std::size_t>(*itVal++);
|
||||
str << AnyCast<std::size_t>(*itVal++);
|
||||
break;
|
||||
case 'I':
|
||||
case 'D':
|
||||
@ -303,7 +303,7 @@ void format(std::string& result, const std::string& fmt, const Any& value)
|
||||
{
|
||||
std::vector<Any> args;
|
||||
args.push_back(value);
|
||||
format(result, fmt, args);
|
||||
formatVector(result, fmt, args);
|
||||
}
|
||||
|
||||
|
||||
@ -312,7 +312,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
|
||||
std::vector<Any> args;
|
||||
args.push_back(value1);
|
||||
args.push_back(value2);
|
||||
format(result, fmt, args);
|
||||
formatVector(result, fmt, args);
|
||||
}
|
||||
|
||||
|
||||
@ -322,7 +322,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
|
||||
args.push_back(value1);
|
||||
args.push_back(value2);
|
||||
args.push_back(value3);
|
||||
format(result, fmt, args);
|
||||
formatVector(result, fmt, args);
|
||||
}
|
||||
|
||||
|
||||
@ -333,7 +333,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
|
||||
args.push_back(value2);
|
||||
args.push_back(value3);
|
||||
args.push_back(value4);
|
||||
format(result, fmt, args);
|
||||
formatVector(result, fmt, args);
|
||||
}
|
||||
|
||||
|
||||
@ -345,7 +345,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
|
||||
args.push_back(value3);
|
||||
args.push_back(value4);
|
||||
args.push_back(value5);
|
||||
format(result, fmt, args);
|
||||
formatVector(result, fmt, args);
|
||||
}
|
||||
|
||||
|
||||
@ -358,7 +358,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
|
||||
args.push_back(value4);
|
||||
args.push_back(value5);
|
||||
args.push_back(value6);
|
||||
format(result, fmt, args);
|
||||
formatVector(result, fmt, args);
|
||||
}
|
||||
|
||||
|
||||
@ -372,7 +372,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
|
||||
args.push_back(value5);
|
||||
args.push_back(value6);
|
||||
args.push_back(value7);
|
||||
format(result, fmt, args);
|
||||
formatVector(result, fmt, args);
|
||||
}
|
||||
|
||||
|
||||
@ -387,7 +387,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
|
||||
args.push_back(value6);
|
||||
args.push_back(value7);
|
||||
args.push_back(value8);
|
||||
format(result, fmt, args);
|
||||
formatVector(result, fmt, args);
|
||||
}
|
||||
|
||||
|
||||
@ -403,7 +403,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
|
||||
args.push_back(value7);
|
||||
args.push_back(value8);
|
||||
args.push_back(value9);
|
||||
format(result, fmt, args);
|
||||
formatVector(result, fmt, args);
|
||||
}
|
||||
|
||||
|
||||
@ -420,16 +420,16 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
|
||||
args.push_back(value8);
|
||||
args.push_back(value9);
|
||||
args.push_back(value10);
|
||||
format(result, fmt, args);
|
||||
formatVector(result, fmt, args);
|
||||
}
|
||||
|
||||
|
||||
void format(std::string& result, const std::string& fmt, const std::vector<Any>& values)
|
||||
void formatVector(std::string& result, const std::string& fmt, const std::vector<Any>& values)
|
||||
{
|
||||
std::string::const_iterator itFmt = fmt.begin();
|
||||
std::string::const_iterator endFmt = fmt.end();
|
||||
std::vector<Any>::const_iterator itVal = values.begin();
|
||||
std::vector<Any>::const_iterator endVal = values.end();
|
||||
std::vector<Any>::const_iterator endVal = values.end();
|
||||
while (itFmt != endFmt)
|
||||
{
|
||||
switch (*itFmt)
|
||||
|
||||
@ -57,7 +57,7 @@ std::string ObjectId::toString(const std::string& fmt) const
|
||||
|
||||
for (int i = 0; i < 12; ++i)
|
||||
{
|
||||
s += format(fmt, (unsigned int) _id[i]);
|
||||
s += Poco::format(fmt, (unsigned int) _id[i]);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
@ -43,9 +43,9 @@ namespace Poco {
|
||||
namespace MongoDB {
|
||||
|
||||
|
||||
static const std::string keyCursor {"cursor"};
|
||||
static const std::string keyFirstBatch {"firstBatch"};
|
||||
static const std::string keyNextBatch {"nextBatch"};
|
||||
[[ maybe_unused ]] static const std::string keyCursor {"cursor"};
|
||||
[[ maybe_unused ]] static const std::string keyFirstBatch {"firstBatch"};
|
||||
[[ maybe_unused ]] static const std::string keyNextBatch {"nextBatch"};
|
||||
|
||||
static Poco::Int64 cursorIdFromResponse(const MongoDB::Document& doc);
|
||||
|
||||
@ -131,7 +131,7 @@ OpMsgMessage& OpMsgCursor::next(Connection& connection)
|
||||
connection.readResponse(_response);
|
||||
}
|
||||
else
|
||||
#endif
|
||||
#endif
|
||||
{
|
||||
_response.clear();
|
||||
_query.setCursor(_cursorID, _batchSize);
|
||||
|
||||
@ -17,9 +17,9 @@
|
||||
#include "Poco/NumberFormatter.h"
|
||||
#include "Poco/NumberParser.h"
|
||||
#include "Poco/String.h"
|
||||
#include <charconv>
|
||||
#include <format>
|
||||
|
||||
|
||||
using Poco::NumberFormatter;
|
||||
using Poco::NumberParser;
|
||||
using Poco::icompare;
|
||||
@ -75,7 +75,7 @@ void HTTPMessage::setContentLength(std::streamsize length)
|
||||
erase(CONTENT_LENGTH);
|
||||
}
|
||||
|
||||
|
||||
|
||||
std::streamsize HTTPMessage::getContentLength() const
|
||||
{
|
||||
const std::string& contentLength = get(CONTENT_LENGTH, EMPTY);
|
||||
@ -98,7 +98,7 @@ void HTTPMessage::setContentLength64(Poco::Int64 length)
|
||||
erase(CONTENT_LENGTH);
|
||||
}
|
||||
|
||||
|
||||
|
||||
Poco::Int64 HTTPMessage::getContentLength64() const
|
||||
{
|
||||
const std::string& contentLength = get(CONTENT_LENGTH, EMPTY);
|
||||
@ -133,13 +133,13 @@ void HTTPMessage::setChunkedTransferEncoding(bool flag)
|
||||
setTransferEncoding(IDENTITY_TRANSFER_ENCODING);
|
||||
}
|
||||
|
||||
|
||||
|
||||
bool HTTPMessage::getChunkedTransferEncoding() const
|
||||
{
|
||||
return icompare(getTransferEncoding(), CHUNKED_TRANSFER_ENCODING) == 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
void HTTPMessage::setContentType(const std::string& mediaType)
|
||||
{
|
||||
if (mediaType.empty())
|
||||
@ -154,7 +154,7 @@ void HTTPMessage::setContentType(const MediaType& mediaType)
|
||||
setContentType(mediaType.toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
const std::string& HTTPMessage::getContentType() const
|
||||
{
|
||||
return get(CONTENT_TYPE, UNKNOWN_CONTENT_TYPE);
|
||||
|
||||
2
contrib/avro
vendored
2
contrib/avro
vendored
@ -1 +1 @@
|
||||
Subproject commit d43acc84d3d455b016f847d6666fbc3cd27f16a9
|
||||
Subproject commit 545e7002683cbc2198164d93088ac8e4955b4628
|
||||
2
contrib/grpc
vendored
2
contrib/grpc
vendored
@ -1 +1 @@
|
||||
Subproject commit 77b2737a709d43d8c6895e3f03ca62b00bd9201c
|
||||
Subproject commit f5b7fdc2dff09ada06dbf6c75df298fb40f898df
|
||||
@ -284,6 +284,11 @@ function run_tests
|
||||
NPROC=1
|
||||
fi
|
||||
|
||||
export CLICKHOUSE_CONFIG_DIR=$FASTTEST_DATA
|
||||
export CLICKHOUSE_CONFIG="$FASTTEST_DATA/config.xml"
|
||||
export CLICKHOUSE_USER_FILES="$FASTTEST_DATA/user_files"
|
||||
export CLICKHOUSE_SCHEMA_FILES="$FASTTEST_DATA/format_schemas"
|
||||
|
||||
local test_opts=(
|
||||
--hung-check
|
||||
--fast-tests-only
|
||||
|
||||
@ -9,6 +9,16 @@ set -e -x -a
|
||||
MAX_RUN_TIME=${MAX_RUN_TIME:-10800}
|
||||
MAX_RUN_TIME=$((MAX_RUN_TIME == 0 ? 10800 : MAX_RUN_TIME))
|
||||
|
||||
USE_DATABASE_REPLICATED=${USE_DATABASE_REPLICATED:=0}
|
||||
USE_SHARED_CATALOG=${USE_SHARED_CATALOG:=0}
|
||||
|
||||
# disable for now
|
||||
RUN_SEQUENTIAL_TESTS_IN_PARALLEL=0
|
||||
|
||||
if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] || [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
RUN_SEQUENTIAL_TESTS_IN_PARALLEL=0
|
||||
fi
|
||||
|
||||
# Choose random timezone for this test run.
|
||||
#
|
||||
# NOTE: that clickhouse-test will randomize session_timezone by itself as well
|
||||
@ -89,10 +99,57 @@ if [ "$NUM_TRIES" -gt "1" ]; then
|
||||
mkdir -p /var/run/clickhouse-server
|
||||
fi
|
||||
|
||||
# Run a CH instance to execute sequential tests on it in parallel with all other tests.
|
||||
if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
|
||||
mkdir -p /var/run/clickhouse-server3 /etc/clickhouse-server3 /var/lib/clickhouse3
|
||||
cp -r -L /etc/clickhouse-server/* /etc/clickhouse-server3/
|
||||
|
||||
sudo chown clickhouse:clickhouse /var/run/clickhouse-server3 /var/lib/clickhouse3 /etc/clickhouse-server3/
|
||||
sudo chown -R clickhouse:clickhouse /etc/clickhouse-server3/*
|
||||
|
||||
function replace(){
|
||||
sudo find /etc/clickhouse-server3/ -type f -name '*.xml' -exec sed -i "$1" {} \;
|
||||
}
|
||||
|
||||
replace "s|<port>9000</port>|<port>19000</port>|g"
|
||||
replace "s|<port>9440</port>|<port>19440</port>|g"
|
||||
replace "s|<port>9988</port>|<port>19988</port>|g"
|
||||
replace "s|<port>9234</port>|<port>19234</port>|g"
|
||||
replace "s|<port>9181</port>|<port>19181</port>|g"
|
||||
replace "s|<https_port>8443</https_port>|<https_port>18443</https_port>|g"
|
||||
replace "s|<tcp_port>9000</tcp_port>|<tcp_port>19000</tcp_port>|g"
|
||||
replace "s|<tcp_port>9181</tcp_port>|<tcp_port>19181</tcp_port>|g"
|
||||
replace "s|<tcp_port_secure>9440</tcp_port_secure>|<tcp_port_secure>19440</tcp_port_secure>|g"
|
||||
replace "s|<tcp_with_proxy_port>9010</tcp_with_proxy_port>|<tcp_with_proxy_port>19010</tcp_with_proxy_port>|g"
|
||||
replace "s|<mysql_port>9004</mysql_port>|<mysql_port>19004</mysql_port>|g"
|
||||
replace "s|<postgresql_port>9005</postgresql_port>|<postgresql_port>19005</postgresql_port>|g"
|
||||
replace "s|<interserver_http_port>9009</interserver_http_port>|<interserver_http_port>19009</interserver_http_port>|g"
|
||||
replace "s|8123|18123|g"
|
||||
replace "s|/var/lib/clickhouse/|/var/lib/clickhouse3/|g"
|
||||
replace "s|/etc/clickhouse-server/|/etc/clickhouse-server3/|g"
|
||||
# distributed cache
|
||||
replace "s|<tcp_port>10001</tcp_port>|<tcp_port>10003</tcp_port>|g"
|
||||
replace "s|<tcp_port>10002</tcp_port>|<tcp_port>10004</tcp_port>|g"
|
||||
|
||||
sudo -E -u clickhouse /usr/bin/clickhouse server --daemon --config /etc/clickhouse-server3/config.xml \
|
||||
--pid-file /var/run/clickhouse-server3/clickhouse-server.pid \
|
||||
-- --path /var/lib/clickhouse3/ --logger.stderr /var/log/clickhouse-server/stderr3.log \
|
||||
--logger.log /var/log/clickhouse-server/clickhouse-server3.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server3.err.log \
|
||||
--tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
|
||||
--prometheus.port 19988 --keeper_server.raft_configuration.server.port 19234 --keeper_server.tcp_port 19181 \
|
||||
--mysql_port 19004 --postgresql_port 19005
|
||||
|
||||
for _ in {1..100}
|
||||
do
|
||||
clickhouse-client --port 19000 --query "SELECT 1" && break
|
||||
sleep 1
|
||||
done
|
||||
fi
|
||||
|
||||
# simplest way to forward env variables to server
|
||||
sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml --daemon --pid-file /var/run/clickhouse-server/clickhouse-server.pid
|
||||
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
sudo sed -i "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_1/</filesystem_caches_path>|" /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
|
||||
|
||||
sudo sed -i "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_2/</filesystem_caches_path>|" /etc/clickhouse-server2/config.d/filesystem_caches_path.xml
|
||||
@ -129,7 +186,7 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
|
||||
MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000)) # set to 2.5 hours if 0 (unlimited)
|
||||
fi
|
||||
|
||||
if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
sudo cat /etc/clickhouse-server1/config.d/filesystem_caches_path.xml \
|
||||
| sed "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_1/</filesystem_caches_path>|" \
|
||||
> /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp
|
||||
@ -209,15 +266,15 @@ function run_tests()
|
||||
ADDITIONAL_OPTIONS+=('--no-random-merge-tree-settings')
|
||||
fi
|
||||
|
||||
if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
ADDITIONAL_OPTIONS+=('--shared-catalog')
|
||||
fi
|
||||
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
ADDITIONAL_OPTIONS+=('--replicated-database')
|
||||
# Too many tests fail for DatabaseReplicated in parallel.
|
||||
ADDITIONAL_OPTIONS+=('--jobs')
|
||||
ADDITIONAL_OPTIONS+=('2')
|
||||
ADDITIONAL_OPTIONS+=('3')
|
||||
elif [[ 1 == $(clickhouse-client --query "SELECT value LIKE '%SANITIZE_COVERAGE%' FROM system.build_options WHERE name = 'CXX_FLAGS'") ]]; then
|
||||
# Coverage on a per-test basis could only be collected sequentially.
|
||||
# Do not set the --jobs parameter.
|
||||
@ -225,7 +282,11 @@ function run_tests()
|
||||
else
|
||||
# All other configurations are OK.
|
||||
ADDITIONAL_OPTIONS+=('--jobs')
|
||||
ADDITIONAL_OPTIONS+=('8')
|
||||
ADDITIONAL_OPTIONS+=('5')
|
||||
fi
|
||||
|
||||
if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
|
||||
ADDITIONAL_OPTIONS+=('--run-sequential-tests-in-parallel')
|
||||
fi
|
||||
|
||||
if [[ -n "$RUN_BY_HASH_NUM" ]] && [[ -n "$RUN_BY_HASH_TOTAL" ]]; then
|
||||
@ -289,7 +350,7 @@ do
|
||||
err=$(clickhouse-client -q "select * from system.$table into outfile '/test_output/$table.tsv.gz' format TSVWithNamesAndTypes")
|
||||
echo "$err"
|
||||
[[ "0" != "${#err}" ]] && failed_to_save_logs=1
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 )
|
||||
echo "$err"
|
||||
[[ "0" != "${#err}" ]] && failed_to_save_logs=1
|
||||
@ -298,7 +359,7 @@ do
|
||||
[[ "0" != "${#err}" ]] && failed_to_save_logs=1
|
||||
fi
|
||||
|
||||
if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 )
|
||||
echo "$err"
|
||||
[[ "0" != "${#err}" ]] && failed_to_save_logs=1
|
||||
@ -309,12 +370,17 @@ done
|
||||
# Why do we read data with clickhouse-local?
|
||||
# Because it's the simplest way to read it when server has crashed.
|
||||
sudo clickhouse stop ||:
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
|
||||
if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
|
||||
sudo clickhouse stop --pid-path /var/run/clickhouse-server3 ||:
|
||||
fi
|
||||
|
||||
if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||:
|
||||
sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||:
|
||||
fi
|
||||
|
||||
if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||:
|
||||
fi
|
||||
|
||||
@ -322,6 +388,12 @@ rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server.log ||:
|
||||
rg -A50 -Fa "============" /var/log/clickhouse-server/stderr.log ||:
|
||||
zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst &
|
||||
|
||||
if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
|
||||
rg -Fa "<Fatal>" /var/log/clickhouse-server3/clickhouse-server.log ||:
|
||||
rg -A50 -Fa "============" /var/log/clickhouse-server3/stderr.log ||:
|
||||
zstd --threads=0 < /var/log/clickhouse-server3/clickhouse-server.log > /test_output/clickhouse-server3.log.zst &
|
||||
fi
|
||||
|
||||
data_path_config="--path=/var/lib/clickhouse/"
|
||||
if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
|
||||
# We need s3 storage configuration (but it's more likely that clickhouse-local will fail for some reason)
|
||||
@ -341,12 +413,17 @@ if [ $failed_to_save_logs -ne 0 ]; then
|
||||
for table in query_log zookeeper_log trace_log transactions_info_log metric_log blob_storage_log error_log
|
||||
do
|
||||
clickhouse-local "$data_path_config" --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst ||:
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
|
||||
if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
|
||||
clickhouse-local --path /var/lib/clickhouse3/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.3.tsv.zst ||:
|
||||
fi
|
||||
|
||||
if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst ||:
|
||||
clickhouse-local --path /var/lib/clickhouse2/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst ||:
|
||||
fi
|
||||
|
||||
if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst ||:
|
||||
fi
|
||||
done
|
||||
@ -382,7 +459,14 @@ rm -rf /var/lib/clickhouse/data/system/*/
|
||||
tar -chf /test_output/store.tar /var/lib/clickhouse/store ||:
|
||||
tar -chf /test_output/metadata.tar /var/lib/clickhouse/metadata/*.sql ||:
|
||||
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
|
||||
rm -rf /var/lib/clickhouse3/data/system/*/
|
||||
tar -chf /test_output/store.tar /var/lib/clickhouse3/store ||:
|
||||
tar -chf /test_output/metadata.tar /var/lib/clickhouse3/metadata/*.sql ||:
|
||||
fi
|
||||
|
||||
|
||||
if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server1.log ||:
|
||||
rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server2.log ||:
|
||||
zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.zst ||:
|
||||
@ -393,7 +477,7 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
|
||||
tar -chf /test_output/coordination2.tar /var/lib/clickhouse2/coordination ||:
|
||||
fi
|
||||
|
||||
if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server1.log ||:
|
||||
zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.zst ||:
|
||||
mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:
|
||||
|
||||
@ -2,15 +2,11 @@
|
||||
slug: /en/operations/opentelemetry
|
||||
sidebar_position: 62
|
||||
sidebar_label: Tracing ClickHouse with OpenTelemetry
|
||||
title: "[experimental] Tracing ClickHouse with OpenTelemetry"
|
||||
title: "Tracing ClickHouse with OpenTelemetry"
|
||||
---
|
||||
|
||||
[OpenTelemetry](https://opentelemetry.io/) is an open standard for collecting traces and metrics from the distributed application. ClickHouse has some support for OpenTelemetry.
|
||||
|
||||
:::note
|
||||
This is an experimental feature that will change in backwards-incompatible ways in future releases.
|
||||
:::
|
||||
|
||||
## Supplying Trace Context to ClickHouse
|
||||
|
||||
ClickHouse accepts trace context HTTP headers, as described by the [W3C recommendation](https://www.w3.org/TR/trace-context/). It also accepts trace context over a native protocol that is used for communication between ClickHouse servers or between the client and server. For manual testing, trace context headers conforming to the Trace Context recommendation can be supplied to `clickhouse-client` using `--opentelemetry-traceparent` and `--opentelemetry-tracestate` flags.
|
||||
|
||||
@ -7,7 +7,7 @@ sidebar_label: Tuples
|
||||
## tuple
|
||||
|
||||
A function that allows grouping multiple columns.
|
||||
For columns with the types T1, T2, ..., it returns a Tuple(T1, T2, ...) type tuple containing these columns. There is no cost to execute the function.
|
||||
For columns C1, C2, ... with the types T1, T2, ..., it returns a named Tuple(C1 T1, C2 T2, ...) type tuple containing these columns if their names are unique and can be treated as unquoted identifiers, otherwise a Tuple(T1, T2, ...) is returned. There is no cost to execute the function.
|
||||
Tuples are normally used as intermediate values for an argument of IN operators, or for creating a list of formal parameters of lambda functions. Tuples can’t be written to a table.
|
||||
|
||||
The function implements the operator `(x, y, ...)`.
|
||||
@ -259,6 +259,60 @@ Result:
|
||||
└───────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## tupleNames
|
||||
|
||||
Converts a tuple into an array of column names. For a tuple in the form `Tuple(a T, b T, ...)`, it returns an array of strings representing the named columns of the tuple. If the tuple elements do not have explicit names, their indices will be used as the column names instead.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
tupleNames(tuple)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — Named tuple. [Tuple](../../sql-reference/data-types/tuple.md) with any types of values.
|
||||
|
||||
**Returned value**
|
||||
|
||||
- An array with strings.
|
||||
|
||||
Type: [Array](../../sql-reference/data-types/array.md)([Tuple](../../sql-reference/data-types/tuple.md)([String](../../sql-reference/data-types/string.md), ...)).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE tupletest (col Tuple(user_ID UInt64, session_ID UInt64)) ENGINE = Memory;
|
||||
|
||||
INSERT INTO tupletest VALUES (tuple(1, 2));
|
||||
|
||||
SELECT tupleNames(col) FROM tupletest;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─tupleNames(col)──────────┐
|
||||
│ ['user_ID','session_ID'] │
|
||||
└──────────────────────────┘
|
||||
```
|
||||
|
||||
If you pass a simple tuple to the function, ClickHouse uses the indexes of the columns as their names:
|
||||
|
||||
``` sql
|
||||
SELECT tupleNames(tuple(3, 2, 1));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─tupleNames((3, 2, 1))─┐
|
||||
│ ['1','2','3'] │
|
||||
└───────────────────────┘
|
||||
```
|
||||
|
||||
## tuplePlus
|
||||
|
||||
Calculates the sum of corresponding values of two tuples of the same size.
|
||||
|
||||
@ -130,7 +130,9 @@ SELECT * FROM file('user_files/archives/archive{1..2}.zip :: table.csv');
|
||||
|
||||
## Globs in path
|
||||
|
||||
Paths may use globbing. Files must match the whole path pattern, not only the suffix or prefix.
|
||||
Paths may use globbing. Files must match the whole path pattern, not only the suffix or prefix. There is one exception that if the path refers to an existing
|
||||
directory and does not use globs, a `*` will be implicitly added to the path so
|
||||
all the files in the directory are selected.
|
||||
|
||||
- `*` — Represents arbitrarily many characters except `/` but including the empty string.
|
||||
- `?` — Represents an arbitrary single character.
|
||||
@ -163,6 +165,12 @@ An alternative path expression which achieves the same:
|
||||
SELECT count(*) FROM file('{some,another}_dir/*', 'TSV', 'name String, value UInt32');
|
||||
```
|
||||
|
||||
Query the total number of rows in `some_dir` using the implicit `*`:
|
||||
|
||||
```sql
|
||||
SELECT count(*) FROM file('some_dir', 'TSV', 'name String, value UInt32');
|
||||
```
|
||||
|
||||
:::note
|
||||
If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`.
|
||||
:::
|
||||
|
||||
@ -1117,6 +1117,7 @@ void Client::processOptions(const OptionsDescription & options_description,
|
||||
if (!options["user"].defaulted())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "User and JWT flags can't be specified together");
|
||||
config().setString("jwt", options["jwt"].as<std::string>());
|
||||
config().setString("user", "");
|
||||
}
|
||||
if (options.count("accept-invalid-certificate"))
|
||||
{
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
#include <Analyzer/Utils.h>
|
||||
|
||||
#include <DataTypes/DataTypeLowCardinality.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -26,12 +27,103 @@ static constexpr std::array boolean_functions{
|
||||
"like"sv, "notLike"sv, "ilike"sv, "notILike"sv, "empty"sv, "notEmpty"sv, "not"sv, "and"sv,
|
||||
"or"sv};
|
||||
|
||||
static bool isBooleanFunction(const String & func_name)
|
||||
|
||||
bool isBooleanFunction(const String & func_name)
|
||||
{
|
||||
return std::any_of(
|
||||
boolean_functions.begin(), boolean_functions.end(), [&](const auto boolean_func) { return func_name == boolean_func; });
|
||||
}
|
||||
|
||||
bool isNodeFunction(const QueryTreeNodePtr & node, const String & func_name)
|
||||
{
|
||||
if (const auto * function_node = node->as<FunctionNode>())
|
||||
return function_node->getFunctionName() == func_name;
|
||||
return false;
|
||||
}
|
||||
|
||||
QueryTreeNodePtr getFunctionArgument(const QueryTreeNodePtr & node, size_t idx)
|
||||
{
|
||||
if (const auto * function_node = node->as<FunctionNode>())
|
||||
{
|
||||
const auto & args = function_node->getArguments().getNodes();
|
||||
if (idx < args.size())
|
||||
return args[idx];
|
||||
}
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected '{}' to be a function with at least {} arguments", node->formatASTForErrorMessage(), idx + 1);
|
||||
}
|
||||
|
||||
QueryTreeNodePtr findEqualsFunction(const QueryTreeNodes & nodes)
|
||||
{
|
||||
for (const auto & node : nodes)
|
||||
{
|
||||
const auto * function_node = node->as<FunctionNode>();
|
||||
if (function_node && function_node->getFunctionName() == "equals" &&
|
||||
function_node->getArguments().getNodes().size() == 2)
|
||||
{
|
||||
return node;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
bool isBooleanConstant(const QueryTreeNodePtr & node, bool expected_value)
|
||||
{
|
||||
const auto * constant_node = node->as<ConstantNode>();
|
||||
if (!constant_node || !constant_node->getResultType()->equals(DataTypeUInt8()))
|
||||
return false;
|
||||
|
||||
UInt64 constant_value;
|
||||
return (constant_node->getValue().tryGet<UInt64>(constant_value) && constant_value == expected_value);
|
||||
}
|
||||
|
||||
/// Returns true if expression consists of only conjunctions of functions with the specified name or true constants
|
||||
bool isOnlyConjunctionOfFunctions(
|
||||
const QueryTreeNodePtr & node,
|
||||
const String & func_name,
|
||||
const QueryTreeNodePtrWithHashSet & allowed_arguments)
|
||||
{
|
||||
if (isBooleanConstant(node, true))
|
||||
return true;
|
||||
|
||||
const auto * node_function = node->as<FunctionNode>();
|
||||
if (!node_function)
|
||||
return false;
|
||||
|
||||
if (node_function->getFunctionName() == func_name
|
||||
&& allowed_arguments.contains(node_function->getArgumentsNode()))
|
||||
return true;
|
||||
|
||||
if (node_function->getFunctionName() == "and")
|
||||
{
|
||||
for (const auto & and_argument : node_function->getArguments().getNodes())
|
||||
{
|
||||
if (!isOnlyConjunctionOfFunctions(and_argument, func_name, allowed_arguments))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// We can rewrite to a <=> b only if we are joining on a and b,
|
||||
/// because the function is not yet implemented for other cases.
|
||||
bool isTwoArgumentsFromDifferentSides(const FunctionNode & node_function, const JoinNode & join_node)
|
||||
{
|
||||
const auto & argument_nodes = node_function.getArguments().getNodes();
|
||||
if (argument_nodes.size() != 2)
|
||||
return false;
|
||||
|
||||
auto first_src = getExpressionSource(argument_nodes[0]);
|
||||
auto second_src = getExpressionSource(argument_nodes[1]);
|
||||
if (!first_src || !second_src)
|
||||
return false;
|
||||
|
||||
const auto & lhs_join = *join_node.getLeftTableExpression();
|
||||
const auto & rhs_join = *join_node.getRightTableExpression();
|
||||
return (first_src->isEqual(lhs_join) && second_src->isEqual(rhs_join)) ||
|
||||
(first_src->isEqual(rhs_join) && second_src->isEqual(lhs_join));
|
||||
}
|
||||
|
||||
/// Visitor that optimizes logical expressions _only_ in JOIN ON section
|
||||
class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithContext<JoinOnLogicalExpressionOptimizerVisitor>
|
||||
{
|
||||
@ -47,15 +139,16 @@ public:
|
||||
{
|
||||
auto * function_node = node->as<FunctionNode>();
|
||||
|
||||
if (!function_node)
|
||||
return;
|
||||
QueryTreeNodePtr new_node = nullptr;
|
||||
if (function_node && function_node->getFunctionName() == "or")
|
||||
new_node = tryOptimizeJoinOnNulls(function_node->getArguments().getNodes(), getContext());
|
||||
else
|
||||
new_node = tryOptimizeJoinOnNulls({node}, getContext());
|
||||
|
||||
if (function_node->getFunctionName() == "or")
|
||||
if (new_node)
|
||||
{
|
||||
bool is_argument_type_changed = tryOptimizeIsNotDistinctOrIsNull(node, getContext());
|
||||
if (is_argument_type_changed)
|
||||
need_rerun_resolve = true;
|
||||
return;
|
||||
need_rerun_resolve |= !new_node->getResultType()->equals(*node->getResultType());
|
||||
node = new_node;
|
||||
}
|
||||
}
|
||||
|
||||
@ -72,15 +165,11 @@ private:
|
||||
const JoinNode * join_node;
|
||||
bool need_rerun_resolve = false;
|
||||
|
||||
/// Returns true if type of some operand is changed and parent function needs to be re-resolved
|
||||
bool tryOptimizeIsNotDistinctOrIsNull(QueryTreeNodePtr & node, const ContextPtr & context)
|
||||
/// Returns optimized node or nullptr if nothing have been changed
|
||||
QueryTreeNodePtr tryOptimizeJoinOnNulls(const QueryTreeNodes & nodes, const ContextPtr & context)
|
||||
{
|
||||
auto & function_node = node->as<FunctionNode &>();
|
||||
chassert(function_node.getFunctionName() == "or");
|
||||
|
||||
|
||||
QueryTreeNodes or_operands;
|
||||
or_operands.reserve(function_node.getArguments().getNodes().size());
|
||||
or_operands.reserve(nodes.size());
|
||||
|
||||
/// Indices of `equals` or `isNotDistinctFrom` functions in the vector above
|
||||
std::vector<size_t> equals_functions_indices;
|
||||
@ -93,47 +182,73 @@ private:
|
||||
* b => [(a IS NULL AND b IS NULL)]
|
||||
* c => [(a IS NULL AND c IS NULL)]
|
||||
* }
|
||||
* Then for each a <=> b we can find all operands that contains both a IS NULL and b IS NULL
|
||||
* Then for each equality a = b we can check if we have operand (a IS NULL AND b IS NULL)
|
||||
*/
|
||||
QueryTreeNodePtrWithHashMap<std::vector<size_t>> is_null_argument_to_indices;
|
||||
|
||||
for (const auto & argument : function_node.getArguments())
|
||||
{
|
||||
or_operands.push_back(argument);
|
||||
bool is_anything_changed = false;
|
||||
|
||||
auto * argument_function = argument->as<FunctionNode>();
|
||||
for (const auto & node : nodes)
|
||||
{
|
||||
if (isBooleanConstant(node, false))
|
||||
{
|
||||
/// Remove false constants from OR
|
||||
is_anything_changed = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
or_operands.push_back(node);
|
||||
auto * argument_function = node->as<FunctionNode>();
|
||||
if (!argument_function)
|
||||
continue;
|
||||
|
||||
const auto & func_name = argument_function->getFunctionName();
|
||||
if (func_name == "equals" || func_name == "isNotDistinctFrom")
|
||||
{
|
||||
const auto & argument_nodes = argument_function->getArguments().getNodes();
|
||||
if (argument_nodes.size() != 2)
|
||||
continue;
|
||||
/// We can rewrite to a <=> b only if we are joining on a and b,
|
||||
/// because the function is not yet implemented for other cases.
|
||||
auto first_src = getExpressionSource(argument_nodes[0]);
|
||||
auto second_src = getExpressionSource(argument_nodes[1]);
|
||||
if (!first_src || !second_src)
|
||||
continue;
|
||||
const auto & lhs_join = *join_node->getLeftTableExpression();
|
||||
const auto & rhs_join = *join_node->getRightTableExpression();
|
||||
bool arguments_from_both_sides = (first_src->isEqual(lhs_join) && second_src->isEqual(rhs_join)) ||
|
||||
(first_src->isEqual(rhs_join) && second_src->isEqual(lhs_join));
|
||||
if (!arguments_from_both_sides)
|
||||
continue;
|
||||
equals_functions_indices.push_back(or_operands.size() - 1);
|
||||
if (isTwoArgumentsFromDifferentSides(*argument_function, *join_node))
|
||||
equals_functions_indices.push_back(or_operands.size() - 1);
|
||||
}
|
||||
else if (func_name == "and")
|
||||
{
|
||||
for (const auto & and_argument : argument_function->getArguments().getNodes())
|
||||
const auto & and_arguments = argument_function->getArguments().getNodes();
|
||||
bool all_are_is_null = and_arguments.size() == 2 && isNodeFunction(and_arguments[0], "isNull") && isNodeFunction(and_arguments[1], "isNull");
|
||||
if (all_are_is_null)
|
||||
{
|
||||
auto * and_argument_function = and_argument->as<FunctionNode>();
|
||||
if (and_argument_function && and_argument_function->getFunctionName() == "isNull")
|
||||
is_null_argument_to_indices[getFunctionArgument(and_arguments.front(), 0)].push_back(or_operands.size() - 1);
|
||||
is_null_argument_to_indices[getFunctionArgument(and_arguments.back(), 0)].push_back(or_operands.size() - 1);
|
||||
}
|
||||
|
||||
/// Expression `a = b AND (a IS NOT NULL) AND true AND (b IS NOT NULL)` we can be replaced with `a = b`
|
||||
/// Even though this expression are not equivalent (first is NULL on NULLs, while second is FALSE),
|
||||
/// it is still correct since for JOIN ON condition NULL is treated as FALSE
|
||||
if (const auto & equals_function = findEqualsFunction(and_arguments))
|
||||
{
|
||||
const auto & equals_arguments = equals_function->as<FunctionNode>()->getArguments().getNodes();
|
||||
/// Expected isNotNull arguments
|
||||
QueryTreeNodePtrWithHashSet allowed_arguments;
|
||||
allowed_arguments.insert(QueryTreeNodePtrWithHash(std::make_shared<ListNode>(QueryTreeNodes{equals_arguments[0]})));
|
||||
allowed_arguments.insert(QueryTreeNodePtrWithHash(std::make_shared<ListNode>(QueryTreeNodes{equals_arguments[1]})));
|
||||
|
||||
bool can_be_optimized = true;
|
||||
for (const auto & and_argument : and_arguments)
|
||||
{
|
||||
const auto & is_null_argument = and_argument_function->getArguments().getNodes()[0];
|
||||
is_null_argument_to_indices[is_null_argument].push_back(or_operands.size() - 1);
|
||||
if (and_argument.get() == equals_function.get())
|
||||
continue;
|
||||
|
||||
if (isOnlyConjunctionOfFunctions(and_argument, "isNotNull", allowed_arguments))
|
||||
continue;
|
||||
|
||||
can_be_optimized = false;
|
||||
break;
|
||||
}
|
||||
|
||||
if (can_be_optimized)
|
||||
{
|
||||
is_anything_changed = true;
|
||||
or_operands.pop_back();
|
||||
or_operands.push_back(equals_function);
|
||||
if (isTwoArgumentsFromDifferentSides(equals_function->as<FunctionNode &>(), *join_node))
|
||||
equals_functions_indices.push_back(or_operands.size() - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -144,9 +259,9 @@ private:
|
||||
|
||||
for (size_t equals_function_idx : equals_functions_indices)
|
||||
{
|
||||
auto * equals_function = or_operands[equals_function_idx]->as<FunctionNode>();
|
||||
const auto * equals_function = or_operands[equals_function_idx]->as<FunctionNode>();
|
||||
|
||||
/// For a <=> b we are looking for expressions containing both `a IS NULL` and `b IS NULL` combined with AND
|
||||
/// For a = b we are looking for all expressions `a IS NULL AND b IS NULL`
|
||||
const auto & argument_nodes = equals_function->getArguments().getNodes();
|
||||
const auto & lhs_is_null_parents = is_null_argument_to_indices[argument_nodes[0]];
|
||||
const auto & rhs_is_null_parents = is_null_argument_to_indices[argument_nodes[1]];
|
||||
@ -161,60 +276,40 @@ private:
|
||||
|
||||
for (size_t to_optimize_idx : operands_to_optimize)
|
||||
{
|
||||
/// We are looking for operand `a IS NULL AND b IS NULL AND ...`
|
||||
auto * operand_to_optimize = or_operands[to_optimize_idx]->as<FunctionNode>();
|
||||
|
||||
/// Remove `a IS NULL` and `b IS NULL` arguments from AND
|
||||
QueryTreeNodes new_arguments;
|
||||
for (const auto & and_argument : operand_to_optimize->getArguments().getNodes())
|
||||
{
|
||||
bool to_eliminate = false;
|
||||
|
||||
const auto * and_argument_function = and_argument->as<FunctionNode>();
|
||||
if (and_argument_function && and_argument_function->getFunctionName() == "isNull")
|
||||
{
|
||||
const auto & is_null_argument = and_argument_function->getArguments().getNodes()[0];
|
||||
to_eliminate = (is_null_argument->isEqual(*argument_nodes[0]) || is_null_argument->isEqual(*argument_nodes[1]));
|
||||
}
|
||||
|
||||
if (to_eliminate)
|
||||
arguments_to_reresolve.insert(to_optimize_idx);
|
||||
else
|
||||
new_arguments.emplace_back(and_argument);
|
||||
}
|
||||
/// If less than two arguments left, we will remove or replace the whole AND below
|
||||
operand_to_optimize->getArguments().getNodes() = std::move(new_arguments);
|
||||
/// Remove `a IS NULL AND b IS NULL`
|
||||
or_operands[to_optimize_idx] = nullptr;
|
||||
is_anything_changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (arguments_to_reresolve.empty())
|
||||
if (arguments_to_reresolve.empty() && !is_anything_changed)
|
||||
/// Nothing have been changed
|
||||
return false;
|
||||
return nullptr;
|
||||
|
||||
auto and_function_resolver = FunctionFactory::instance().get("and", context);
|
||||
auto strict_equals_function_resolver = FunctionFactory::instance().get("isNotDistinctFrom", context);
|
||||
|
||||
bool need_reresolve = false;
|
||||
QueryTreeNodes new_or_operands;
|
||||
for (size_t i = 0; i < or_operands.size(); ++i)
|
||||
{
|
||||
if (arguments_to_reresolve.contains(i))
|
||||
{
|
||||
auto * function = or_operands[i]->as<FunctionNode>();
|
||||
const auto * function = or_operands[i]->as<FunctionNode>();
|
||||
if (function->getFunctionName() == "equals")
|
||||
{
|
||||
/// We should replace `a = b` with `a <=> b` because we removed checks for IS NULL
|
||||
need_reresolve |= function->getResultType()->isNullable();
|
||||
function->resolveAsFunction(strict_equals_function_resolver);
|
||||
new_or_operands.emplace_back(std::move(or_operands[i]));
|
||||
auto new_function = or_operands[i]->clone();
|
||||
new_function->as<FunctionNode>()->resolveAsFunction(strict_equals_function_resolver);
|
||||
new_or_operands.emplace_back(std::move(new_function));
|
||||
}
|
||||
else if (function->getFunctionName() == "and")
|
||||
{
|
||||
const auto & and_arguments = function->getArguments().getNodes();
|
||||
if (and_arguments.size() > 1)
|
||||
{
|
||||
function->resolveAsFunction(and_function_resolver);
|
||||
new_or_operands.emplace_back(std::move(or_operands[i]));
|
||||
auto new_function = or_operands[i]->clone();
|
||||
new_function->as<FunctionNode>()->resolveAsFunction(and_function_resolver);
|
||||
new_or_operands.emplace_back(std::move(new_function));
|
||||
}
|
||||
else if (and_arguments.size() == 1)
|
||||
{
|
||||
@ -223,25 +318,26 @@ private:
|
||||
}
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected function name: '{}'", function->getFunctionName());
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected function '{}'", function->getFunctionName());
|
||||
}
|
||||
else
|
||||
else if (or_operands[i])
|
||||
{
|
||||
new_or_operands.emplace_back(std::move(or_operands[i]));
|
||||
}
|
||||
}
|
||||
|
||||
if (new_or_operands.empty())
|
||||
return nullptr;
|
||||
|
||||
if (new_or_operands.size() == 1)
|
||||
{
|
||||
node = std::move(new_or_operands[0]);
|
||||
return need_reresolve;
|
||||
}
|
||||
return new_or_operands[0];
|
||||
|
||||
/// Rebuild OR function
|
||||
auto or_function_resolver = FunctionFactory::instance().get("or", context);
|
||||
function_node.getArguments().getNodes() = std::move(new_or_operands);
|
||||
function_node.resolveAsFunction(or_function_resolver);
|
||||
return need_reresolve;
|
||||
auto function_node = std::make_shared<FunctionNode>("or");
|
||||
function_node->getArguments().getNodes() = std::move(new_or_operands);
|
||||
function_node->resolveAsFunction(or_function_resolver);
|
||||
return function_node;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -940,6 +940,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select
|
||||
table_join.locality,
|
||||
result_join_strictness,
|
||||
result_join_kind);
|
||||
join_node->setOriginalAST(table_element.table_join);
|
||||
|
||||
/** Original AST is not set because it will contain only join part and does
|
||||
* not include left table expression.
|
||||
|
||||
@ -3,6 +3,8 @@
|
||||
#include <Backups/BackupStatus.h>
|
||||
#include <Common/ProfileEvents.h>
|
||||
|
||||
#include <exception>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
#include <memory>
|
||||
|
||||
#include <base/types.h>
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <list>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <mutex>
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include <Common/ConcurrentBoundedQueue.h>
|
||||
|
||||
#include <map>
|
||||
#include <variant>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <future>
|
||||
|
||||
@ -2,9 +2,11 @@
|
||||
|
||||
#include "config.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <boost/noncopyable.hpp>
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -63,7 +63,7 @@ const char USER_INTERSERVER_MARKER[] = " INTERSERVER SECRET ";
|
||||
/// Marker for SSH-keys-based authentication (passed as the user name)
|
||||
const char SSH_KEY_AUTHENTICAION_MARKER[] = " SSH KEY AUTHENTICATION ";
|
||||
|
||||
/// Market for JSON Web Token authentication
|
||||
/// Marker for JSON Web Token authentication
|
||||
const char JWT_AUTHENTICAION_MARKER[] = " JWT AUTHENTICATION ";
|
||||
|
||||
};
|
||||
|
||||
@ -530,6 +530,7 @@ class IColumn;
|
||||
M(Bool, optimize_read_in_order, true, "Enable ORDER BY optimization for reading data in corresponding order in MergeTree tables.", 0) \
|
||||
M(Bool, optimize_read_in_window_order, true, "Enable ORDER BY optimization in window clause for reading data in corresponding order in MergeTree tables.", 0) \
|
||||
M(Bool, optimize_aggregation_in_order, false, "Enable GROUP BY optimization for aggregating data in corresponding order in MergeTree tables.", 0) \
|
||||
M(Bool, read_in_order_use_buffering, true, "Use buffering before merging while reading in order of primary key. It increases the parallelism of query execution", 0) \
|
||||
M(UInt64, aggregation_in_order_max_block_bytes, 50000000, "Maximal size of block in bytes accumulated during aggregation in order of primary key. Lower block size allows to parallelize more final merge stage of aggregation.", 0) \
|
||||
M(UInt64, read_in_order_two_level_merge_threshold, 100, "Minimal number of parts to read to run preliminary merge step during multithread reading in order of primary key.", 0) \
|
||||
M(Bool, low_cardinality_allow_in_native_format, true, "Use LowCardinality type in Native format. Otherwise, convert LowCardinality columns to ordinary for select query, and convert ordinary columns to required LowCardinality for insert query.", 0) \
|
||||
@ -718,6 +719,7 @@ class IColumn;
|
||||
M(Bool, optimize_group_by_function_keys, true, "Eliminates functions of other keys in GROUP BY section", 0) \
|
||||
M(Bool, optimize_group_by_constant_keys, true, "Optimize GROUP BY when all keys in block are constant", 0) \
|
||||
M(Bool, legacy_column_name_of_tuple_literal, false, "List all names of element of large tuple literals in their column names instead of hash. This settings exists only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher.", 0) \
|
||||
M(Bool, enable_named_columns_in_function_tuple, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers.", 0) \
|
||||
\
|
||||
M(Bool, query_plan_enable_optimizations, true, "Globally enable/disable query optimization at the query plan level", 0) \
|
||||
M(UInt64, query_plan_max_optimizations_to_apply, 10000, "Limit the total number of optimizations applied to query plan. If zero, ignored. If limit reached, throw exception", 0) \
|
||||
@ -1062,6 +1064,7 @@ class IColumn;
|
||||
M(Bool, input_format_orc_allow_missing_columns, true, "Allow missing columns while reading ORC input formats", 0) \
|
||||
M(Bool, input_format_orc_use_fast_decoder, true, "Use a faster ORC decoder implementation.", 0) \
|
||||
M(Bool, input_format_orc_filter_push_down, true, "When reading ORC files, skip whole stripes or row groups based on the WHERE/PREWHERE expressions, min/max statistics or bloom filter in the ORC metadata.", 0) \
|
||||
M(Bool, input_format_orc_read_use_writer_time_zone, false, "Whether use the writer's time zone in ORC stripe for ORC row reader, the default ORC row reader's time zone is GMT.", 0) \
|
||||
M(Bool, input_format_parquet_allow_missing_columns, true, "Allow missing columns while reading Parquet input formats", 0) \
|
||||
M(UInt64, input_format_parquet_local_file_min_bytes_for_seek, 8192, "Min bytes required for local read (file) to do seek, instead of read with ignore in Parquet input format", 0) \
|
||||
M(Bool, input_format_arrow_allow_missing_columns, true, "Allow missing columns while reading Arrow input formats", 0) \
|
||||
|
||||
@ -58,10 +58,13 @@ String ClickHouseVersion::toString() const
|
||||
static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory::SettingsChanges>> settings_changes_history_initializer =
|
||||
{
|
||||
{"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
|
||||
{"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
|
||||
{"optimize_functions_to_subcolumns", false, true, "Enable optimization by default"},
|
||||
{"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
|
||||
{"input_format_json_ignore_key_case", false, false, "Ignore json key case while read json field from string."},
|
||||
{"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
|
||||
{"input_format_try_infer_variants", false, false, "Try to infer Variant type in text formats when there is more than one possible type for column/array elements"},
|
||||
{"input_format_orc_read_use_writer_time_zone", false, false, "Whether use the writer's time zone in ORC stripe for ORC row reader, the default ORC row reader's time zone is GMT."},
|
||||
{"lightweight_mutation_projection_mode", "throw", "throw", "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete."},
|
||||
{"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"},
|
||||
{"query_plan_merge_filters", false, false, "Allow to merge filters in the query plan"},
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
#include <filesystem>
|
||||
#include <base/isSharedPtrUnique.h>
|
||||
#include <Databases/DatabaseAtomic.h>
|
||||
#include <Databases/DatabaseFactory.h>
|
||||
#include <Databases/DatabaseOnDisk.h>
|
||||
@ -12,7 +13,7 @@
|
||||
#include <Interpreters/ExternalDictionariesLoader.h>
|
||||
#include <Parsers/formatAST.h>
|
||||
#include <Storages/StorageMaterializedView.h>
|
||||
#include "Common/logger_useful.h"
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/PoolId.h>
|
||||
#include <Common/atomicRename.h>
|
||||
#include <Common/filesystemHelpers.h>
|
||||
@ -397,7 +398,7 @@ DatabaseAtomic::DetachedTables DatabaseAtomic::cleanupDetachedTables()
|
||||
LOG_DEBUG(log, "There are {} detached tables. Start searching non used tables.", detached_tables.size());
|
||||
while (it != detached_tables.end())
|
||||
{
|
||||
if (it->second.unique())
|
||||
if (isSharedPtrUnique(it->second))
|
||||
{
|
||||
not_in_use.emplace(it->first, it->second);
|
||||
it = detached_tables.erase(it);
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#include <Databases/DatabaseLazy.h>
|
||||
|
||||
#include <base/sort.h>
|
||||
#include <base/isSharedPtrUnique.h>
|
||||
#include <iomanip>
|
||||
#include <filesystem>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
@ -305,7 +306,7 @@ try
|
||||
String table_name = expired_tables.front().table_name;
|
||||
auto it = tables_cache.find(table_name);
|
||||
|
||||
if (!it->second.table || it->second.table.unique())
|
||||
if (!it->second.table || isSharedPtrUnique(it->second.table))
|
||||
{
|
||||
LOG_DEBUG(log, "Drop table {} from cache.", backQuote(it->first));
|
||||
it->second.table.reset();
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
#if USE_MYSQL
|
||||
# include <string>
|
||||
# include <base/isSharedPtrUnique.h>
|
||||
# include <Databases/DatabaseFactory.h>
|
||||
# include <DataTypes/DataTypeDateTime.h>
|
||||
# include <DataTypes/DataTypeNullable.h>
|
||||
@ -354,7 +355,7 @@ void DatabaseMySQL::cleanOutdatedTables()
|
||||
{
|
||||
for (auto iterator = outdated_tables.begin(); iterator != outdated_tables.end();)
|
||||
{
|
||||
if (!iterator->unique())
|
||||
if (!isSharedPtrUnique(*iterator))
|
||||
++iterator;
|
||||
else
|
||||
{
|
||||
|
||||
@ -716,6 +716,16 @@ static void writeFieldsToColumn(
|
||||
|
||||
null_map_column->insertValue(0);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Column is not null but field is null. It's possible due to overrides
|
||||
if (field.isNull())
|
||||
{
|
||||
column_to.insertDefault();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return true;
|
||||
};
|
||||
@ -791,7 +801,7 @@ static void writeFieldsToColumn(
|
||||
|
||||
if (write_data_to_null_map(value, index))
|
||||
{
|
||||
const String & data = value.get<const String &>();
|
||||
const String & data = value.safeGet<const String &>();
|
||||
casted_string_column->insertData(data.data(), data.size());
|
||||
}
|
||||
}
|
||||
|
||||
@ -243,6 +243,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
|
||||
format_settings.orc.output_row_index_stride = settings.output_format_orc_row_index_stride;
|
||||
format_settings.orc.use_fast_decoder = settings.input_format_orc_use_fast_decoder;
|
||||
format_settings.orc.filter_push_down = settings.input_format_orc_filter_push_down;
|
||||
format_settings.orc.read_use_writer_time_zone = settings.input_format_orc_read_use_writer_time_zone;
|
||||
format_settings.defaults_for_omitted_fields = settings.input_format_defaults_for_omitted_fields;
|
||||
format_settings.capn_proto.enum_comparing_mode = settings.format_capn_proto_enum_comparising_mode;
|
||||
format_settings.capn_proto.skip_fields_with_unsupported_types_in_schema_inference = settings.input_format_capn_proto_skip_fields_with_unsupported_types_in_schema_inference;
|
||||
|
||||
@ -404,6 +404,7 @@ struct FormatSettings
|
||||
bool use_fast_decoder = true;
|
||||
bool filter_push_down = true;
|
||||
UInt64 output_row_index_stride = 10'000;
|
||||
bool read_use_writer_time_zone = false;
|
||||
} orc{};
|
||||
|
||||
/// For capnProto format we should determine how to
|
||||
|
||||
@ -284,7 +284,12 @@ public:
|
||||
{
|
||||
while (x)
|
||||
{
|
||||
result_array_values_data.push_back(std::countr_zero(x));
|
||||
/// С++20 char8_t is not an unsigned integral type anymore https://godbolt.org/z/Mqcb7qn58
|
||||
/// and thus you cannot use std::countr_zero on it.
|
||||
if constexpr (std::is_same_v<UnsignedType, UInt8>)
|
||||
result_array_values_data.push_back(std::countr_zero(static_cast<unsigned char>(x)));
|
||||
else
|
||||
result_array_values_data.push_back(std::countr_zero(x));
|
||||
x &= (x - 1);
|
||||
}
|
||||
}
|
||||
@ -336,4 +341,3 @@ REGISTER_FUNCTION(BitToArray)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -103,27 +103,40 @@ private:
|
||||
sorted_labels[i].label = label;
|
||||
}
|
||||
|
||||
/// Stable sort is required for for labels to apply in same order if score is equal
|
||||
std::stable_sort(sorted_labels.begin(), sorted_labels.end(), [](const auto & lhs, const auto & rhs) { return lhs.score > rhs.score; });
|
||||
/// Sorting scores in descending order to traverse the ROC curve from left to right
|
||||
std::sort(sorted_labels.begin(), sorted_labels.end(), [](const auto & lhs, const auto & rhs) { return lhs.score > rhs.score; });
|
||||
|
||||
/// We will first calculate non-normalized area.
|
||||
|
||||
size_t area = 0;
|
||||
size_t count_positive = 0;
|
||||
Float64 area = 0.0;
|
||||
Float64 prev_score = sorted_labels[0].score;
|
||||
size_t prev_fp = 0, prev_tp = 0;
|
||||
size_t curr_fp = 0, curr_tp = 0;
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
// Only increment the area when the score changes
|
||||
if (sorted_labels[i].score != prev_score)
|
||||
{
|
||||
area += (curr_fp - prev_fp) * (curr_tp + prev_tp) / 2.0; // Trapezoidal area under curve (might degenerate to zero or to a rectangle)
|
||||
prev_fp = curr_fp;
|
||||
prev_tp = curr_tp;
|
||||
prev_score = sorted_labels[i].score;
|
||||
}
|
||||
|
||||
if (sorted_labels[i].label)
|
||||
++count_positive; /// The curve moves one step up. No area increase.
|
||||
curr_tp += 1; /// The curve moves one step up.
|
||||
else
|
||||
area += count_positive; /// The curve moves one step right. Area is increased by 1 * height = count_positive.
|
||||
curr_fp += 1; /// The curve moves one step right.
|
||||
}
|
||||
|
||||
/// Then divide the area to the area of rectangle.
|
||||
area += (curr_fp - prev_fp) * (curr_tp + prev_tp) / 2.0;
|
||||
|
||||
if (count_positive == 0 || count_positive == size)
|
||||
/// Then normalize it dividing by the area to the area of rectangle.
|
||||
|
||||
if (curr_tp == 0 || curr_tp == size)
|
||||
return std::numeric_limits<Float64>::quiet_NaN();
|
||||
|
||||
return static_cast<Float64>(area) / count_positive / (size - count_positive);
|
||||
return area / curr_tp / (size - curr_tp);
|
||||
}
|
||||
|
||||
static void vector(
|
||||
|
||||
@ -5,7 +5,17 @@ namespace DB
|
||||
|
||||
REGISTER_FUNCTION(Tuple)
|
||||
{
|
||||
factory.registerFunction<FunctionTuple>();
|
||||
factory.registerFunction<FunctionTuple>(FunctionDocumentation{
|
||||
.description = R"(
|
||||
Returns a tuple by grouping input arguments.
|
||||
|
||||
For columns C1, C2, ... with the types T1, T2, ..., it returns a named Tuple(C1 T1, C2 T2, ...) type tuple containing these columns if their names are unique and can be treated as unquoted identifiers, otherwise a Tuple(T1, T2, ...) is returned. There is no cost to execute the function.
|
||||
Tuples are normally used as intermediate values for an argument of IN operators, or for creating a list of formal parameters of lambda functions. Tuples can’t be written to a table.
|
||||
|
||||
The function implements the operator `(x, y, ...)`.
|
||||
)",
|
||||
.examples{{"typical", "SELECT tuple(1, 2)", "(1,2)"}},
|
||||
.categories{"Miscellaneous"}});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -6,20 +6,28 @@
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Parsers/isUnquotedIdentifier.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** tuple(x, y, ...) is a function that allows you to group several columns
|
||||
/** tuple(x, y, ...) is a function that allows you to group several columns.
|
||||
* tupleElement(tuple, n) is a function that allows you to retrieve a column from tuple.
|
||||
*/
|
||||
class FunctionTuple : public IFunction
|
||||
{
|
||||
bool enable_named_columns;
|
||||
|
||||
public:
|
||||
static constexpr auto name = "tuple";
|
||||
|
||||
/// maybe_unused: false-positive
|
||||
[[ maybe_unused ]] static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionTuple>(); }
|
||||
[[maybe_unused]] static FunctionPtr create(ContextPtr context)
|
||||
{
|
||||
return std::make_shared<FunctionTuple>(context->getSettingsRef().enable_named_columns_in_function_tuple);
|
||||
}
|
||||
|
||||
explicit FunctionTuple(bool enable_named_columns_ = false) : enable_named_columns(enable_named_columns_) { }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
@ -38,9 +46,26 @@ public:
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
return std::make_shared<DataTypeTuple>(arguments);
|
||||
if (arguments.empty())
|
||||
return std::make_shared<DataTypeTuple>(DataTypes{});
|
||||
|
||||
DataTypes types;
|
||||
Names names;
|
||||
NameSet name_set;
|
||||
for (const auto & argument : arguments)
|
||||
{
|
||||
types.emplace_back(argument.type);
|
||||
names.emplace_back(argument.name);
|
||||
name_set.emplace(argument.name);
|
||||
}
|
||||
|
||||
if (enable_named_columns && name_set.size() == names.size()
|
||||
&& std::all_of(names.cbegin(), names.cend(), [](const auto & n) { return isUnquotedIdentifier(n); }))
|
||||
return std::make_shared<DataTypeTuple>(types, names);
|
||||
else
|
||||
return std::make_shared<DataTypeTuple>(types);
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
@ -53,9 +78,9 @@ public:
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
{
|
||||
/** If tuple is mixed of constant and not constant columns,
|
||||
* convert all to non-constant columns,
|
||||
* because many places in code expect all non-constant columns in non-constant tuple.
|
||||
*/
|
||||
* convert all to non-constant columns,
|
||||
* because many places in code expect all non-constant columns in non-constant tuple.
|
||||
*/
|
||||
tuple_columns[i] = arguments[i].column->convertToFullColumnIfConst();
|
||||
}
|
||||
return ColumnTuple::create(tuple_columns);
|
||||
|
||||
118
src/Functions/tupleNames.cpp
Normal file
118
src/Functions/tupleNames.cpp
Normal file
@ -0,0 +1,118 @@
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/** Transform a named tuple into names, which is a constant array of strings.
|
||||
*/
|
||||
class ExecutableFunctionTupleNames : public IExecutableFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "tupleNames";
|
||||
|
||||
explicit ExecutableFunctionTupleNames(Array name_fields_) : name_fields(std::move(name_fields_)) { }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName &, const DataTypePtr & result_type, size_t input_rows_count) const override
|
||||
{
|
||||
return result_type->createColumnConst(input_rows_count, name_fields);
|
||||
}
|
||||
|
||||
private:
|
||||
Array name_fields;
|
||||
};
|
||||
|
||||
class FunctionBaseTupleNames : public IFunctionBase
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "tupleNames";
|
||||
|
||||
explicit FunctionBaseTupleNames(DataTypePtr argument_type, DataTypePtr result_type_, Array name_fields_)
|
||||
: argument_types({std::move(argument_type)}), result_type(std::move(result_type_)), name_fields(std::move(name_fields_))
|
||||
{
|
||||
}
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
bool isSuitableForConstantFolding() const override { return true; }
|
||||
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
const DataTypes & getArgumentTypes() const override { return argument_types; }
|
||||
|
||||
const DataTypePtr & getResultType() const override { return result_type; }
|
||||
|
||||
ExecutableFunctionPtr prepare(const ColumnsWithTypeAndName &) const override
|
||||
{
|
||||
return std::make_unique<ExecutableFunctionTupleNames>(name_fields);
|
||||
}
|
||||
|
||||
private:
|
||||
DataTypes argument_types;
|
||||
DataTypePtr result_type;
|
||||
Array name_fields;
|
||||
};
|
||||
|
||||
class TupleNamesOverloadResolver : public IFunctionOverloadResolver
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "tupleNames";
|
||||
|
||||
static FunctionOverloadResolverPtr create(ContextPtr) { return std::make_unique<TupleNamesOverloadResolver>(); }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
const DataTypeTuple * tuple = checkAndGetDataType<DataTypeTuple>(arguments[0].type.get());
|
||||
|
||||
if (!tuple)
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be a tuple", getName());
|
||||
|
||||
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
|
||||
}
|
||||
|
||||
FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) const override
|
||||
{
|
||||
const DataTypeTuple * tuple = checkAndGetDataType<DataTypeTuple>(arguments[0].type.get());
|
||||
|
||||
if (!tuple)
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be a tuple", getName());
|
||||
|
||||
DataTypes types = tuple->getElements();
|
||||
Array name_fields;
|
||||
for (const auto & elem_name : tuple->getElementNames())
|
||||
name_fields.emplace_back(elem_name);
|
||||
|
||||
return std::make_unique<FunctionBaseTupleNames>(arguments[0].type, result_type, std::move(name_fields));
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
REGISTER_FUNCTION(TupleNames)
|
||||
{
|
||||
factory.registerFunction<TupleNamesOverloadResolver>(FunctionDocumentation{
|
||||
.description = R"(
|
||||
Converts a tuple into an array of column names. For a tuple in the form `Tuple(a T, b T, ...)`, it returns an array of strings representing the named columns of the tuple. If the tuple elements do not have explicit names, their indices will be used as the column names instead.
|
||||
)",
|
||||
.examples{{"typical", "SELECT tupleNames(tuple(1 as a, 2 as b))", "['a','b']"}},
|
||||
.categories{"Miscellaneous"}});
|
||||
}
|
||||
|
||||
}
|
||||
@ -405,10 +405,6 @@ Block createBlockForSet(
|
||||
|
||||
}
|
||||
|
||||
ScopeStack::Level::Level() = default;
|
||||
ScopeStack::Level::~Level() = default;
|
||||
ScopeStack::Level::Level(Level &&) noexcept = default;
|
||||
|
||||
FutureSetPtr makeExplicitSet(
|
||||
const ASTFunction * node, const ActionsDAG & actions, ContextPtr context, PreparedSets & prepared_sets)
|
||||
{
|
||||
@ -462,6 +458,7 @@ public:
|
||||
for (const auto * node : index)
|
||||
map.emplace(node->result_name, node);
|
||||
}
|
||||
~Index() = default;
|
||||
|
||||
void addNode(const ActionsDAG::Node * node)
|
||||
{
|
||||
@ -502,6 +499,10 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
ScopeStack::Level::Level() = default;
|
||||
ScopeStack::Level::~Level() = default;
|
||||
ScopeStack::Level::Level(Level &&) noexcept = default;
|
||||
|
||||
ActionsMatcher::Data::Data(
|
||||
ContextPtr context_,
|
||||
SizeLimits set_size_limit_,
|
||||
|
||||
@ -1,11 +1,16 @@
|
||||
#pragma once
|
||||
|
||||
|
||||
#include <boost/noncopyable.hpp>
|
||||
#include <base/isSharedPtrUnique.h>
|
||||
#include <Interpreters/Cache/Guards.h>
|
||||
#include <Interpreters/Cache/IFileCachePriority.h>
|
||||
#include <Interpreters/Cache/FileCacheKey.h>
|
||||
#include <Interpreters/Cache/FileSegment.h>
|
||||
#include <Interpreters/Cache/FileCache_fwd_internal.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
|
||||
#include <memory>
|
||||
#include <shared_mutex>
|
||||
|
||||
namespace DB
|
||||
@ -30,7 +35,7 @@ struct FileSegmentMetadata : private boost::noncopyable
|
||||
|
||||
explicit FileSegmentMetadata(FileSegmentPtr && file_segment_);
|
||||
|
||||
bool releasable() const { return file_segment.unique(); }
|
||||
bool releasable() const { return isSharedPtrUnique(file_segment); }
|
||||
|
||||
size_t size() const;
|
||||
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <mutex>
|
||||
#include <Interpreters/DatabaseCatalog.h>
|
||||
@ -27,6 +28,7 @@
|
||||
#include <Common/noexcept_scope.h>
|
||||
#include <Common/checkStackSize.h>
|
||||
|
||||
#include <base/isSharedPtrUnique.h>
|
||||
#include <boost/range/adaptor/map.hpp>
|
||||
|
||||
#include "config.h"
|
||||
@ -1197,7 +1199,7 @@ void DatabaseCatalog::dequeueDroppedTableCleanup(StorageID table_id)
|
||||
|
||||
/// It's unsafe to create another instance while the old one exists
|
||||
/// We cannot wait on shared_ptr's refcount, so it's busy wait
|
||||
while (!dropped_table.table.unique())
|
||||
while (!isSharedPtrUnique(dropped_table.table))
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
dropped_table.table.reset();
|
||||
|
||||
@ -1237,7 +1239,7 @@ void DatabaseCatalog::dropTableDataTask()
|
||||
size_t tables_in_use_count = 0;
|
||||
auto it = std::find_if(tables_marked_dropped.begin(), tables_marked_dropped.end(), [&](const auto & elem)
|
||||
{
|
||||
bool not_in_use = !elem.table || elem.table.unique();
|
||||
bool not_in_use = !elem.table || isSharedPtrUnique(elem.table);
|
||||
bool old_enough = elem.drop_time <= current_time;
|
||||
min_drop_time = std::min(min_drop_time, elem.drop_time);
|
||||
tables_in_use_count += !not_in_use;
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
#include <chrono>
|
||||
#include <functional>
|
||||
#include <exception>
|
||||
#include <unordered_map>
|
||||
#include <base/types.h>
|
||||
#include <Interpreters/IExternalLoadable.h>
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include <Interpreters/Session.h>
|
||||
|
||||
#include <base/isSharedPtrUnique.h>
|
||||
#include <Access/AccessControl.h>
|
||||
#include <Access/Credentials.h>
|
||||
#include <Access/ContextAccess.h>
|
||||
@ -130,7 +131,7 @@ public:
|
||||
|
||||
LOG_TRACE(log, "Reuse session from storage with session_id: {}, user_id: {}", key.second, key.first);
|
||||
|
||||
if (!session.unique())
|
||||
if (!isSharedPtrUnique(session))
|
||||
throw Exception(ErrorCodes::SESSION_IS_LOCKED, "Session {} is locked by a concurrent client", session_id);
|
||||
return {session, false};
|
||||
}
|
||||
@ -156,7 +157,7 @@ public:
|
||||
return;
|
||||
}
|
||||
|
||||
if (!it->second.unique())
|
||||
if (!isSharedPtrUnique(it->second))
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot close session {} with refcount {}", session_id, it->second.use_count());
|
||||
|
||||
sessions.erase(it);
|
||||
|
||||
@ -233,7 +233,8 @@ void ThreadStatus::attachToGroupImpl(const ThreadGroupPtr & thread_group_)
|
||||
{
|
||||
/// Attach or init current thread to thread group and copy useful information from it
|
||||
thread_group = thread_group_;
|
||||
thread_group->linkThread(thread_id);
|
||||
if (!internal_thread)
|
||||
thread_group->linkThread(thread_id);
|
||||
|
||||
performance_counters.setParent(&thread_group->performance_counters);
|
||||
memory_tracker.setParent(&thread_group->memory_tracker);
|
||||
@ -269,7 +270,8 @@ void ThreadStatus::detachFromGroup()
|
||||
/// Extract MemoryTracker out from query and user context
|
||||
memory_tracker.setParent(&total_memory_tracker);
|
||||
|
||||
thread_group->unlinkThread();
|
||||
if (!internal_thread)
|
||||
thread_group->unlinkThread();
|
||||
|
||||
thread_group.reset();
|
||||
|
||||
|
||||
@ -237,6 +237,7 @@ private:
|
||||
Int64 timeout_seconds = 120;
|
||||
bool is_replicated_database = false;
|
||||
bool throw_on_timeout = true;
|
||||
bool throw_on_timeout_only_active = false;
|
||||
bool only_running_hosts = false;
|
||||
|
||||
bool timeout_exceeded = false;
|
||||
@ -316,8 +317,8 @@ DDLQueryStatusSource::DDLQueryStatusSource(
|
||||
, log(getLogger("DDLQueryStatusSource"))
|
||||
{
|
||||
auto output_mode = context->getSettingsRef().distributed_ddl_output_mode;
|
||||
throw_on_timeout = output_mode == DistributedDDLOutputMode::THROW || output_mode == DistributedDDLOutputMode::THROW_ONLY_ACTIVE
|
||||
|| output_mode == DistributedDDLOutputMode::NONE || output_mode == DistributedDDLOutputMode::NONE_ONLY_ACTIVE;
|
||||
throw_on_timeout = output_mode == DistributedDDLOutputMode::THROW || output_mode == DistributedDDLOutputMode::NONE;
|
||||
throw_on_timeout_only_active = output_mode == DistributedDDLOutputMode::THROW_ONLY_ACTIVE || output_mode == DistributedDDLOutputMode::NONE_ONLY_ACTIVE;
|
||||
|
||||
if (hosts_to_wait)
|
||||
{
|
||||
@ -451,7 +452,7 @@ Chunk DDLQueryStatusSource::generate()
|
||||
"({} of them are currently executing the task, {} are inactive). "
|
||||
"They are going to execute the query in background. Was waiting for {} seconds{}";
|
||||
|
||||
if (throw_on_timeout)
|
||||
if (throw_on_timeout || (throw_on_timeout_only_active && !stop_waiting_offline_hosts))
|
||||
{
|
||||
if (!first_exception)
|
||||
first_exception = std::make_unique<Exception>(Exception(ErrorCodes::TIMEOUT_EXCEEDED,
|
||||
|
||||
@ -408,25 +408,26 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
|
||||
{
|
||||
const char * operators[] =
|
||||
{
|
||||
"multiply", " * ",
|
||||
"divide", " / ",
|
||||
"modulo", " % ",
|
||||
"plus", " + ",
|
||||
"minus", " - ",
|
||||
"notEquals", " != ",
|
||||
"lessOrEquals", " <= ",
|
||||
"greaterOrEquals", " >= ",
|
||||
"less", " < ",
|
||||
"greater", " > ",
|
||||
"equals", " = ",
|
||||
"like", " LIKE ",
|
||||
"ilike", " ILIKE ",
|
||||
"notLike", " NOT LIKE ",
|
||||
"notILike", " NOT ILIKE ",
|
||||
"in", " IN ",
|
||||
"notIn", " NOT IN ",
|
||||
"globalIn", " GLOBAL IN ",
|
||||
"globalNotIn", " GLOBAL NOT IN ",
|
||||
"multiply", " * ",
|
||||
"divide", " / ",
|
||||
"modulo", " % ",
|
||||
"plus", " + ",
|
||||
"minus", " - ",
|
||||
"notEquals", " != ",
|
||||
"lessOrEquals", " <= ",
|
||||
"greaterOrEquals", " >= ",
|
||||
"less", " < ",
|
||||
"greater", " > ",
|
||||
"equals", " = ",
|
||||
"isNotDistinctFrom", " <=> ",
|
||||
"like", " LIKE ",
|
||||
"ilike", " ILIKE ",
|
||||
"notLike", " NOT LIKE ",
|
||||
"notILike", " NOT ILIKE ",
|
||||
"in", " IN ",
|
||||
"notIn", " NOT IN ",
|
||||
"globalIn", " GLOBAL IN ",
|
||||
"globalNotIn", " GLOBAL NOT IN ",
|
||||
nullptr
|
||||
};
|
||||
|
||||
|
||||
@ -243,7 +243,7 @@ void ASTTableJoin::formatImplAfterTable(const FormatSettings & settings, FormatS
|
||||
void ASTTableJoin::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
||||
{
|
||||
formatImplBeforeTable(settings, state, frame);
|
||||
settings.ostr << " ... ";
|
||||
settings.ostr << " ...";
|
||||
formatImplAfterTable(settings, state, frame);
|
||||
}
|
||||
|
||||
|
||||
20
src/Parsers/isUnquotedIdentifier.cpp
Normal file
20
src/Parsers/isUnquotedIdentifier.cpp
Normal file
@ -0,0 +1,20 @@
|
||||
#include <Parsers/isUnquotedIdentifier.h>
|
||||
|
||||
#include <Parsers/Lexer.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
bool isUnquotedIdentifier(const String & name)
|
||||
{
|
||||
Lexer lexer(name.data(), name.data() + name.size());
|
||||
|
||||
auto maybe_ident = lexer.nextToken();
|
||||
|
||||
if (maybe_ident.type != TokenType::BareWord)
|
||||
return false;
|
||||
|
||||
return lexer.nextToken().isEnd();
|
||||
}
|
||||
|
||||
}
|
||||
10
src/Parsers/isUnquotedIdentifier.h
Normal file
10
src/Parsers/isUnquotedIdentifier.h
Normal file
@ -0,0 +1,10 @@
|
||||
#pragma once
|
||||
|
||||
#include <base/types.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
bool isUnquotedIdentifier(const String & name);
|
||||
|
||||
}
|
||||
@ -647,7 +647,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
|
||||
auto table_expression_query_info = select_query_info;
|
||||
table_expression_query_info.table_expression = table_expression;
|
||||
table_expression_query_info.filter_actions_dag = table_expression_data.getFilterActions();
|
||||
table_expression_query_info.analyzer_can_use_parallel_replicas_on_follower = table_node == planner_context->getGlobalPlannerContext()->parallel_replicas_table;
|
||||
table_expression_query_info.current_table_chosen_for_reading_with_parallel_replicas
|
||||
= table_node == planner_context->getGlobalPlannerContext()->parallel_replicas_table;
|
||||
|
||||
size_t max_streams = settings.max_threads;
|
||||
size_t max_threads_execute_query = settings.max_threads;
|
||||
@ -862,6 +863,15 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
|
||||
from_stage = storage->getQueryProcessingStage(
|
||||
query_context, select_query_options.to_stage, storage_snapshot, table_expression_query_info);
|
||||
|
||||
/// It is just a safety check needed until we have a proper sending plan to replicas.
|
||||
/// If we have a non-trivial storage like View it might create its own Planner inside read(), run findTableForParallelReplicas()
|
||||
/// and find some other table that might be used for reading with parallel replicas. It will lead to errors.
|
||||
const bool other_table_already_chosen_for_reading_with_parallel_replicas
|
||||
= planner_context->getGlobalPlannerContext()->parallel_replicas_table
|
||||
&& !table_expression_query_info.current_table_chosen_for_reading_with_parallel_replicas;
|
||||
if (other_table_already_chosen_for_reading_with_parallel_replicas)
|
||||
planner_context->getMutableQueryContext()->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0));
|
||||
|
||||
storage->read(
|
||||
query_plan,
|
||||
columns_names,
|
||||
|
||||
@ -528,7 +528,7 @@ JoinClausesAndActions buildJoinClausesAndActions(
|
||||
size_t join_clause_key_nodes_size = join_clause.getLeftKeyNodes().size();
|
||||
|
||||
if (join_clause_key_nodes_size == 0)
|
||||
throw Exception(ErrorCodes::INVALID_JOIN_ON_EXPRESSION, "JOIN {} cannot get JOIN keys",
|
||||
throw Exception(ErrorCodes::INVALID_JOIN_ON_EXPRESSION, "Cannot determine join keys in {}",
|
||||
join_node.formatASTForErrorMessage());
|
||||
|
||||
for (size_t i = 0; i < join_clause_key_nodes_size; ++i)
|
||||
|
||||
@ -227,7 +227,12 @@ private:
|
||||
return true;
|
||||
}
|
||||
|
||||
String column_name = "_dummy_" + std::to_string(replaced_literals.size());
|
||||
/// When generating placeholder names, ensure that we use names
|
||||
/// requiring quotes to be valid identifiers. This prevents the
|
||||
/// tuple() function from generating named tuples. Otherwise,
|
||||
/// inserting named tuples with different names into another named
|
||||
/// tuple will result in only default values being inserted.
|
||||
String column_name = "-dummy-" + std::to_string(replaced_literals.size());
|
||||
replaced_literals.emplace_back(literal, column_name, force_nullable);
|
||||
setDataType(replaced_literals.back());
|
||||
ast = std::make_shared<ASTIdentifier>(column_name);
|
||||
|
||||
@ -900,6 +900,11 @@ bool NativeORCBlockInputFormat::prepareStripeReader()
|
||||
|
||||
orc::RowReaderOptions row_reader_options;
|
||||
row_reader_options.includeTypes(include_indices);
|
||||
if (format_settings.orc.read_use_writer_time_zone)
|
||||
{
|
||||
String writer_time_zone = current_stripe_info->getWriterTimezone();
|
||||
row_reader_options.setTimezoneName(writer_time_zone);
|
||||
}
|
||||
row_reader_options.range(current_stripe_info->getOffset(), current_stripe_info->getLength());
|
||||
if (format_settings.orc.filter_push_down && sarg)
|
||||
{
|
||||
|
||||
85
src/Processors/QueryPlan/BufferChunksTransform.cpp
Normal file
85
src/Processors/QueryPlan/BufferChunksTransform.cpp
Normal file
@ -0,0 +1,85 @@
|
||||
#include <Processors/QueryPlan/BufferChunksTransform.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
BufferChunksTransform::BufferChunksTransform(
|
||||
const Block & header_,
|
||||
size_t max_rows_to_buffer_,
|
||||
size_t max_bytes_to_buffer_,
|
||||
size_t limit_)
|
||||
: IProcessor({header_}, {header_})
|
||||
, input(inputs.front())
|
||||
, output(outputs.front())
|
||||
, max_rows_to_buffer(max_rows_to_buffer_)
|
||||
, max_bytes_to_buffer(max_bytes_to_buffer_)
|
||||
, limit(limit_)
|
||||
{
|
||||
}
|
||||
|
||||
IProcessor::Status BufferChunksTransform::prepare()
|
||||
{
|
||||
if (output.isFinished())
|
||||
{
|
||||
chunks = {};
|
||||
input.close();
|
||||
return Status::Finished;
|
||||
}
|
||||
|
||||
if (input.isFinished() && chunks.empty())
|
||||
{
|
||||
output.finish();
|
||||
return Status::Finished;
|
||||
}
|
||||
|
||||
if (output.canPush())
|
||||
{
|
||||
input.setNeeded();
|
||||
|
||||
if (!chunks.empty())
|
||||
{
|
||||
auto chunk = std::move(chunks.front());
|
||||
chunks.pop();
|
||||
|
||||
num_buffered_rows -= chunk.getNumRows();
|
||||
num_buffered_bytes -= chunk.bytes();
|
||||
|
||||
output.push(std::move(chunk));
|
||||
}
|
||||
else if (input.hasData())
|
||||
{
|
||||
auto chunk = pullChunk();
|
||||
output.push(std::move(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
if (input.hasData() && (num_buffered_rows < max_rows_to_buffer || num_buffered_bytes < max_bytes_to_buffer))
|
||||
{
|
||||
auto chunk = pullChunk();
|
||||
num_buffered_rows += chunk.getNumRows();
|
||||
num_buffered_bytes += chunk.bytes();
|
||||
chunks.push(std::move(chunk));
|
||||
}
|
||||
|
||||
if (num_buffered_rows >= max_rows_to_buffer && num_buffered_bytes >= max_bytes_to_buffer)
|
||||
{
|
||||
input.setNotNeeded();
|
||||
return Status::PortFull;
|
||||
}
|
||||
|
||||
input.setNeeded();
|
||||
return Status::NeedData;
|
||||
}
|
||||
|
||||
Chunk BufferChunksTransform::pullChunk()
|
||||
{
|
||||
auto chunk = input.pull();
|
||||
num_processed_rows += chunk.getNumRows();
|
||||
|
||||
if (limit && num_processed_rows >= limit)
|
||||
input.close();
|
||||
|
||||
return chunk;
|
||||
}
|
||||
|
||||
}
|
||||
42
src/Processors/QueryPlan/BufferChunksTransform.h
Normal file
42
src/Processors/QueryPlan/BufferChunksTransform.h
Normal file
@ -0,0 +1,42 @@
|
||||
#pragma once
|
||||
#include <Processors/IProcessor.h>
|
||||
#include <queue>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Transform that buffers chunks from the input
|
||||
/// up to the certain limit and pushes chunks to
|
||||
/// the output whenever it is ready. It can be used
|
||||
/// to increase parallelism of execution, for example
|
||||
/// when it is adeded before MergingSortedTransform.
|
||||
class BufferChunksTransform : public IProcessor
|
||||
{
|
||||
public:
|
||||
/// OR condition is used for the limits on rows and bytes.
|
||||
BufferChunksTransform(
|
||||
const Block & header_,
|
||||
size_t max_rows_to_buffer_,
|
||||
size_t max_bytes_to_buffer_,
|
||||
size_t limit_);
|
||||
|
||||
Status prepare() override;
|
||||
String getName() const override { return "BufferChunks"; }
|
||||
|
||||
private:
|
||||
Chunk pullChunk();
|
||||
|
||||
InputPort & input;
|
||||
OutputPort & output;
|
||||
|
||||
size_t max_rows_to_buffer;
|
||||
size_t max_bytes_to_buffer;
|
||||
size_t limit;
|
||||
|
||||
std::queue<Chunk> chunks;
|
||||
size_t num_buffered_rows = 0;
|
||||
size_t num_buffered_bytes = 0;
|
||||
size_t num_processed_rows = 0;
|
||||
};
|
||||
|
||||
}
|
||||
@ -919,15 +919,23 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes)
|
||||
{
|
||||
auto & union_node = node.children.front();
|
||||
|
||||
std::vector<InputOrderInfoPtr> infos;
|
||||
bool use_buffering = false;
|
||||
const SortDescription * max_sort_descr = nullptr;
|
||||
|
||||
std::vector<InputOrderInfoPtr> infos;
|
||||
infos.reserve(node.children.size());
|
||||
|
||||
for (auto * child : union_node->children)
|
||||
{
|
||||
infos.push_back(buildInputOrderInfo(*sorting, *child, steps_to_update));
|
||||
|
||||
if (infos.back() && (!max_sort_descr || max_sort_descr->size() < infos.back()->sort_description_for_merging.size()))
|
||||
max_sort_descr = &infos.back()->sort_description_for_merging;
|
||||
if (infos.back())
|
||||
{
|
||||
if (!max_sort_descr || max_sort_descr->size() < infos.back()->sort_description_for_merging.size())
|
||||
max_sort_descr = &infos.back()->sort_description_for_merging;
|
||||
|
||||
use_buffering |= infos.back()->limit == 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (!max_sort_descr || max_sort_descr->empty())
|
||||
@ -972,12 +980,13 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes)
|
||||
}
|
||||
}
|
||||
|
||||
sorting->convertToFinishSorting(*max_sort_descr);
|
||||
sorting->convertToFinishSorting(*max_sort_descr, use_buffering);
|
||||
}
|
||||
else if (auto order_info = buildInputOrderInfo(*sorting, *node.children.front(), steps_to_update))
|
||||
{
|
||||
sorting->convertToFinishSorting(order_info->sort_description_for_merging);
|
||||
/// update data stream's sorting properties
|
||||
/// Use buffering only if have filter or don't have limit.
|
||||
bool use_buffering = order_info->limit == 0;
|
||||
sorting->convertToFinishSorting(order_info->sort_description_for_merging, use_buffering);
|
||||
updateStepsDataStreams(steps_to_update);
|
||||
}
|
||||
}
|
||||
@ -1091,7 +1100,7 @@ size_t tryReuseStorageOrderingForWindowFunctions(QueryPlan::Node * parent_node,
|
||||
bool can_read = read_from_merge_tree->requestReadingInOrder(order_info->used_prefix_of_sorting_key_size, order_info->direction, order_info->limit);
|
||||
if (!can_read)
|
||||
return 0;
|
||||
sorting->convertToFinishSorting(order_info->sort_description_for_merging);
|
||||
sorting->convertToFinishSorting(order_info->sort_description_for_merging, false);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <IO/Operators.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Processors/Merges/MergingSortedTransform.h>
|
||||
@ -8,6 +7,7 @@
|
||||
#include <Processors/Transforms/LimitsCheckingTransform.h>
|
||||
#include <Processors/Transforms/MergeSortingTransform.h>
|
||||
#include <Processors/Transforms/PartialSortingTransform.h>
|
||||
#include <Processors/QueryPlan/BufferChunksTransform.h>
|
||||
#include <QueryPipeline/QueryPipelineBuilder.h>
|
||||
#include <Common/JSONBuilder.h>
|
||||
|
||||
@ -38,6 +38,7 @@ SortingStep::Settings::Settings(const Context & context)
|
||||
tmp_data = context.getTempDataOnDisk();
|
||||
min_free_disk_space = settings.min_free_disk_space_for_temporary_data;
|
||||
max_block_bytes = settings.prefer_external_sort_block_bytes;
|
||||
read_in_order_use_buffering = settings.read_in_order_use_buffering;
|
||||
}
|
||||
|
||||
SortingStep::Settings::Settings(size_t max_block_size_)
|
||||
@ -153,10 +154,11 @@ void SortingStep::updateLimit(size_t limit_)
|
||||
}
|
||||
}
|
||||
|
||||
void SortingStep::convertToFinishSorting(SortDescription prefix_description_)
|
||||
void SortingStep::convertToFinishSorting(SortDescription prefix_description_, bool use_buffering_)
|
||||
{
|
||||
type = Type::FinishSorting;
|
||||
prefix_description = std::move(prefix_description_);
|
||||
use_buffering = use_buffering_;
|
||||
}
|
||||
|
||||
void SortingStep::scatterByPartitionIfNeeded(QueryPipelineBuilder& pipeline)
|
||||
@ -244,6 +246,14 @@ void SortingStep::mergingSorted(QueryPipelineBuilder & pipeline, const SortDescr
|
||||
/// If there are several streams, then we merge them into one
|
||||
if (pipeline.getNumStreams() > 1)
|
||||
{
|
||||
if (use_buffering && sort_settings.read_in_order_use_buffering)
|
||||
{
|
||||
pipeline.addSimpleTransform([&](const Block & header)
|
||||
{
|
||||
return std::make_shared<BufferChunksTransform>(header, sort_settings.max_block_size, sort_settings.max_block_bytes, limit_);
|
||||
});
|
||||
}
|
||||
|
||||
auto transform = std::make_shared<MergingSortedTransform>(
|
||||
pipeline.getHeader(),
|
||||
pipeline.getNumStreams(),
|
||||
@ -373,9 +383,8 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build
|
||||
mergingSorted(pipeline, prefix_description, (need_finish_sorting ? 0 : limit));
|
||||
|
||||
if (need_finish_sorting)
|
||||
{
|
||||
finishSorting(pipeline, prefix_description, result_description, limit);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@ -28,6 +28,7 @@ public:
|
||||
TemporaryDataOnDiskScopePtr tmp_data = nullptr;
|
||||
size_t min_free_disk_space = 0;
|
||||
size_t max_block_bytes = 0;
|
||||
size_t read_in_order_use_buffering = 0;
|
||||
|
||||
explicit Settings(const Context & context);
|
||||
explicit Settings(size_t max_block_size_);
|
||||
@ -80,7 +81,7 @@ public:
|
||||
|
||||
const SortDescription & getSortDescription() const { return result_description; }
|
||||
|
||||
void convertToFinishSorting(SortDescription prefix_description);
|
||||
void convertToFinishSorting(SortDescription prefix_description, bool use_buffering_);
|
||||
|
||||
Type getType() const { return type; }
|
||||
const Settings & getSettings() const { return sort_settings; }
|
||||
@ -126,6 +127,7 @@ private:
|
||||
|
||||
UInt64 limit;
|
||||
bool always_read_till_end = false;
|
||||
bool use_buffering = false;
|
||||
|
||||
Settings sort_settings;
|
||||
|
||||
|
||||
@ -193,7 +193,15 @@ PostgreSQLSource<T>::~PostgreSQLSource()
|
||||
{
|
||||
if (stream)
|
||||
{
|
||||
/** Internally libpqxx::stream_from runs PostgreSQL copy query `COPY query TO STDOUT`.
|
||||
* During transaction abort we try to execute PostgreSQL `ROLLBACK` command and if
|
||||
* copy query is not cancelled, we wait until it finishes.
|
||||
*/
|
||||
tx->conn().cancel_query();
|
||||
|
||||
/** If stream is not closed, libpqxx::stream_from closes stream in destructor, but that way
|
||||
* exception is added into transaction pending error and we can potentially ignore exception message.
|
||||
*/
|
||||
stream->close();
|
||||
}
|
||||
|
||||
|
||||
@ -87,6 +87,7 @@
|
||||
|
||||
#include <base/insertAtEnd.h>
|
||||
#include <base/interpolate.h>
|
||||
#include <base/isSharedPtrUnique.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
@ -2464,7 +2465,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts(bool force)
|
||||
}
|
||||
|
||||
/// Grab only parts that are not used by anyone (SELECTs for example).
|
||||
if (!part.unique())
|
||||
if (!isSharedPtrUnique(part))
|
||||
{
|
||||
part->removal_state.store(DataPartRemovalState::NON_UNIQUE_OWNERSHIP, std::memory_order_relaxed);
|
||||
skipped_parts.push_back(part->info);
|
||||
@ -4360,13 +4361,13 @@ bool MergeTreeData::tryRemovePartImmediately(DataPartPtr && part)
|
||||
|
||||
part.reset();
|
||||
|
||||
if (!((*it)->getState() == DataPartState::Outdated && it->unique()))
|
||||
if (!((*it)->getState() == DataPartState::Outdated && isSharedPtrUnique(*it)))
|
||||
{
|
||||
if ((*it)->getState() != DataPartState::Outdated)
|
||||
LOG_WARNING(log, "Cannot immediately remove part {} because it's not in Outdated state "
|
||||
"usage counter {}", part_name_with_state, it->use_count());
|
||||
|
||||
if (!it->unique())
|
||||
if (!isSharedPtrUnique(*it))
|
||||
LOG_WARNING(log, "Cannot immediately remove part {} because someone using it right now "
|
||||
"usage counter {}", part_name_with_state, it->use_count());
|
||||
return false;
|
||||
@ -4432,7 +4433,7 @@ size_t MergeTreeData::getNumberOfOutdatedPartsWithExpiredRemovalTime() const
|
||||
for (const auto & part : outdated_parts_range)
|
||||
{
|
||||
auto part_remove_time = part->remove_time.load(std::memory_order_relaxed);
|
||||
if (part_remove_time <= time_now && time_now - part_remove_time >= getSettings()->old_parts_lifetime.totalSeconds() && part.unique())
|
||||
if (part_remove_time <= time_now && time_now - part_remove_time >= getSettings()->old_parts_lifetime.totalSeconds() && isSharedPtrUnique(part))
|
||||
++res;
|
||||
}
|
||||
|
||||
@ -8640,7 +8641,7 @@ size_t MergeTreeData::unloadPrimaryKeysOfOutdatedParts()
|
||||
/// Outdated part may be hold by SELECT query and still needs the index.
|
||||
/// This check requires lock of index_mutex but if outdated part is unique then there is no
|
||||
/// contention on it, so it's relatively cheap and it's ok to check under a global parts lock.
|
||||
if (part.unique() && part->isIndexLoaded())
|
||||
if (isSharedPtrUnique(part) && part->isIndexLoaded())
|
||||
parts_to_unload_index.push_back(part);
|
||||
}
|
||||
}
|
||||
|
||||
@ -444,6 +444,9 @@ void DefaultCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRangesAn
|
||||
ErrorCodes::LOGICAL_ERROR, "Replica number ({}) is bigger than total replicas count ({})", replica_num, stats.size());
|
||||
|
||||
++stats[replica_num].number_of_requests;
|
||||
|
||||
if (replica_status[replica_num].is_announcement_received)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Duplicate announcement received for replica number {}", replica_num);
|
||||
replica_status[replica_num].is_announcement_received = true;
|
||||
|
||||
LOG_DEBUG(log, "Sent initial requests: {} Replicas count: {}", sent_initial_requests, replicas_count);
|
||||
|
||||
@ -166,7 +166,7 @@ struct SelectQueryInfo
|
||||
/// It's guaranteed to be present in JOIN TREE of `query_tree`
|
||||
QueryTreeNodePtr table_expression;
|
||||
|
||||
bool analyzer_can_use_parallel_replicas_on_follower = false;
|
||||
bool current_table_chosen_for_reading_with_parallel_replicas = false;
|
||||
|
||||
/// Table expression modifiers for storage
|
||||
std::optional<TableExpressionModifiers> table_expression_modifiers;
|
||||
|
||||
@ -366,12 +366,21 @@ Strings StorageFile::getPathsList(const String & table_path, const String & user
|
||||
}
|
||||
else if (path.find_first_of("*?{") == std::string::npos)
|
||||
{
|
||||
std::error_code error;
|
||||
size_t size = fs::file_size(path, error);
|
||||
if (!error)
|
||||
total_bytes_to_read += size;
|
||||
if (!fs::is_directory(path))
|
||||
{
|
||||
std::error_code error;
|
||||
size_t size = fs::file_size(path, error);
|
||||
if (!error)
|
||||
total_bytes_to_read += size;
|
||||
|
||||
paths.push_back(path);
|
||||
paths.push_back(path);
|
||||
}
|
||||
else
|
||||
{
|
||||
/// We list non-directory files under that directory.
|
||||
paths = listFilesWithRegexpMatching(path / fs::path("*"), total_bytes_to_read);
|
||||
can_be_directory = false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -252,7 +252,7 @@ void StorageMergeTree::read(
|
||||
|
||||
const bool enable_parallel_reading = local_context->canUseParallelReplicasOnFollower()
|
||||
&& local_context->getSettingsRef().parallel_replicas_for_non_replicated_merge_tree
|
||||
&& (!local_context->getSettingsRef().allow_experimental_analyzer || query_info.analyzer_can_use_parallel_replicas_on_follower);
|
||||
&& (!local_context->getSettingsRef().allow_experimental_analyzer || query_info.current_table_chosen_for_reading_with_parallel_replicas);
|
||||
|
||||
if (auto plan = reader.read(
|
||||
column_names,
|
||||
|
||||
@ -5540,7 +5540,8 @@ void StorageReplicatedMergeTree::readLocalImpl(
|
||||
const size_t num_streams)
|
||||
{
|
||||
const bool enable_parallel_reading = local_context->canUseParallelReplicasOnFollower()
|
||||
&& (!local_context->getSettingsRef().allow_experimental_analyzer || query_info.analyzer_can_use_parallel_replicas_on_follower);
|
||||
&& (!local_context->getSettingsRef().allow_experimental_analyzer
|
||||
|| query_info.current_table_chosen_for_reading_with_parallel_replicas);
|
||||
|
||||
auto plan = reader.read(
|
||||
column_names, storage_snapshot, query_info,
|
||||
|
||||
@ -311,42 +311,42 @@ class CI:
|
||||
random_bucket="parrepl_with_sanitizer",
|
||||
),
|
||||
JobNames.STATELESS_TEST_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_ASAN], num_batches=4
|
||||
required_builds=[BuildNames.PACKAGE_ASAN], num_batches=2
|
||||
),
|
||||
JobNames.STATELESS_TEST_TSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_TSAN], num_batches=5
|
||||
required_builds=[BuildNames.PACKAGE_TSAN], num_batches=2
|
||||
),
|
||||
JobNames.STATELESS_TEST_MSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_MSAN], num_batches=6
|
||||
required_builds=[BuildNames.PACKAGE_MSAN], num_batches=3
|
||||
),
|
||||
JobNames.STATELESS_TEST_UBSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_UBSAN], num_batches=2
|
||||
required_builds=[BuildNames.PACKAGE_UBSAN], num_batches=1
|
||||
),
|
||||
JobNames.STATELESS_TEST_DEBUG: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=5
|
||||
required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=2
|
||||
),
|
||||
JobNames.STATELESS_TEST_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_RELEASE],
|
||||
),
|
||||
JobNames.STATELESS_TEST_RELEASE_COVERAGE: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_RELEASE_COVERAGE], num_batches=6
|
||||
required_builds=[BuildNames.PACKAGE_RELEASE_COVERAGE], num_batches=5
|
||||
),
|
||||
JobNames.STATELESS_TEST_AARCH64: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_AARCH64],
|
||||
runner_type=Runners.FUNC_TESTER_ARM,
|
||||
),
|
||||
JobNames.STATELESS_TEST_OLD_ANALYZER_S3_REPLICATED_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=4
|
||||
required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=3
|
||||
),
|
||||
JobNames.STATELESS_TEST_S3_DEBUG: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=6
|
||||
required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=2
|
||||
),
|
||||
JobNames.STATELESS_TEST_AZURE_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_ASAN], num_batches=4, release_only=True
|
||||
required_builds=[BuildNames.PACKAGE_ASAN], num_batches=2, release_only=True
|
||||
),
|
||||
JobNames.STATELESS_TEST_S3_TSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_TSAN],
|
||||
num_batches=5,
|
||||
num_batches=3,
|
||||
),
|
||||
JobNames.STRESS_TEST_DEBUG: CommonJobConfigs.STRESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_DEBUG],
|
||||
|
||||
@ -709,9 +709,9 @@ def get_localzone():
|
||||
|
||||
class SettingsRandomizer:
|
||||
settings = {
|
||||
"max_insert_threads": lambda: (
|
||||
0 if random.random() < 0.5 else random.randint(1, 16)
|
||||
),
|
||||
"max_insert_threads": lambda: 32
|
||||
if random.random() < 0.03
|
||||
else random.randint(1, 3),
|
||||
"group_by_two_level_threshold": threshold_generator(0.2, 0.2, 1, 1000000),
|
||||
"group_by_two_level_threshold_bytes": threshold_generator(
|
||||
0.2, 0.2, 1, 50000000
|
||||
@ -727,7 +727,7 @@ class SettingsRandomizer:
|
||||
"prefer_localhost_replica": lambda: random.randint(0, 1),
|
||||
"max_block_size": lambda: random.randint(8000, 100000),
|
||||
"max_joined_block_size_rows": lambda: random.randint(8000, 100000),
|
||||
"max_threads": lambda: random.randint(1, 64),
|
||||
"max_threads": lambda: 64 if random.random() < 0.03 else random.randint(1, 3),
|
||||
"optimize_append_index": lambda: random.randint(0, 1),
|
||||
"optimize_if_chain_to_multiif": lambda: random.randint(0, 1),
|
||||
"optimize_if_transform_strings_to_enum": lambda: random.randint(0, 1),
|
||||
@ -1217,6 +1217,11 @@ class TestCase:
|
||||
):
|
||||
return FailureReason.OBJECT_STORAGE
|
||||
|
||||
elif "no-batch" in tags and (
|
||||
args.run_by_hash_num is not None or args.run_by_hash_total is not None
|
||||
):
|
||||
return FailureReason.SKIP
|
||||
|
||||
elif tags:
|
||||
for build_flag in args.build_flags:
|
||||
if "no-" + build_flag in tags:
|
||||
@ -1447,8 +1452,7 @@ class TestCase:
|
||||
description_full = messages[result.status]
|
||||
description_full += self.print_test_time(result.total_time)
|
||||
if result.reason is not None:
|
||||
description_full += " - "
|
||||
description_full += result.reason.value
|
||||
description_full += f"\nReason: {result.reason.value} "
|
||||
|
||||
description_full += result.description
|
||||
|
||||
@ -1575,10 +1579,11 @@ class TestCase:
|
||||
# pylint:disable-next=consider-using-with; TODO: fix
|
||||
proc = Popen(command, shell=True, env=os.environ, start_new_session=True)
|
||||
|
||||
while (
|
||||
datetime.now() - start_time
|
||||
).total_seconds() < args.timeout and proc.poll() is None:
|
||||
sleep(0.01)
|
||||
try:
|
||||
proc.wait(args.timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
# Whether the test timed out will be decided later
|
||||
pass
|
||||
|
||||
debug_log = ""
|
||||
if os.path.exists(self.testcase_args.debug_log_file):
|
||||
@ -1600,6 +1605,44 @@ class TestCase:
|
||||
# Normalize hostname in stdout file.
|
||||
replace_in_file(self.stdout_file, socket.gethostname(), "localhost")
|
||||
|
||||
if os.environ.get("CLICKHOUSE_PORT_TCP"):
|
||||
replace_in_file(
|
||||
self.stdout_file,
|
||||
f"PORT {os.environ['CLICKHOUSE_PORT_TCP']}",
|
||||
"PORT 9000",
|
||||
)
|
||||
replace_in_file(
|
||||
self.stdout_file,
|
||||
f"localhost {os.environ['CLICKHOUSE_PORT_TCP']}",
|
||||
"localhost 9000",
|
||||
)
|
||||
|
||||
if os.environ.get("CLICKHOUSE_PORT_TCP_SECURE"):
|
||||
replace_in_file(
|
||||
self.stdout_file,
|
||||
f"PORT {os.environ['CLICKHOUSE_PORT_TCP_SECURE']}",
|
||||
"PORT 9440",
|
||||
)
|
||||
replace_in_file(
|
||||
self.stdout_file,
|
||||
f"localhost {os.environ['CLICKHOUSE_PORT_TCP_SECURE']}",
|
||||
"localhost 9440",
|
||||
)
|
||||
|
||||
if os.environ.get("CLICKHOUSE_PATH"):
|
||||
replace_in_file(
|
||||
self.stdout_file,
|
||||
os.environ["CLICKHOUSE_PATH"],
|
||||
"/var/lib/clickhouse",
|
||||
)
|
||||
|
||||
if os.environ.get("CLICKHOUSE_PORT_HTTPS"):
|
||||
replace_in_file(
|
||||
self.stdout_file,
|
||||
f"https://localhost:{os.environ['CLICKHOUSE_PORT_HTTPS']}/",
|
||||
"https://localhost:8443/",
|
||||
)
|
||||
|
||||
stdout = ""
|
||||
if os.path.exists(self.stdout_file):
|
||||
with open(self.stdout_file, "rb") as stdfd:
|
||||
@ -2056,8 +2099,13 @@ class GlobalTimeout(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def run_tests_array(all_tests_with_params: Tuple[List[str], int, TestSuite]):
|
||||
all_tests, num_tests, test_suite = all_tests_with_params
|
||||
def run_tests_array(all_tests_with_params: Tuple[List[str], int, TestSuite, bool]):
|
||||
(
|
||||
all_tests,
|
||||
num_tests,
|
||||
test_suite,
|
||||
is_concurrent,
|
||||
) = all_tests_with_params
|
||||
global stop_time
|
||||
global exit_code
|
||||
global server_died
|
||||
@ -2100,14 +2148,12 @@ def run_tests_array(all_tests_with_params: Tuple[List[str], int, TestSuite]):
|
||||
failures_chain = 0
|
||||
start_time = datetime.now()
|
||||
|
||||
is_concurrent = multiprocessing.current_process().name != "MainProcess"
|
||||
|
||||
client_options = get_additional_client_options(args)
|
||||
|
||||
if num_tests > 0:
|
||||
about = "about " if is_concurrent else ""
|
||||
proc_name = multiprocessing.current_process().name
|
||||
print(f"\nRunning {about}{num_tests} {test_suite.suite} tests ({proc_name}).\n")
|
||||
print(f"Running {about}{num_tests} {test_suite.suite} tests ({proc_name}).")
|
||||
|
||||
while True:
|
||||
if all_tests:
|
||||
@ -2128,16 +2174,17 @@ def run_tests_array(all_tests_with_params: Tuple[List[str], int, TestSuite]):
|
||||
|
||||
try:
|
||||
description = ""
|
||||
test_cace_name = removesuffix(test_case.name, ".gen", ".sql") + ": "
|
||||
if not is_concurrent:
|
||||
test_case_name = removesuffix(test_case.name, ".gen", ".sql") + ": "
|
||||
|
||||
if is_concurrent or args.run_sequential_tests_in_parallel:
|
||||
description = f"{test_case_name:72}"
|
||||
else:
|
||||
sys.stdout.flush()
|
||||
sys.stdout.write(f"{test_cace_name:72}")
|
||||
sys.stdout.write(f"{test_case_name:72}")
|
||||
# This flush is needed so you can see the test name of the long
|
||||
# running test before it will finish. But don't do it in parallel
|
||||
# mode, so that the lines don't mix.
|
||||
sys.stdout.flush()
|
||||
else:
|
||||
description = f"{test_cace_name:72}"
|
||||
|
||||
while True:
|
||||
test_result = test_case.run(
|
||||
@ -2372,6 +2419,35 @@ def extract_key(key: str) -> str:
|
||||
)[1]
|
||||
|
||||
|
||||
def override_envs(*args_, **kwargs):
|
||||
global args
|
||||
args.client += " --port 19000"
|
||||
args.http_port = 18123
|
||||
args.https_port = 18443
|
||||
|
||||
updated_env = {
|
||||
"CLICKHOUSE_CONFIG": "/etc/clickhouse-server3/config.xml",
|
||||
"CLICKHOUSE_CONFIG_DIR": "/etc/clickhouse-server3",
|
||||
"CLICKHOUSE_CONFIG_GREP": "/etc/clickhouse-server3/preprocessed/config.xml",
|
||||
"CLICKHOUSE_USER_FILES": "/var/lib/clickhouse3/user_files",
|
||||
"CLICKHOUSE_SCHEMA_FILES": "/var/lib/clickhouse3/format_schemas",
|
||||
"CLICKHOUSE_PATH": "/var/lib/clickhouse3",
|
||||
"CLICKHOUSE_PORT_TCP": "19000",
|
||||
"CLICKHOUSE_PORT_TCP_SECURE": "19440",
|
||||
"CLICKHOUSE_PORT_TCP_WITH_PROXY": "19010",
|
||||
"CLICKHOUSE_PORT_HTTP": "18123",
|
||||
"CLICKHOUSE_PORT_HTTPS": "18443",
|
||||
"CLICKHOUSE_PORT_INTERSERVER": "19009",
|
||||
"CLICKHOUSE_PORT_KEEPER": "19181",
|
||||
"CLICKHOUSE_PORT_PROMTHEUS_PORT": "19988",
|
||||
"CLICKHOUSE_PORT_MYSQL": "19004",
|
||||
"CLICKHOUSE_PORT_POSTGRESQL": "19005",
|
||||
}
|
||||
os.environ.update(updated_env)
|
||||
|
||||
run_tests_array(*args_, **kwargs)
|
||||
|
||||
|
||||
def do_run_tests(jobs, test_suite: TestSuite):
|
||||
if jobs > 1 and len(test_suite.parallel_tests) > 0:
|
||||
print(
|
||||
@ -2400,24 +2476,55 @@ def do_run_tests(jobs, test_suite: TestSuite):
|
||||
for job in range(jobs):
|
||||
range_ = job * batch_size, job * batch_size + batch_size
|
||||
batch = test_suite.parallel_tests[range_[0] : range_[1]]
|
||||
parallel_tests_array.append((batch, batch_size, test_suite))
|
||||
parallel_tests_array.append((batch, batch_size, test_suite, True))
|
||||
|
||||
try:
|
||||
with multiprocessing.Pool(processes=jobs) as pool:
|
||||
with multiprocessing.Pool(processes=jobs + 1) as pool:
|
||||
future = pool.map_async(run_tests_array, parallel_tests_array)
|
||||
|
||||
if args.run_sequential_tests_in_parallel:
|
||||
# Run parallel tests and sequential tests at the same time
|
||||
# Sequential tests will use different ClickHouse instance
|
||||
# In this process we can safely override values in `args` and `os.environ`
|
||||
future_seq = pool.map_async(
|
||||
override_envs,
|
||||
[
|
||||
(
|
||||
test_suite.sequential_tests,
|
||||
len(test_suite.sequential_tests),
|
||||
test_suite,
|
||||
False,
|
||||
)
|
||||
],
|
||||
)
|
||||
future_seq.wait()
|
||||
|
||||
future.wait()
|
||||
finally:
|
||||
pool.terminate()
|
||||
pool.close()
|
||||
pool.join()
|
||||
|
||||
run_tests_array(
|
||||
(test_suite.sequential_tests, len(test_suite.sequential_tests), test_suite)
|
||||
)
|
||||
if not args.run_sequential_tests_in_parallel:
|
||||
run_tests_array(
|
||||
(
|
||||
test_suite.sequential_tests,
|
||||
len(test_suite.sequential_tests),
|
||||
test_suite,
|
||||
False,
|
||||
)
|
||||
)
|
||||
return len(test_suite.sequential_tests) + len(test_suite.parallel_tests)
|
||||
else:
|
||||
num_tests = len(test_suite.all_tests)
|
||||
run_tests_array((test_suite.all_tests, num_tests, test_suite))
|
||||
run_tests_array(
|
||||
(
|
||||
test_suite.all_tests,
|
||||
num_tests,
|
||||
test_suite,
|
||||
False,
|
||||
)
|
||||
)
|
||||
return num_tests
|
||||
|
||||
|
||||
@ -2722,6 +2829,7 @@ def main(args):
|
||||
f"{get_db_engine(args, db_name)}",
|
||||
settings=get_create_database_settings(args, None),
|
||||
)
|
||||
break
|
||||
except HTTPError as e:
|
||||
total_time = (datetime.now() - start_time).total_seconds()
|
||||
if not need_retry(args, e.message, e.message, total_time):
|
||||
@ -3234,6 +3342,15 @@ def parse_args():
|
||||
help="Replace ordinary MergeTree engine with SharedMergeTree",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--run-sequential-tests-in-parallel",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="If `true`, tests with the tag `no-parallel` will run on a "
|
||||
"separate ClickHouse instance in parallel with other tests. "
|
||||
"This is used in CI to make test jobs run faster.",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@ -57,7 +57,6 @@ ln -sf $SRC_PATH/config.d/forbidden_headers.xml $DEST_SERVER_PATH/config.d/
|
||||
ln -sf $SRC_PATH/config.d/enable_keeper_map.xml $DEST_SERVER_PATH/config.d/
|
||||
ln -sf $SRC_PATH/config.d/custom_disks_base_path.xml $DEST_SERVER_PATH/config.d/
|
||||
ln -sf $SRC_PATH/config.d/display_name.xml $DEST_SERVER_PATH/config.d/
|
||||
ln -sf $SRC_PATH/config.d/reverse_dns_query_function.xml $DEST_SERVER_PATH/config.d/
|
||||
ln -sf $SRC_PATH/config.d/compressed_marks_and_index.xml $DEST_SERVER_PATH/config.d/
|
||||
ln -sf $SRC_PATH/config.d/disable_s3_env_credentials.xml $DEST_SERVER_PATH/config.d/
|
||||
ln -sf $SRC_PATH/config.d/enable_wait_for_shutdown_replicated_tables.xml $DEST_SERVER_PATH/config.d/
|
||||
|
||||
@ -48,6 +48,7 @@
|
||||
"test_system_metrics/test.py::test_readonly_metrics",
|
||||
"test_system_replicated_fetches/test.py::test_system_replicated_fetches",
|
||||
"test_zookeeper_config_load_balancing/test.py::test_round_robin",
|
||||
"test_zookeeper_config_load_balancing/test.py::test_az",
|
||||
"test_zookeeper_fallback_session/test.py::test_fallback_session",
|
||||
|
||||
"test_global_overcommit_tracker/test.py::test_global_overcommit",
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
-- Tags: no-parallel, no-fasttest
|
||||
-- Tags: no-parallel, no-fasttest, no-ubsan, no-batch
|
||||
-- no-parallel because we want to run this test when most of the other tests already passed
|
||||
|
||||
-- If this test fails, see the "Top patterns of log messages" diagnostics in the end of run.log
|
||||
|
||||
@ -1,2 +1,2 @@
|
||||
SET output_format_write_statistics = 0;
|
||||
SELECT 'Hello & world' AS s, 'Hello\n<World>', toDateTime('2001-02-03 04:05:06') AS time, arrayMap(x -> toString(x), range(10)) AS arr, (s, time) AS tpl SETTINGS extremes = 1 FORMAT XML;
|
||||
SELECT 'Hello & world' AS s, 'Hello\n<World>', toDateTime('2001-02-03 04:05:06') AS time, arrayMap(x -> toString(x), range(10)) AS arr, (s, time) AS tpl SETTINGS extremes = 1, enable_named_columns_in_function_tuple = 0 FORMAT XML;
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
SET output_format_write_statistics = 0;
|
||||
SET enable_named_columns_in_function_tuple = 0;
|
||||
|
||||
SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT RowBinary;
|
||||
SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT RowBinaryWithNamesAndTypes;
|
||||
SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT TabSeparatedWithNamesAndTypes;
|
||||
|
||||
@ -1,5 +1,8 @@
|
||||
|
||||
set allow_deprecated_syntax_for_merge_tree=1;
|
||||
set max_threads = 1;
|
||||
set max_insert_threads = 1;
|
||||
|
||||
drop table if exists test_ins_arr;
|
||||
create table test_ins_arr (date Date, val Array(UInt64)) engine = MergeTree(date, (date), 8192);
|
||||
insert into test_ins_arr select toDate('2017-10-02'), [number, 42] from system.numbers limit 10000;
|
||||
|
||||
@ -23,7 +23,7 @@ CREATE TABLE multiword_types (
|
||||
SHOW CREATE TABLE multiword_types;
|
||||
|
||||
INSERT INTO multiword_types(a) VALUES (1);
|
||||
SELECT toTypeName((*,)) FROM multiword_types;
|
||||
SELECT toTypeName((*,)) FROM multiword_types SETTINGS enable_named_columns_in_function_tuple = 0;
|
||||
|
||||
CREATE TABLE unsigned_types (
|
||||
a TINYINT SIGNED,
|
||||
@ -43,7 +43,7 @@ CREATE TABLE unsigned_types (
|
||||
SHOW CREATE TABLE unsigned_types;
|
||||
|
||||
INSERT INTO unsigned_types(a) VALUES (1);
|
||||
SELECT toTypeName((*,)) FROM unsigned_types;
|
||||
SELECT toTypeName((*,)) FROM unsigned_types SETTINGS enable_named_columns_in_function_tuple = 0;
|
||||
|
||||
SELECT CAST('42' AS DOUBLE PRECISION), CAST(42, 'NATIONAL CHARACTER VARYING'), CAST(-1 AS tinyint UnSiGnEd), CAST(65535, ' sMaLlInT signed ');
|
||||
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
nan
|
||||
nan
|
||||
nan
|
||||
0
|
||||
1
|
||||
0
|
||||
0.5
|
||||
1
|
||||
0.5
|
||||
0
|
||||
0.75
|
||||
1
|
||||
0.75
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
hello 1 3 world
|
||||
9
|
||||
9 (0,1)
|
||||
key tupleElement(argMax((v1, v2, v3, v4, v5), v1), \'1\') tupleElement(argMax((v1, v2, v3, v4, v5), v1), \'2\') tupleElement(argMax((v1, v2, v3, v4, v5), v1), \'3\') tupleElement(argMax((v1, v2, v3, v4, v5), v1), \'4\') tupleElement(argMax((v1, v2, v3, v4, v5), v1), \'5\')
|
||||
key tupleElement(argMax((v1, v2, v3, v4, v5), v1), \'v1\') tupleElement(argMax((v1, v2, v3, v4, v5), v1), \'v2\') tupleElement(argMax((v1, v2, v3, v4, v5), v1), \'v3\') tupleElement(argMax((v1, v2, v3, v4, v5), v1), \'v4\') tupleElement(argMax((v1, v2, v3, v4, v5), v1), \'v5\')
|
||||
1 20 20 10 20 30
|
||||
2 11 20 10 20 30
|
||||
3 70 20 10 20 30
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
SET allow_experimental_analyzer = 1;
|
||||
SET enable_named_columns_in_function_tuple = 1;
|
||||
|
||||
select untuple((* except (b),)) from (select 1 a, 2 b, 3 c);
|
||||
select 'hello', untuple((* except (b),)), 'world' from (select 1 a, 2 b, 3 c);
|
||||
|
||||
@ -72,7 +72,7 @@ SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t1.id + 2; -- { serverE
|
||||
SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.id + 2; -- { serverError 403 }
|
||||
SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t1.key; -- { serverError 43, 403 }
|
||||
SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key; -- { serverError 43, 403 }
|
||||
SELECT * FROM t1 JOIN t2 ON t2.key == t2.key2 AND (t1.id == t2.id OR isNull(t2.key2)); -- { serverError 403 }
|
||||
SELECT * FROM t1 JOIN t2_nullable as t2 ON t2.key == t2.key2 AND (t1.id == t2.id OR isNull(t2.key2)); -- { serverError 403 }
|
||||
SELECT * FROM t1 JOIN t2 ON t2.key == t2.key2 OR t1.id == t2.id; -- { serverError 403 }
|
||||
SELECT * FROM t1 JOIN t2 ON (t2.key == t2.key2 AND (t1.key == t1.key2 AND t1.key != 'XXX' OR t1.id == t2.id)) AND t1.id == t2.id; -- { serverError 403 }
|
||||
SELECT * FROM t1 JOIN t2 ON t2.key == t2.key2 AND t1.key == t1.key2 AND t1.key != 'XXX' AND t1.id == t2.id OR t2.key == t2.key2 AND t1.id == t2.id AND t1.id == t2.id;
|
||||
|
||||
@ -70,7 +70,7 @@ SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t1.id + 2; -- { serverE
|
||||
SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.id + 2; -- { serverError 403 }
|
||||
SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t1.key; -- { serverError 43, 403 }
|
||||
SELECT * FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key; -- { serverError 43, 403 }
|
||||
SELECT * FROM t1 JOIN t2 ON t2.key == t2.key2 AND (t1.id == t2.id OR isNull(t2.key2)); -- { serverError 403 }
|
||||
SELECT * FROM t1 JOIN t2_nullable as t2 ON t2.key == t2.key2 AND (t1.id == t2.id OR isNull(t2.key2)); -- { serverError 403 }
|
||||
SELECT * FROM t1 JOIN t2 ON t2.key == t2.key2 OR t1.id == t2.id; -- { serverError 403 }
|
||||
SELECT * FROM t1 JOIN t2 ON (t2.key == t2.key2 AND (t1.key == t1.key2 AND t1.key != 'XXX' OR t1.id == t2.id)) AND t1.id == t2.id; -- { serverError 403 }
|
||||
SELECT * FROM t1 JOIN t2 ON t2.key == t2.key2 AND t1.key == t1.key2 AND t1.key != 'XXX' AND t1.id == t2.id OR t2.key == t2.key2 AND t1.id == t2.id AND t1.id == t2.id; -- { serverError 48 }
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
set enable_named_columns_in_function_tuple = 0;
|
||||
|
||||
select arrayMap(x -> 2 * x, []);
|
||||
select toTypeName(arrayMap(x -> 2 * x, []));
|
||||
select arrayMap((x, y) -> x + y, [], []);
|
||||
|
||||
@ -876,7 +876,6 @@ tryBase58Decode
|
||||
tumble
|
||||
tumbleEnd
|
||||
tumbleStart
|
||||
tuple
|
||||
tupleConcat
|
||||
tupleDivide
|
||||
tupleDivideByNumber
|
||||
|
||||
@ -12,11 +12,21 @@ t
|
||||
2
|
||||
rdb_default 1 1 s1 r1 1
|
||||
2
|
||||
2
|
||||
2
|
||||
skip inactive
|
||||
s1 r1 OK 2 0
|
||||
s1 r2 QUEUED 2 0
|
||||
s2 r1 QUEUED 2 0
|
||||
s1 r1 OK 2 0
|
||||
s1 r2 QUEUED 2 0
|
||||
s2 r1 QUEUED 2 0
|
||||
timeout on active
|
||||
2
|
||||
2
|
||||
s1 r1 OK 3 0
|
||||
s1 r2 QUEUED 3 0
|
||||
s2 r1 QUEUED 3 0
|
||||
s9 r9 QUEUED 3 0
|
||||
drop replica
|
||||
2
|
||||
rdb_default 1 1 s1 r1 1
|
||||
rdb_default 1 2 s1 r2 0
|
||||
@ -24,6 +34,9 @@ rdb_default 1 2 s1 r2 0
|
||||
2
|
||||
t
|
||||
t2
|
||||
t22
|
||||
t3
|
||||
t33
|
||||
t4
|
||||
t44
|
||||
rdb_default_4 1 1 s1 r1 1
|
||||
|
||||
@ -33,10 +33,27 @@ $CLICKHOUSE_CLIENT -q "select cluster, shard_num, replica_num, database_shard_na
|
||||
$CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from database $db2" 2>&1| grep -Fac "is active, cannot drop it"
|
||||
|
||||
# Also check that it doesn't exceed distributed_ddl_task_timeout waiting for inactive replicas
|
||||
timeout 60s $CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=1000 --distributed_ddl_output_mode=none_only_active -q "create table $db.t2 (n int) engine=Log" 2>&1| grep -Fac "TIMEOUT_EXCEEDED"
|
||||
timeout 60s $CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=1000 --distributed_ddl_output_mode=throw_only_active -q "create table $db.t3 (n int) engine=Log" 2>&1| grep -Fac "TIMEOUT_EXCEEDED"
|
||||
echo 'skip inactive'
|
||||
timeout 60s $CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=1000 --distributed_ddl_output_mode=none_only_active -q "create table $db.t2 (n int) engine=Log"
|
||||
timeout 60s $CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=1000 --distributed_ddl_output_mode=throw_only_active -q "create table $db.t3 (n int) engine=Log" | sort
|
||||
timeout 60s $CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=1000 --distributed_ddl_output_mode=null_status_on_timeout_only_active -q "create table $db.t4 (n int) engine=Log" | sort
|
||||
|
||||
# And that it still throws TIMEOUT_EXCEEDED for active replicas
|
||||
echo 'timeout on active'
|
||||
db9="${db}_9"
|
||||
$CLICKHOUSE_CLIENT -q "create database $db9 engine=Replicated('/test/$CLICKHOUSE_DATABASE/rdb', 's9', 'r9')"
|
||||
$CLICKHOUSE_CLIENT -q "detach database $db9"
|
||||
$CLICKHOUSE_CLIENT -q "insert into system.zookeeper(name, path, value) values ('active', '/test/$CLICKHOUSE_DATABASE/rdb/replicas/s9|r9', '$($CLICKHOUSE_CLIENT -q "select serverUUID()")')"
|
||||
|
||||
$CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=5 --distributed_ddl_output_mode=none_only_active -q "create table $db.t22 (n int) engine=Log" 2>&1| grep -Fac "TIMEOUT_EXCEEDED"
|
||||
$CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=5 --distributed_ddl_output_mode=throw_only_active -q "create table $db.t33 (n int) engine=Log" 2>&1| grep -Fac "TIMEOUT_EXCEEDED"
|
||||
$CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=5 --distributed_ddl_output_mode=null_status_on_timeout_only_active -q "create table $db.t44 (n int) engine=Log" | sort
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "attach database $db9"
|
||||
$CLICKHOUSE_CLIENT -q "drop database $db9"
|
||||
|
||||
echo 'drop replica'
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "detach database $db3"
|
||||
$CLICKHOUSE_CLIENT -q "system drop database replica 'r1' from shard 's2' from database $db"
|
||||
$CLICKHOUSE_CLIENT -q "attach database $db3" 2>/dev/null
|
||||
|
||||
@ -9,7 +9,7 @@ SETTINGS index_granularity = 8192;
|
||||
|
||||
INSERT INTO test_tuple_element VALUES (tuple(1,2)), (tuple(NULL, 3));
|
||||
|
||||
SELECT
|
||||
SELECT
|
||||
tupleElement(tuple, 'k1', 0) fine_k1_with_0,
|
||||
tupleElement(tuple, 'k1', NULL) k1_with_null,
|
||||
tupleElement(tuple, 'k2', 0) k2_with_0,
|
||||
|
||||
@ -0,0 +1,9 @@
|
||||
Tuple(\n i Int32,\n j Int32)
|
||||
['i','j']
|
||||
Tuple(UInt8, Int32)
|
||||
['1','2']
|
||||
Tuple(\n k UInt8,\n j Int32)
|
||||
['k','j']
|
||||
Tuple(Int32, Int32, Int32, Int32)
|
||||
['1','2','3','4']
|
||||
(1,2,3)
|
||||
31
tests/queries/0_stateless/02890_named_tuple_functions.sql
Normal file
31
tests/queries/0_stateless/02890_named_tuple_functions.sql
Normal file
@ -0,0 +1,31 @@
|
||||
set enable_named_columns_in_function_tuple = 1;
|
||||
set allow_experimental_analyzer = 1;
|
||||
|
||||
drop table if exists x;
|
||||
create table x (i int, j int) engine MergeTree order by i;
|
||||
insert into x values (1, 2);
|
||||
|
||||
select toTypeName(tuple(i, j)) from x;
|
||||
select tupleNames(tuple(i, j)) from x;
|
||||
|
||||
select toTypeName(tuple(1, j)) from x;
|
||||
select tupleNames(tuple(1, j)) from x;
|
||||
|
||||
select toTypeName(tuple(1 as k, j)) from x;
|
||||
select tupleNames(tuple(1 as k, j)) from x;
|
||||
|
||||
select toTypeName(tuple(i, i, j, j)) from x;
|
||||
select tupleNames(tuple(i, i, j, j)) from x;
|
||||
|
||||
select tupleNames(1); -- { serverError 43 }
|
||||
|
||||
drop table x;
|
||||
|
||||
drop table if exists tbl;
|
||||
|
||||
-- Make sure named tuple won't break Values insert
|
||||
create table tbl (x Tuple(a Int32, b Int32, c Int32)) engine MergeTree order by ();
|
||||
insert into tbl values (tuple(1, 2, 3)); -- without tuple it's interpreted differently inside values block.
|
||||
select * from tbl;
|
||||
|
||||
drop table tbl
|
||||
@ -57,6 +57,10 @@ t.1: 1
|
||||
Row 1:
|
||||
──────
|
||||
t.1: 1
|
||||
-- tuple() with enable_named_columns_in_function_tuple = 1 and allow_experimental_analyzer = 1 keeps the column names
|
||||
Row 1:
|
||||
──────
|
||||
t.a: 1
|
||||
-- thankfully JSONExtract() keeps them
|
||||
Row 1:
|
||||
──────
|
||||
|
||||
@ -37,8 +37,11 @@ SELECT untuple(tuple(1)::Tuple(Int)), untuple(tuple(1)::Tuple(Int)) FORMAT Verti
|
||||
SELECT untuple(tuple(1)::Tuple(Int)), untuple(tuple(1)::Tuple(Int)) FORMAT Vertical SETTINGS allow_experimental_analyzer = 1; -- Bug: doesn't throw an exception
|
||||
|
||||
SELECT '-- tuple() loses the column names (would be good to fix, see #36773)';
|
||||
SELECT untuple(tuple(1 as a)) as t FORMAT Vertical SETTINGS allow_experimental_analyzer = 0;
|
||||
SELECT untuple(tuple(1 as a)) as t FORMAT Vertical SETTINGS allow_experimental_analyzer = 1;
|
||||
SELECT untuple(tuple(1 as a)) as t FORMAT Vertical SETTINGS allow_experimental_analyzer = 0, enable_named_columns_in_function_tuple = 0;
|
||||
SELECT untuple(tuple(1 as a)) as t FORMAT Vertical SETTINGS allow_experimental_analyzer = 1, enable_named_columns_in_function_tuple = 0;
|
||||
|
||||
SELECT '-- tuple() with enable_named_columns_in_function_tuple = 1 and allow_experimental_analyzer = 1 keeps the column names';
|
||||
SELECT untuple(tuple(1 as a)) as t FORMAT Vertical SETTINGS allow_experimental_analyzer = 1, enable_named_columns_in_function_tuple = 1;
|
||||
|
||||
SELECT '-- thankfully JSONExtract() keeps them';
|
||||
SELECT untuple(JSONExtract('{"key": "value"}', 'Tuple(key String)')) x FORMAT Vertical SETTINGS allow_experimental_analyzer = 0;
|
||||
|
||||
@ -3,7 +3,7 @@ SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR (t1.x IS NULL AND t2.x IS NULL)) O
|
||||
2 2 2 2
|
||||
3 3 3 33
|
||||
\N \N \N \N
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR t1.x IS NULL AND t1.y <=> t2.y AND t2.x IS NULL) ORDER BY t1.x NULLS LAST;
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR t1.x IS NULL AND t2.x IS NULL) OR t1.y <=> t2.y ORDER BY t1.x NULLS LAST;
|
||||
1 42 4 42
|
||||
2 2 2 2
|
||||
3 3 3 33
|
||||
@ -12,14 +12,14 @@ SELECT * FROM t1 JOIN t2 ON (t1.x = t2.x OR t1.x IS NULL AND t2.x IS NULL) ORDER
|
||||
2 2 2 2
|
||||
3 3 3 33
|
||||
\N \N \N \N
|
||||
SELECT * FROM t1 JOIN t2 ON t1.x <=> t2.x AND (t1.x = t1.y OR t1.x IS NULL AND t1.y IS NULL) ORDER BY t1.x;
|
||||
SELECT * FROM t1 JOIN t2 ON t1.x <=> t2.x AND ((t1.x = t1.y) OR t1.x IS NULL AND t1.y IS NULL) ORDER BY t1.x;
|
||||
2 2 2 2
|
||||
3 3 3 33
|
||||
\N \N \N \N
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x = t2.x OR t1.x IS NULL AND t2.x IS NULL) AND t1.y <=> t2.y ORDER BY t1.x NULLS LAST;
|
||||
2 2 2 2
|
||||
\N \N \N \N
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR t1.y <=> t2.y OR (t1.x IS NULL AND t1.y IS NULL AND t2.x IS NULL AND t2.y IS NULL)) ORDER BY t1.x NULLS LAST;
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR t1.y <=> t2.y OR (t1.x IS NULL AND t2.x IS NULL) OR (t1.y IS NULL AND t2.y IS NULL)) ORDER BY t1.x NULLS LAST;
|
||||
1 42 4 42
|
||||
2 2 2 2
|
||||
3 3 3 33
|
||||
@ -31,3 +31,30 @@ SELECT x = y OR (x IS NULL AND y IS NULL) FROM t1 ORDER BY x NULLS LAST;
|
||||
1
|
||||
1
|
||||
1
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST;
|
||||
2 2 2 2
|
||||
3 3 3 33
|
||||
\N \N \N \N
|
||||
-- aliases defined in the join condition are valid
|
||||
SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST;
|
||||
1 42 \N \N \N 0
|
||||
2 2 2 2 1 1
|
||||
3 3 3 33 1 1
|
||||
\N \N 4 42 \N 0
|
||||
\N \N \N \N \N 1
|
||||
SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST;
|
||||
1 42 \N \N \N 0
|
||||
2 2 2 2 1 1
|
||||
3 3 3 33 1 1
|
||||
\N \N 4 42 \N 0
|
||||
\N \N \N \N \N 0
|
||||
\N \N \N \N \N 0
|
||||
-- check for non-nullable columns for which `is null` is replaced with constant
|
||||
SELECT * FROM t1n as t1 JOIN t2n as t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST;
|
||||
2 2 2 2
|
||||
3 3 3 33
|
||||
--
|
||||
0
|
||||
0
|
||||
2
|
||||
2
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
DROP TABLE IF EXISTS t1;
|
||||
DROP TABLE IF EXISTS t2;
|
||||
DROP TABLE IF EXISTS t1n;
|
||||
DROP TABLE IF EXISTS t2n;
|
||||
|
||||
CREATE TABLE t1 (x Nullable(Int64), y Nullable(UInt64)) ENGINE = TinyLog;
|
||||
CREATE TABLE t2 (x Nullable(Int64), y Nullable(UInt64)) ENGINE = TinyLog;
|
||||
@ -7,24 +9,62 @@ CREATE TABLE t2 (x Nullable(Int64), y Nullable(UInt64)) ENGINE = TinyLog;
|
||||
INSERT INTO t1 VALUES (1,42), (2,2), (3,3), (NULL,NULL);
|
||||
INSERT INTO t2 VALUES (NULL,NULL), (2,2), (3,33), (4,42);
|
||||
|
||||
CREATE TABLE t1n (x Int64, y UInt64) ENGINE = TinyLog;
|
||||
CREATE TABLE t2n (x Int64, y UInt64) ENGINE = TinyLog;
|
||||
|
||||
INSERT INTO t1n VALUES (1,42), (2,2), (3,3);
|
||||
INSERT INTO t2n VALUES (2,2), (3,33), (4,42);
|
||||
|
||||
SET allow_experimental_analyzer = 1;
|
||||
|
||||
-- { echoOn }
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR (t1.x IS NULL AND t2.x IS NULL)) ORDER BY t1.x NULLS LAST;
|
||||
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR t1.x IS NULL AND t1.y <=> t2.y AND t2.x IS NULL) ORDER BY t1.x NULLS LAST;
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR t1.x IS NULL AND t2.x IS NULL) OR t1.y <=> t2.y ORDER BY t1.x NULLS LAST;
|
||||
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x = t2.x OR t1.x IS NULL AND t2.x IS NULL) ORDER BY t1.x;
|
||||
SELECT * FROM t1 JOIN t2 ON t1.x <=> t2.x AND (t1.x = t1.y OR t1.x IS NULL AND t1.y IS NULL) ORDER BY t1.x;
|
||||
SELECT * FROM t1 JOIN t2 ON t1.x <=> t2.x AND ((t1.x = t1.y) OR t1.x IS NULL AND t1.y IS NULL) ORDER BY t1.x;
|
||||
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x = t2.x OR t1.x IS NULL AND t2.x IS NULL) AND t1.y <=> t2.y ORDER BY t1.x NULLS LAST;
|
||||
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR t1.y <=> t2.y OR (t1.x IS NULL AND t1.y IS NULL AND t2.x IS NULL AND t2.y IS NULL)) ORDER BY t1.x NULLS LAST;
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR t1.y <=> t2.y OR (t1.x IS NULL AND t2.x IS NULL) OR (t1.y IS NULL AND t2.y IS NULL)) ORDER BY t1.x NULLS LAST;
|
||||
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR (t1.x IS NULL AND t2.x IS NULL)) AND (t1.y == t2.y OR (t1.y IS NULL AND t2.y IS NULL)) AND COALESCE(t1.x, 0) != 2 ORDER BY t1.x NULLS LAST;
|
||||
|
||||
SELECT x = y OR (x IS NULL AND y IS NULL) FROM t1 ORDER BY x NULLS LAST;
|
||||
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST;
|
||||
|
||||
-- aliases defined in the join condition are valid
|
||||
SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST;
|
||||
SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST;
|
||||
|
||||
-- check for non-nullable columns for which `is null` is replaced with constant
|
||||
SELECT * FROM t1n as t1 JOIN t2n as t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST;
|
||||
|
||||
-- { echoOff }
|
||||
|
||||
SELECT '--';
|
||||
|
||||
-- IS NOT NULL and constants are optimized out
|
||||
SELECT count() FROM ( EXPLAIN QUERY TREE
|
||||
SELECT * FROM t1 JOIN t2 ON ( (t1.x = t2.x) AND (t1.x IS NOT NULL) AND true AND (t2.x IS NOT NULL) )
|
||||
) WHERE explain like '%CONSTANT%' OR explain ilike '%is%null%';
|
||||
|
||||
SELECT count() FROM ( EXPLAIN QUERY TREE
|
||||
SELECT * FROM t1 JOIN t2 ON ( (t1.x = t2.x) AND true )
|
||||
) WHERE explain like '%CONSTANT%' OR explain ilike '%is%null%';
|
||||
|
||||
-- this is not optimized out
|
||||
SELECT count() FROM ( EXPLAIN QUERY TREE
|
||||
SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR t1.x IS NULL AND t1.y <=> t2.y AND t2.x IS NULL)
|
||||
) WHERE explain like '%CONSTANT%' OR explain ilike '%is%null%';
|
||||
|
||||
SELECT count() FROM ( EXPLAIN QUERY TREE
|
||||
SELECT * FROM t1 JOIN t2 ON t1.x <=> t2.x AND (t1.x = t1.y OR t1.x IS NULL AND t1.y IS NULL)
|
||||
) WHERE explain like '%CONSTANT%' OR explain ilike '%is%null%';
|
||||
|
||||
DROP TABLE IF EXISTS t1;
|
||||
DROP TABLE IF EXISTS t2;
|
||||
DROP TABLE IF EXISTS t1n;
|
||||
DROP TABLE IF EXISTS t2n;
|
||||
|
||||
@ -7,7 +7,7 @@ CLICKHOUSE_LOG_COMMENT=
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
|
||||
CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --enable_named_columns_in_function_tuple=0"
|
||||
|
||||
|
||||
function test()
|
||||
|
||||
@ -0,0 +1,6 @@
|
||||
1
|
||||
0
|
||||
1
|
||||
0
|
||||
0
|
||||
0
|
||||
@ -0,0 +1,45 @@
|
||||
DROP TABLE IF EXISTS t_read_in_order_1;
|
||||
|
||||
CREATE TABLE t_read_in_order_1 (id UInt64, v UInt64)
|
||||
ENGINE = MergeTree ORDER BY id
|
||||
SETTINGS index_granularity = 1024, index_granularity_bytes = '10M';
|
||||
|
||||
INSERT INTO t_read_in_order_1 SELECT number, number FROM numbers(1000000);
|
||||
|
||||
SET max_threads = 8;
|
||||
SET optimize_read_in_order = 1;
|
||||
SET read_in_order_use_buffering = 1;
|
||||
|
||||
SELECT count() FROM
|
||||
(
|
||||
EXPLAIN PIPELINE SELECT * FROM t_read_in_order_1 ORDER BY id
|
||||
) WHERE explain LIKE '%BufferChunks%';
|
||||
|
||||
SELECT count() FROM
|
||||
(
|
||||
EXPLAIN PIPELINE SELECT * FROM t_read_in_order_1 ORDER BY id LIMIT 10
|
||||
) WHERE explain LIKE '%BufferChunks%';
|
||||
|
||||
SELECT count() FROM
|
||||
(
|
||||
EXPLAIN PIPELINE SELECT * FROM t_read_in_order_1 WHERE v % 10 = 0 ORDER BY id LIMIT 10
|
||||
) WHERE explain LIKE '%BufferChunks%';
|
||||
|
||||
SET read_in_order_use_buffering = 0;
|
||||
|
||||
SELECT count() FROM
|
||||
(
|
||||
EXPLAIN PIPELINE SELECT * FROM t_read_in_order_1 ORDER BY id
|
||||
) WHERE explain LIKE '%BufferChunks%';
|
||||
|
||||
SELECT count() FROM
|
||||
(
|
||||
EXPLAIN PIPELINE SELECT * FROM t_read_in_order_1 ORDER BY id LIMIT 10
|
||||
) WHERE explain LIKE '%BufferChunks%';
|
||||
|
||||
SELECT count() FROM
|
||||
(
|
||||
EXPLAIN PIPELINE SELECT * FROM t_read_in_order_1 WHERE v % 10 = 0 ORDER BY id LIMIT 10
|
||||
) WHERE explain LIKE '%BufferChunks%';
|
||||
|
||||
DROP TABLE t_read_in_order_1;
|
||||
@ -0,0 +1,17 @@
|
||||
-- Tags: long, no-random-settings, no-tsan, no-asan, no-msan, no-s3-storage
|
||||
|
||||
DROP TABLE IF EXISTS t_read_in_order_2;
|
||||
|
||||
CREATE TABLE t_read_in_order_2 (id UInt64, v UInt64) ENGINE = MergeTree ORDER BY id;
|
||||
|
||||
INSERT INTO t_read_in_order_2 SELECT number, number FROM numbers(10000000);
|
||||
OPTIMIZE TABLE t_read_in_order_2 FINAL;
|
||||
|
||||
SET optimize_read_in_order = 1;
|
||||
SET max_threads = 4;
|
||||
SET read_in_order_use_buffering = 1;
|
||||
SET max_memory_usage = '100M';
|
||||
|
||||
SELECT * FROM t_read_in_order_2 ORDER BY id FORMAT Null;
|
||||
|
||||
DROP TABLE t_read_in_order_2;
|
||||
@ -0,0 +1,10 @@
|
||||
a1451105-722e-4fe7-bfaa-65ad2ae249c2 whatever
|
||||
a1451105-722e-4fe7-bfaa-65ad2ae249c2 whatever
|
||||
---------------------------
|
||||
a1451105-722e-4fe7-bfaa-65ad2ae249c2
|
||||
a1451105-722e-4fe7-bfaa-65ad2ae249c2
|
||||
---------------------------
|
||||
a1451105-722e-4fe7-bfaa-65ad2ae249c2
|
||||
---------------------------
|
||||
a1451105-722e-4fe7-bfaa-65ad2ae249c2
|
||||
a1451105-722e-4fe7-bfaa-65ad2ae249c2
|
||||
83
tests/queries/0_stateless/03173_parallel_replicas_join_bug.sh
Executable file
83
tests/queries/0_stateless/03173_parallel_replicas_join_bug.sh
Executable file
@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
|
||||
$CLICKHOUSE_CLIENT -nq "
|
||||
CREATE TABLE ids (id UUID, whatever String) Engine=MergeTree ORDER BY tuple();
|
||||
INSERT INTO ids VALUES ('a1451105-722e-4fe7-bfaa-65ad2ae249c2', 'whatever');
|
||||
|
||||
CREATE TABLE data (id UUID, event_time DateTime, status String) Engine=MergeTree ORDER BY tuple();
|
||||
INSERT INTO data VALUES ('a1451105-722e-4fe7-bfaa-65ad2ae249c2', '2000-01-01', 'CREATED');
|
||||
|
||||
CREATE TABLE data2 (id UUID, event_time DateTime, status String) Engine=MergeTree ORDER BY tuple();
|
||||
INSERT INTO data2 VALUES ('a1451105-722e-4fe7-bfaa-65ad2ae249c2', '2000-01-02', 'CREATED');
|
||||
"
|
||||
|
||||
$CLICKHOUSE_CLIENT -nq "
|
||||
SET allow_experimental_analyzer = 1, cluster_for_parallel_replicas = 'parallel_replicas', max_parallel_replicas = 10, allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, max_threads = 1;
|
||||
|
||||
SELECT
|
||||
id,
|
||||
whatever
|
||||
FROM ids AS l
|
||||
INNER JOIN view(
|
||||
SELECT *
|
||||
FROM merge($CLICKHOUSE_DATABASE, 'data.*')
|
||||
) AS s ON l.id = s.id
|
||||
WHERE status IN ['CREATED', 'CREATING']
|
||||
ORDER BY event_time DESC;
|
||||
|
||||
SELECT '---------------------------';
|
||||
|
||||
with
|
||||
results1 as (
|
||||
SELECT id
|
||||
FROM data t1
|
||||
inner join ids t2
|
||||
on t1.id = t2.id
|
||||
),
|
||||
results2 as (
|
||||
SELECT id
|
||||
FROM ids t1
|
||||
inner join data t2
|
||||
on t1.id = t2.id
|
||||
)
|
||||
select * from results1 union all select * from results2;
|
||||
|
||||
SELECT '---------------------------';
|
||||
|
||||
with
|
||||
results1 as (
|
||||
SELECT id
|
||||
FROM data t1
|
||||
inner join ids t2
|
||||
on t1.id = t2.id
|
||||
),
|
||||
results2 as (
|
||||
SELECT id
|
||||
FROM ids t1
|
||||
inner join data t2
|
||||
on t1.id = t2.id
|
||||
)
|
||||
select * from results1 t1 inner join results2 t2 using (id);
|
||||
|
||||
SELECT '---------------------------';
|
||||
|
||||
with
|
||||
results1 as (
|
||||
SELECT t1.id
|
||||
FROM data t1
|
||||
inner join ids t2 on t1.id = t2.id
|
||||
left join data t3 on t2.id = t3.id
|
||||
),
|
||||
results2 as (
|
||||
SELECT id
|
||||
FROM ids t1
|
||||
inner join data t2
|
||||
on t1.id = t2.id
|
||||
)
|
||||
select * from results1 union all select * from results2;
|
||||
"
|
||||
@ -0,0 +1 @@
|
||||
1 2024-06-30 20:00:00.000
|
||||
12
tests/queries/0_stateless/03198_orc_read_time_zone.sh
Executable file
12
tests/queries/0_stateless/03198_orc_read_time_zone.sh
Executable file
@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "drop table if exists test"
|
||||
$CLICKHOUSE_CLIENT -q "create table test(id UInt64, t DateTime64) Engine=MergeTree order by id"
|
||||
$CLICKHOUSE_CLIENT -q "insert into test from infile '$CURDIR/data_orc/test_reader_time_zone.snappy.orc' SETTINGS input_format_orc_read_use_writer_time_zone=true FORMAT ORC"
|
||||
$CLICKHOUSE_CLIENT -q "select * from test SETTINGS session_timezone='Asia/Shanghai'"
|
||||
$CLICKHOUSE_CLIENT -q "drop table test"
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user