thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into pr-local-plan

Browse Source
This commit is contained in:
Igor Nikonov 2024-08-16 11:07:35 +00:00
commit 9dc288faa8
134 changed files with 838 additions and 463 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
contrib/usearch-cmake/CMakeLists.txt
View File

@ -9,4 +9,14 @@ target_include_directories(_usearch SYSTEM INTERFACE
${SIMSIMD_PROJECT_DIR}/include
${USEARCH_PROJECT_DIR}/include)
target_compile_definitions(_usearch INTERFACE USEARCH_USE_FP16LIB)
# target_compile_definitions(_usearch INTERFACE USEARCH_USE_SIMSIMD)
# ^^ simsimd is not enabled at the moment. Reasons:
# - Vectorization is important for raw scans but not so much for HNSW. We use usearch only for HNSW.
# - Simsimd does compile-time dispatch (choice of SIMD kernels determined by capabilities of the build machine) or dynamic dispatch (SIMD
# kernels chosen at runtime based on cpuid instruction). Since current builds are limited to SSE 4.2 (x86) and NEON (ARM), the speedup of
# the former would be moderate compared to AVX-512 / SVE. The latter is at the moment too fragile with respect to portability across x86
# and ARM machines ... certain conbinations of quantizations / distance functions / SIMD instructions are not implemented at the moment.
add_library(ch_contrib::usearch ALIAS _usearch)

7
docker/test/upgrade/run.sh
View File

@ -129,6 +129,7 @@ configure
# Check that all new/changed setting were added in settings changes history.
# Some settings can be different for builds with sanitizers, so we check
# Also the automatic value of 'max_threads' and similar was displayed as "'auto(...)'" in previous versions instead of "auto(...)".
# settings changes only for non-sanitizer builds.
IS_SANITIZED=$(clickhouse-local --query "SELECT value LIKE '%-fsanitize=%' FROM system.build_options WHERE name = 'CXX_FLAGS'")
if [ "${IS_SANITIZED}" -eq "0" ]
@ -145,7 +146,9 @@ then
old_settings.value AS old_value
FROM new_settings
LEFT JOIN old_settings ON new_settings.name = old_settings.name
WHERE (new_settings.value != old_settings.value) AND (name NOT IN (
WHERE (new_value != old_value)
AND NOT (startsWith(new_value, 'auto(') AND old_value LIKE '%auto(%')
AND (name NOT IN (
SELECT arrayJoin(tupleElement(changes, 'name'))
FROM
(
@ -177,7 +180,7 @@ then
if [ -s changed_settings.txt ]
then
mv changed_settings.txt /test_output/
echo -e "Changed settings are not reflected in settings changes history (see changed_settings.txt)$FAIL$(head_escaped /test_output/changed_settings.txt)" >> /test_output/test_results.tsv
echo -e "Changed settings are not reflected in the settings changes history (see changed_settings.txt)$FAIL$(head_escaped /test_output/changed_settings.txt)" >> /test_output/test_results.tsv
else
echo -e "There are no changed settings or they are reflected in settings changes history$OK" >> /test_output/test_results.tsv
fi

8
docs/en/engines/table-engines/mergetree-family/annindexes.md
View File

@ -22,10 +22,10 @@ ORDER BY Distance(vectors, Point)
LIMIT N
```
`vectors` contains N-dimensional values of type [Array(Float32)](../../../sql-reference/data-types/array.md), for example embeddings.
Function `Distance` computes the distance between two vectors. Often, the Euclidean (L2) distance is chosen as distance function but [other
distance functions](/docs/en/sql-reference/functions/distance-functions.md) are also possible. `Point` is the reference point, e.g. `(0.17,
0.33, ...)`, and `N` limits the number of search results.
`vectors` contains N-dimensional values of type [Array(Float32)](../../../sql-reference/data-types/array.md) or Array(Float64), for example
embeddings. Function `Distance` computes the distance between two vectors. Often, the Euclidean (L2) distance is chosen as distance function
but [other distance functions](/docs/en/sql-reference/functions/distance-functions.md) are also possible. `Point` is the reference point,
e.g. `(0.17, 0.33, ...)`, and `N` limits the number of search results.
This query returns the top-`N` closest points to the reference point. Parameter `N` limits the number of returned values which is useful for
situations where `MaxDistance` is difficult to determine in advance.

2
src/Access/AccessBackup.cpp
View File

@ -93,7 +93,7 @@ namespace
break;
}
UUID id = parse<UUID>(line);
UUID id = parse<UUID>(line.substr(0, line.find('\t')));
line.clear();
String queries;

3
src/Analyzer/FunctionNode.cpp
View File

@ -242,7 +242,8 @@ ASTPtr FunctionNode::toASTImpl(const ConvertToASTOptions & options) const
/// Avoid cast for `IN tuple(...)` expression.
/// Tuples could be quite big, and adding a type may significantly increase query size.
/// It should be safe because set type for `column IN tuple` is deduced from `column` type.
if (isNameOfInFunction(function_name) && argument_nodes.size() > 1 && argument_nodes[1]->getNodeType() == QueryTreeNodeType::CONSTANT)
if (isNameOfInFunction(function_name) && argument_nodes.size() > 1 && argument_nodes[1]->getNodeType() == QueryTreeNodeType::CONSTANT
&& !static_cast<const ConstantNode *>(argument_nodes[1].get())->hasSourceExpression())
new_options.add_cast_for_constants = false;
const auto & parameters = getParameters();

4
src/Columns/ColumnLowCardinality.h
View File

@ -46,8 +46,8 @@ public:
return Base::create(std::move(column_unique), std::move(indexes), is_shared);
}
std::string getName() const override { return "ColumnLowCardinality"; }
const char * getFamilyName() const override { return "ColumnLowCardinality"; }
std::string getName() const override { return "LowCardinality(" + getDictionary().getNestedColumn()->getName() + ")"; }
const char * getFamilyName() const override { return "LowCardinality"; }
TypeIndex getDataType() const override { return TypeIndex::LowCardinality; }
ColumnPtr convertToFullColumn() const { return getDictionary().getNestedColumn()->index(getIndexes(), 0); }

2
src/Columns/ColumnUnique.h
View File

@ -48,6 +48,8 @@ private:
ColumnUnique(const ColumnUnique & other);
public:
std::string getName() const override { return "Unique(" + getNestedColumn()->getName() + ")"; }
MutableColumnPtr cloneEmpty() const override;
const ColumnPtr & getNestedColumn() const override;

2
src/Columns/IColumnUnique.h
View File

@ -73,7 +73,7 @@ public:
/// Returns dictionary hash which is SipHash is applied to each row of nested column.
virtual UInt128 getHash() const = 0;
const char * getFamilyName() const override { return "ColumnUnique"; }
const char * getFamilyName() const override { return "Unique"; }
TypeIndex getDataType() const override { return getNestedColumn()->getDataType(); }
void insert(const Field &) override

2
src/Columns/tests/gtest_column_dump_structure.cpp
View File

@ -10,7 +10,7 @@ TEST(IColumn, dumpStructure)
{
auto type_lc = std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>());
ColumnPtr column_lc = type_lc->createColumn();
String expected_structure = "ColumnLowCardinality(size = 0, UInt8(size = 0), ColumnUnique(size = 1, String(size = 1)))";
String expected_structure = "LowCardinality(size = 0, UInt8(size = 0), Unique(size = 1, String(size = 1)))";
std::vector<std::thread> threads;
for (size_t i = 0; i < 6; ++i)

20
src/Common/QueryProfiler.cpp
View File

@ -110,7 +110,7 @@ namespace
errno = saved_errno;
}
[[maybe_unused]] constexpr UInt32 TIMER_PRECISION = 1e9;
[[maybe_unused]] constexpr UInt64 TIMER_PRECISION = 1e9;
}
namespace ErrorCodes
@ -167,18 +167,18 @@ void Timer::createIfNecessary(UInt64 thread_id, int clock_type, int pause_signal
}
}
void Timer::set(UInt32 period)
void Timer::set(UInt64 period)
{
/// Too high frequency can introduce infinite busy loop of signal handlers. We will limit maximum frequency (with 1000 signals per second).
period = std::max<UInt32>(period, 1000000);
period = std::max<UInt64>(period, 1000000);
/// Randomize offset as uniform random value from 0 to period - 1.
/// It will allow to sample short queries even if timer period is large.
/// (For example, with period of 1 second, query with 50 ms duration will be sampled with 1 / 20 probability).
/// It also helps to avoid interference (moire).
UInt32 period_rand = std::uniform_int_distribution<UInt32>(0, period)(thread_local_rng);
UInt64 period_rand = std::uniform_int_distribution<UInt64>(0, period)(thread_local_rng);
struct timespec interval{.tv_sec = period / TIMER_PRECISION, .tv_nsec = period % TIMER_PRECISION};
struct timespec offset{.tv_sec = period_rand / TIMER_PRECISION, .tv_nsec = period_rand % TIMER_PRECISION};
struct timespec interval{.tv_sec = time_t(period / TIMER_PRECISION), .tv_nsec = int64_t(period % TIMER_PRECISION)};
struct timespec offset{.tv_sec = time_t(period_rand / TIMER_PRECISION), .tv_nsec = int64_t(period_rand % TIMER_PRECISION)};
struct itimerspec timer_spec = {.it_interval = interval, .it_value = offset};
if (timer_settime(*timer_id, 0, &timer_spec, nullptr))
@ -229,7 +229,7 @@ void Timer::cleanup()
template <typename ProfilerImpl>
QueryProfilerBase<ProfilerImpl>::QueryProfilerBase(
[[maybe_unused]] UInt64 thread_id, [[maybe_unused]] int clock_type, [[maybe_unused]] UInt32 period, [[maybe_unused]] int pause_signal_)
[[maybe_unused]] UInt64 thread_id, [[maybe_unused]] int clock_type, [[maybe_unused]] UInt64 period, [[maybe_unused]] int pause_signal_)
: log(getLogger("QueryProfiler")), pause_signal(pause_signal_)
{
#if defined(SANITIZER)
@ -270,7 +270,7 @@ QueryProfilerBase<ProfilerImpl>::QueryProfilerBase(
template <typename ProfilerImpl>
void QueryProfilerBase<ProfilerImpl>::setPeriod([[maybe_unused]] UInt32 period_)
void QueryProfilerBase<ProfilerImpl>::setPeriod([[maybe_unused]] UInt64 period_)
{
#if defined(SANITIZER)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "QueryProfiler disabled because they cannot work under sanitizers");
@ -307,7 +307,7 @@ void QueryProfilerBase<ProfilerImpl>::cleanup()
template class QueryProfilerBase<QueryProfilerReal>;
template class QueryProfilerBase<QueryProfilerCPU>;
QueryProfilerReal::QueryProfilerReal(UInt64 thread_id, UInt32 period)
QueryProfilerReal::QueryProfilerReal(UInt64 thread_id, UInt64 period)
: QueryProfilerBase(thread_id, CLOCK_MONOTONIC, period, SIGUSR1)
{}
@ -320,7 +320,7 @@ void QueryProfilerReal::signalHandler(int sig, siginfo_t * info, void * context)
writeTraceInfo(TraceType::Real, sig, info, context);
}
QueryProfilerCPU::QueryProfilerCPU(UInt64 thread_id, UInt32 period)
QueryProfilerCPU::QueryProfilerCPU(UInt64 thread_id, UInt64 period)
: QueryProfilerBase(thread_id, CLOCK_THREAD_CPUTIME_ID, period, SIGUSR2)
{}

10
src/Common/QueryProfiler.h
View File

@ -40,7 +40,7 @@ public:
~Timer();
void createIfNecessary(UInt64 thread_id, int clock_type, int pause_signal);
void set(UInt32 period);
void set(UInt64 period);
void stop();
void cleanup();
@ -54,10 +54,10 @@ template <typename ProfilerImpl>
class QueryProfilerBase
{
public:
QueryProfilerBase(UInt64 thread_id, int clock_type, UInt32 period, int pause_signal_);
QueryProfilerBase(UInt64 thread_id, int clock_type, UInt64 period, int pause_signal_);
~QueryProfilerBase();
void setPeriod(UInt32 period_);
void setPeriod(UInt64 period_);
private:
void cleanup();
@ -76,7 +76,7 @@ private:
class QueryProfilerReal : public QueryProfilerBase<QueryProfilerReal>
{
public:
QueryProfilerReal(UInt64 thread_id, UInt32 period); /// NOLINT
QueryProfilerReal(UInt64 thread_id, UInt64 period); /// NOLINT
static void signalHandler(int sig, siginfo_t * info, void * context);
};
@ -85,7 +85,7 @@ public:
class QueryProfilerCPU : public QueryProfilerBase<QueryProfilerCPU>
{
public:
QueryProfilerCPU(UInt64 thread_id, UInt32 period); /// NOLINT
QueryProfilerCPU(UInt64 thread_id, UInt64 period); /// NOLINT
static void signalHandler(int sig, siginfo_t * info, void * context);
};

2
src/Common/ZooKeeper/ZooKeeper.h
View File

@ -44,7 +44,7 @@ namespace ErrorCodes
namespace zkutil
{
/// Preferred size of multi() command (in number of ops)
/// Preferred size of multi command (in the number of operations)
constexpr size_t MULTI_BATCH_SIZE = 100;
struct ShuffleHost

15
src/Common/parseRemoteDescription.cpp
View File

@ -79,11 +79,16 @@ std::vector<String> parseRemoteDescription(
/// Look for the corresponding closing bracket
for (m = i + 1; m < r; ++m)
{
if (description[m] == '{') ++cnt;
if (description[m] == '}') --cnt;
if (description[m] == '.' && description[m-1] == '.') last_dot = m;
if (description[m] == separator) have_splitter = true;
if (cnt == 0) break;
if (description[m] == '{')
++cnt;
if (description[m] == '}')
--cnt;
if (description[m] == '.' && description[m-1] == '.')
last_dot = m;
if (description[m] == separator)
have_splitter = true;
if (cnt == 0)
break;
}
if (cnt != 0)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table function '{}': incorrect brace sequence in first argument", func_name);

2
src/Coordination/KeeperSnapshotManager.cpp
View File

@ -54,7 +54,7 @@ namespace
std::filesystem::path path(snapshot_path);
std::string filename = path.stem();
Strings name_parts;
splitInto<'_'>(name_parts, filename);
splitInto<'_', '.'>(name_parts, filename);
return parse<uint64_t>(name_parts[1]);
}

8
src/Coordination/RaftServerConfig.cpp
View File

@ -26,12 +26,16 @@ std::optional<RaftServerConfig> RaftServerConfig::parse(std::string_view server)
if (!with_id_endpoint && !with_server_type && !with_priority)
return std::nullopt;
const std::string_view id_str = parts[0];
std::string_view id_str = parts[0];
if (!id_str.starts_with("server."))
return std::nullopt;
id_str = id_str.substr(7);
if (auto eq_pos = id_str.find('='); std::string_view::npos != eq_pos)
id_str = id_str.substr(0, eq_pos);
Int32 id;
if (!tryParse(id, std::next(id_str.begin(), 7)))
if (!tryParse(id, id_str))
return std::nullopt;
if (id <= 0)
return std::nullopt;

2
src/Core/MySQL/MySQLGtid.cpp
View File

@ -24,9 +24,7 @@ void GTIDSet::tryMerge(size_t i)
void GTIDSets::parse(String gtid_format)
{
if (gtid_format.empty())
{
return;
}
std::vector<String> gtid_sets;
boost::split(gtid_sets, gtid_format, [](char c) { return c == ','; });

17
src/Core/MySQL/tests/gtest_MySQLGtid.cpp
View File

@ -10,20 +10,19 @@ GTEST_TEST(GTIDSetsContains, Tests)
contained1, contained2, contained3, contained4, contained5,
not_contained1, not_contained2, not_contained3, not_contained4, not_contained5, not_contained6;
gtid_set.parse("2174B383-5441-11E8-B90A-C80AA9429562:1-3:11:47-49, 24DA167-0C0C-11E8-8442-00059A3C7B00:1-19:47-49:60");
contained1.parse("2174B383-5441-11E8-B90A-C80AA9429562:1-3:11:47-49, 24DA167-0C0C-11E8-8442-00059A3C7B00:1-19:47-49:60");
gtid_set.parse("2174B383-5441-11E8-B90A-C80AA9429562:1-3:11:47-49, FBC30C64-F8C9-4DDF-8CDD-066208EB433B:1-19:47-49:60");
contained1.parse("2174B383-5441-11E8-B90A-C80AA9429562:1-3:11:47-49, FBC30C64-F8C9-4DDF-8CDD-066208EB433B:1-19:47-49:60");
contained2.parse("2174B383-5441-11E8-B90A-C80AA9429562:2-3:11:47-49");
contained3.parse("2174B383-5441-11E8-B90A-C80AA9429562:11");
contained4.parse("24DA167-0C0C-11E8-8442-00059A3C7B00:2-16:47-49:60");
contained5.parse("24DA167-0C0C-11E8-8442-00059A3C7B00:60");
contained4.parse("FBC30C64-F8C9-4DDF-8CDD-066208EB433B:2-16:47-49:60");
contained5.parse("FBC30C64-F8C9-4DDF-8CDD-066208EB433B:60");
not_contained1.parse("2174B383-5441-11E8-B90A-C80AA9429562:1-3:11:47-50, 24DA167-0C0C-11E8-8442-00059A3C7B00:1-19:47-49:60");
not_contained1.parse("2174B383-5441-11E8-B90A-C80AA9429562:1-3:11:47-50, FBC30C64-F8C9-4DDF-8CDD-066208EB433B:1-19:47-49:60");
not_contained2.parse("2174B383-5441-11E8-B90A-C80AA9429562:0-3:11:47-49");
not_contained3.parse("2174B383-5441-11E8-B90A-C80AA9429562:99");
not_contained4.parse("24DA167-0C0C-11E8-8442-00059A3C7B00:2-16:46-49:60");
not_contained5.parse("24DA167-0C0C-11E8-8442-00059A3C7B00:99");
not_contained6.parse("2174B383-5441-11E8-B90A-C80AA9429562:1-3:11:47-49, 24DA167-0C0C-11E8-8442-00059A3C7B00:1-19:47-49:60, 00000000-0000-0000-0000-000000000000");
not_contained4.parse("FBC30C64-F8C9-4DDF-8CDD-066208EB433B:2-16:46-49:60");
not_contained5.parse("FBC30C64-F8C9-4DDF-8CDD-066208EB433B:99");
not_contained6.parse("2174B383-5441-11E8-B90A-C80AA9429562:1-3:11:47-49, FBC30C64-F8C9-4DDF-8CDD-066208EB433B:1-19:47-49:60, 00000000-0000-0000-0000-000000000000");
ASSERT_TRUE(gtid_set.contains(contained1));
ASSERT_TRUE(gtid_set.contains(contained2));

2
src/Core/SettingsFields.cpp
View File

@ -237,7 +237,7 @@ SettingFieldMaxThreads & SettingFieldMaxThreads::operator=(const Field & f)
String SettingFieldMaxThreads::toString() const
{
if (is_auto)
return "'auto(" + ::DB::toString(value) + ")'";
return "auto(" + ::DB::toString(value) + ")";
else
return ::DB::toString(value);
}

2
src/Core/SettingsFields.h
View File

@ -153,7 +153,7 @@ struct SettingFieldMaxThreads
operator UInt64() const { return value; } /// NOLINT
explicit operator Field() const { return value; }
/// Writes "auto(<number>)" instead of simple "<number>" if `is_auto==true`.
/// Writes "auto(<number>)" instead of simple "<number>" if `is_auto == true`.
String toString() const;
void parseFromString(const String & str);

1
src/Databases/TablesLoader.h
View File

@ -1,4 +1,5 @@
#pragma once
#include <map>
#include <mutex>
#include <unordered_map>

2
src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
View File

@ -1031,7 +1031,7 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep()
}
else
LOG_TRACE(log, "No space left in cache to reserve {} bytes, reason: {}, "
"will continue without cache download", failure_reason, size);
"will continue without cache download", size, failure_reason);
if (!success)
{

19
src/IO/ReadHelpers.h
View File

@ -258,6 +258,20 @@ inline void readBoolText(bool & x, ReadBuffer & buf)
char tmp = '0';
readChar(tmp, buf);
x = tmp != '0';
if (!buf.eof() && isAlphaASCII(tmp))
{
if (tmp == 't' || tmp == 'T')
{
assertStringCaseInsensitive("rue", buf);
x = true;
}
else if (tmp == 'f' || tmp == 'F')
{
assertStringCaseInsensitive("alse", buf);
x = false;
}
}
}
template <typename ReturnType = void>
@ -1735,6 +1749,7 @@ inline T parse(const char * data, size_t size)
T res;
ReadBufferFromMemory buf(data, size);
readText(res, buf);
assertEOF(buf);
return res;
}
@ -1742,7 +1757,9 @@ template <typename T>
inline bool tryParse(T & res, const char * data, size_t size)
{
ReadBufferFromMemory buf(data, size);
return tryReadText(res, buf);
if (!tryReadText(res, buf))
return false;
return buf.eof();
}
template <typename T>

42
src/Interpreters/Cluster.cpp
View File

@ -230,21 +230,37 @@ String Cluster::Address::toFullString(bool use_compact_format) const
}
}
Cluster::Address Cluster::Address::fromFullString(const String & full_string)
Cluster::Address Cluster::Address::fromFullString(std::string_view full_string)
{
const char * address_begin = full_string.data();
const char * address_end = address_begin + full_string.size();
const char * user_pw_end = strchr(full_string.data(), '@');
std::string_view user_password;
if (auto pos = full_string.find('@'); pos != std::string_view::npos)
user_password = full_string.substr(pos + 1);
/// parsing with the new shard{shard_index}[_replica{replica_index}] format
if (!user_pw_end && startsWith(full_string, "shard"))
if (user_password.empty() && full_string.starts_with("shard"))
{
const char * underscore = strchr(full_string.data(), '_');
Address address;
address.shard_index = parse<UInt32>(address_begin + strlen("shard"));
address.replica_index = underscore ? parse<UInt32>(underscore + strlen("_replica")) : 0;
if (auto underscore_pos = full_string.find('_'); underscore_pos != std::string_view::npos)
{
address.shard_index = parse<UInt32>(full_string.substr(0, underscore_pos).substr(strlen("shard")));
if (full_string.substr(underscore_pos + 1).starts_with("replica"))
{
address.replica_index = parse<UInt32>(full_string.substr(underscore_pos + 1 + strlen("replica")));
}
else if (full_string.substr(underscore_pos + 1).starts_with("all_replicas"))
{
address.replica_index = 0;
}
else
throw Exception(ErrorCodes::SYNTAX_ERROR, "Incorrect address '{}', should be in a form of `shardN_all_replicas` or `shardN_replicaM`", full_string);
}
else
{
address.shard_index = parse<UInt32>(full_string.substr(strlen("shard")));
address.replica_index = 0;
}
return address;
}
@ -255,9 +271,13 @@ Cluster::Address Cluster::Address::fromFullString(const String & full_string)
/// - credentials are exposed in file name;
/// - the file name can be too long.
const char * address_begin = full_string.data();
const char * address_end = address_begin + full_string.size();
const char * user_pw_end = strchr(address_begin, '@');
Protocol::Secure secure = Protocol::Secure::Disable;
const char * secure_tag = "+secure";
if (endsWith(full_string, secure_tag))
if (full_string.ends_with(secure_tag))
{
address_end -= strlen(secure_tag);
secure = Protocol::Secure::Enable;

2
src/Interpreters/Cluster.h
View File

@ -168,7 +168,7 @@ public:
String toFullString(bool use_compact_format) const;
/// Returns address with only shard index and replica index or full address without shard index and replica index
static Address fromFullString(const String & address_full_string);
static Address fromFullString(std::string_view full_string);
/// Returns resolved address if it does resolve.
std::optional<Poco::Net::SocketAddress> getResolvedAddress() const;

3
src/Interpreters/Context.cpp
View File

@ -2957,6 +2957,9 @@ ProgressCallback Context::getProgressCallback() const
void Context::setProcessListElement(QueryStatusPtr elem)
{
if (isGlobalContext())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context cannot have process list element");
/// Set to a session or query. In the session, only one query is processed at a time. Therefore, the lock is not needed.
process_list_elem = elem;
has_process_list_elem = elem.get();

12
src/Interpreters/ThreadStatusExt.cpp
View File

@ -473,11 +473,11 @@ void ThreadStatus::initGlobalProfiler([[maybe_unused]] UInt64 global_profiler_re
{
if (global_profiler_real_time_period > 0)
query_profiler_real = std::make_unique<QueryProfilerReal>(thread_id,
/* period= */ static_cast<UInt32>(global_profiler_real_time_period));
/* period= */ global_profiler_real_time_period);
if (global_profiler_cpu_time_period > 0)
query_profiler_cpu = std::make_unique<QueryProfilerCPU>(thread_id,
/* period= */ static_cast<UInt32>(global_profiler_cpu_time_period));
/* period= */ global_profiler_cpu_time_period);
}
catch (...)
{
@ -506,18 +506,18 @@ void ThreadStatus::initQueryProfiler()
{
if (!query_profiler_real)
query_profiler_real = std::make_unique<QueryProfilerReal>(thread_id,
/* period= */ static_cast<UInt32>(settings.query_profiler_real_time_period_ns));
/* period= */ settings.query_profiler_real_time_period_ns);
else
query_profiler_real->setPeriod(static_cast<UInt32>(settings.query_profiler_real_time_period_ns));
query_profiler_real->setPeriod(settings.query_profiler_real_time_period_ns);
}
if (settings.query_profiler_cpu_time_period_ns > 0)
{
if (!query_profiler_cpu)
query_profiler_cpu = std::make_unique<QueryProfilerCPU>(thread_id,
/* period= */ static_cast<UInt32>(settings.query_profiler_cpu_time_period_ns));
/* period= */ settings.query_profiler_cpu_time_period_ns);
else
query_profiler_cpu->setPeriod(static_cast<UInt32>(settings.query_profiler_cpu_time_period_ns));
query_profiler_cpu->setPeriod(settings.query_profiler_cpu_time_period_ns);
}
}
catch (...)

9
src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp
View File

@ -758,6 +758,15 @@ static ColumnWithTypeAndName readNonNullableColumnFromArrowColumn(
case TypeIndex::IPv6:
return readIPv6ColumnFromBinaryData(arrow_column, column_name);
/// ORC format outputs big integers as binary column, because there is no fixed binary in ORC.
///
/// When ORC/Parquet file says the type is "byte array" or "fixed len byte array",
/// but the clickhouse query says to interpret the column as e.g. Int128, it
/// may mean one of two things:
/// * The byte array is the 16 bytes of Int128, little-endian.
/// * The byte array is an ASCII string containing the Int128 formatted in base 10.
/// There's no reliable way to distinguish these cases. We just guess: if the
/// byte array is variable-length, and the length is different from sizeof(type),
/// we parse as text, otherwise as binary.
case TypeIndex::Int128:
return readColumnWithBigNumberFromBinaryData<ColumnInt128>(arrow_column, column_name, type_hint);
case TypeIndex::UInt128:

120
src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp
View File

@ -25,6 +25,7 @@
#include <DataTypes/DataTypeNullable.h>
#include <Common/FieldVisitorsAccurateComparison.h>
#include <Processors/Formats/Impl/Parquet/ParquetRecordReader.h>
#include <Interpreters/convertFieldToType.h>
namespace CurrentMetrics
{
@ -54,7 +55,7 @@ namespace ErrorCodes
} \
} while (false)
/// Decode min/max value from column chunk statistics.
/// Decode min/max value from column chunk statistics. Returns Null if missing or unsupported.
///
/// There are two questionable decisions in this implementation:
/// * We parse the value from the encoded byte string instead of casting the parquet::Statistics
@ -62,7 +63,7 @@ namespace ErrorCodes
/// * We dispatch based on the parquet logical+converted+physical type instead of the ClickHouse type.
/// The idea is that this is similar to what we'll have to do when reimplementing Parquet parsing in
/// ClickHouse instead of using Arrow (for speed). So, this is an exercise in parsing Parquet manually.
static std::optional<Field> decodePlainParquetValueSlow(const std::string & data, parquet::Type::type physical_type, const parquet::ColumnDescriptor & descr)
static Field decodePlainParquetValueSlow(const std::string & data, parquet::Type::type physical_type, const parquet::ColumnDescriptor & descr, TypeIndex type_hint)
{
using namespace parquet;
@ -118,8 +119,6 @@ static std::optional<Field> decodePlainParquetValueSlow(const std::string & data
if (data.size() != size || size < 1 || size > 32)
throw Exception(ErrorCodes::CANNOT_PARSE_NUMBER, "Unexpected decimal size: {} (actual {})", size, data.size());
/// For simplicity, widen all decimals to 256-bit. It should compare correctly with values
/// of different bitness.
Int256 val = 0;
memcpy(&val, data.data(), size);
if (big_endian)
@ -128,7 +127,19 @@ static std::optional<Field> decodePlainParquetValueSlow(const std::string & data
if (size < 32 && (val >> (size * 8 - 1)) != 0)
val |= ~((Int256(1) << (size * 8)) - 1);
return Field(DecimalField<Decimal256>(Decimal256(val), static_cast<UInt32>(scale)));
auto narrow = [&](auto x) -> Field
{
memcpy(&x, &val, sizeof(x));
return Field(DecimalField<decltype(x)>(x, static_cast<UInt32>(scale)));
};
if (size <= 4)
return narrow(Decimal32(0));
else if (size <= 8)
return narrow(Decimal64(0));
else if (size <= 16)
return narrow(Decimal128(0));
else
return narrow(Decimal256(0));
}
while (false);
@ -185,8 +196,6 @@ static std::optional<Field> decodePlainParquetValueSlow(const std::string & data
return Field(val);
}
/// Strings.
if (physical_type == Type::type::BYTE_ARRAY || physical_type == Type::type::FIXED_LEN_BYTE_ARRAY)
{
/// Arrow's parquet decoder handles missing min/max values slightly incorrectly.
@ -213,14 +222,31 @@ static std::optional<Field> decodePlainParquetValueSlow(const std::string & data
/// TODO: Remove this workaround either when we implement our own Parquet decoder that
/// doesn't have this bug, or if it's fixed in Arrow.
if (data.empty())
return std::nullopt;
return Field();
/// Long integers, encoded either as text or as little-endian bytes.
/// The parquet file doesn't know that it's numbers, so the min/max are produced by comparing
/// strings lexicographically. So these min and max are mostly useless to us.
/// There's one case where they're not useless: min == max; currently we don't make use of this.
switch (type_hint)
{
case TypeIndex::UInt128:
case TypeIndex::UInt256:
case TypeIndex::Int128:
case TypeIndex::Int256:
case TypeIndex::IPv6:
return Field();
default: break;
}
/// Strings.
return Field(data);
}
/// This one's deprecated in Parquet.
/// This type is deprecated in Parquet.
/// TODO: But turns out it's still used in practice, we should support it.
if (physical_type == Type::type::INT96)
throw Exception(ErrorCodes::CANNOT_PARSE_NUMBER, "Parquet INT96 type is deprecated and not supported");
return Field();
/// Integers.
@ -283,15 +309,13 @@ static std::vector<Range> getHyperrectangleForRowGroup(const parquet::FileMetaDa
continue;
auto stats = it->second;
auto default_value = [&]() -> Field
{
DataTypePtr type = header.getByPosition(idx).type;
if (type->lowCardinality())
type = assert_cast<const DataTypeLowCardinality &>(*type).getDictionaryType();
if (type->isNullable())
type = assert_cast<const DataTypeNullable &>(*type).getNestedType();
return type->getDefault();
};
Field default_value = type->getDefault();
TypeIndex type_index = type->getTypeId();
/// Only primitive fields are supported, not arrays, maps, tuples, or Nested.
/// Arrays, maps, and Nested can't be meaningfully supported because Parquet only has min/max
@ -299,14 +323,47 @@ static std::vector<Range> getHyperrectangleForRowGroup(const parquet::FileMetaDa
/// Same limitation for tuples, but maybe it would make sense to have some kind of tuple
/// expansion in KeyCondition to accept ranges per element instead of whole tuple.
std::optional<Field> min;
std::optional<Field> max;
Field min;
Field max;
if (stats->HasMinMax())
{
try
{
min = decodePlainParquetValueSlow(stats->EncodeMin(), stats->physical_type(), *stats->descr());
max = decodePlainParquetValueSlow(stats->EncodeMax(), stats->physical_type(), *stats->descr());
min = decodePlainParquetValueSlow(stats->EncodeMin(), stats->physical_type(), *stats->descr(), type_index);
max = decodePlainParquetValueSlow(stats->EncodeMax(), stats->physical_type(), *stats->descr(), type_index);
/// If the data type in parquet file substantially differs from the requested data type,
/// it's sometimes correct to just typecast the min/max values.
/// Other times it's incorrect, e.g.:
/// INSERT INTO FUNCTION file('t.parquet', Parquet, 'x String') VALUES ('1'), ('100'), ('2');
/// SELECT * FROM file('t.parquet', Parquet, 'x Int64') WHERE x >= 3;
/// If we just typecast min/max from string to integer, this query will incorrectly return empty result.
/// Allow conversion in some simple cases, otherwise ignore the min/max values.
auto min_type = min.getType();
auto max_type = max.getType();
min = convertFieldToType(min, *type);
max = convertFieldToType(max, *type);
auto ok_cast = [&](Field::Types::Which from, Field::Types::Which to) -> bool
{
if (from == to)
return true;
/// Decimal -> wider decimal.
if (Field::isDecimal(from) || Field::isDecimal(to))
return Field::isDecimal(from) && Field::isDecimal(to) && to >= from;
/// Integer -> IP.
if (to == Field::Types::IPv4)
return from == Field::Types::UInt64;
/// Disable index for everything else, especially string <-> number.
return false;
};
if (!(ok_cast(min_type, min.getType()) && ok_cast(max_type, max.getType())) &&
!(min == max) &&
!(min_type == Field::Types::Int64 && min.getType() == Field::Types::UInt64 && min.safeGet<Int64>() >= 0) &&
!(max_type == Field::Types::UInt64 && max.getType() == Field::Types::Int64 && max.safeGet<UInt64>() <= UInt64(INT64_MAX)))
{
min = Field();
max = Field();
}
}
catch (Exception & e)
{
@ -328,7 +385,7 @@ static std::vector<Range> getHyperrectangleForRowGroup(const parquet::FileMetaDa
{
/// Single-point range containing either the default value of one of the infinities.
if (null_as_default)
hyperrectangle[idx].right = hyperrectangle[idx].left = default_value();
hyperrectangle[idx].right = hyperrectangle[idx].left = default_value;
else
hyperrectangle[idx].right = hyperrectangle[idx].left;
continue;
@ -339,32 +396,31 @@ static std::vector<Range> getHyperrectangleForRowGroup(const parquet::FileMetaDa
if (null_as_default)
{
/// Make sure the range contains the default value.
Field def = default_value();
if (min.has_value() && applyVisitor(FieldVisitorAccurateLess(), def, *min))
min = def;
if (max.has_value() && applyVisitor(FieldVisitorAccurateLess(), *max, def))
max = def;
if (!min.isNull() && applyVisitor(FieldVisitorAccurateLess(), default_value, min))
min = default_value;
if (!max.isNull() && applyVisitor(FieldVisitorAccurateLess(), max, default_value))
max = default_value;
}
else
{
/// Make sure the range reaches infinity on at least one side.
if (min.has_value() && max.has_value())
min.reset();
if (!min.isNull() && !max.isNull())
min = Field();
}
}
else
{
/// If the column doesn't have nulls, exclude both infinities.
if (!min.has_value())
if (min.isNull())
hyperrectangle[idx].left_included = false;
if (!max.has_value())
if (max.isNull())
hyperrectangle[idx].right_included = false;
}
if (min.has_value())
hyperrectangle[idx].left = std::move(min.value());
if (max.has_value())
hyperrectangle[idx].right = std::move(max.value());
if (!min.isNull())
hyperrectangle[idx].left = std::move(min);
if (!max.isNull())
hyperrectangle[idx].right = std::move(max);
}
return hyperrectangle;

12
src/Storages/MergeTree/KeyCondition.cpp
View File

@ -888,13 +888,22 @@ static Field applyFunctionForField(
return (*col)[0];
}
/// applyFunction will execute the function with one `field` or the column which `field` refers to.
static FieldRef applyFunction(const FunctionBasePtr & func, const DataTypePtr & current_type, const FieldRef & field)
{
chassert(func != nullptr);
/// Fallback for fields without block reference.
if (field.isExplicit())
return applyFunctionForField(func, current_type, field);
String result_name = "_" + func->getName() + "_" + toString(field.column_idx);
/// We will cache the function result inside `field.columns`, because this function will call many times
/// from many fields from same column. When the column is huge, for example there are thousands of marks, we need a cache.
/// The cache key is like `_[function_pointer]_[param_column_id]` to identify a unique <function, param> pair.
WriteBufferFromOwnString buf;
writeText("_", buf);
writePointerHex(func.get(), buf);
writeText("_" + toString(field.column_idx), buf);
String result_name = buf.str();
const auto & columns = field.columns;
size_t result_idx = columns->size();
@ -906,6 +915,7 @@ static FieldRef applyFunction(const FunctionBasePtr & func, const DataTypePtr &
if (result_idx == columns->size())
{
/// When cache is missed, we calculate the whole column where the field comes from. This will avoid repeated calculation.
ColumnsWithTypeAndName args{(*columns)[field.column_idx]};
field.columns->emplace_back(ColumnWithTypeAndName {nullptr, func->getResultType(), result_name});
(*columns)[result_idx].column = func->execute(args, (*columns)[result_idx].type, columns->front().column->size());

125
src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp
View File

@ -55,6 +55,7 @@ const std::unordered_map<String, unum::usearch::scalar_kind_t> quantizationToSca
{"f32", unum::usearch::scalar_kind_t::f32_k},
{"f16", unum::usearch::scalar_kind_t::f16_k},
{"i8", unum::usearch::scalar_kind_t::i8_k}};
/// Usearch provides more quantizations but ^^ above ones seem the only ones comprehensively supported across all distance functions.
template<typename T>
concept is_set = std::same_as<T, std::set<typename T::key_type, typename T::key_compare, typename T::allocator_type>>;
@ -98,9 +99,6 @@ USearchIndexWithSerialization::USearchIndexWithSerialization(
unum::usearch::index_dense_config_t config(usearch_hnsw_params.m, usearch_hnsw_params.ef_construction, usearch_hnsw_params.ef_search);
config.enable_key_lookups = false; /// we don't do row-to-vector lookups
if (auto error = config.validate(); error) /// already called in vectorSimilarityIndexValidator, call again because usearch may change the config in-place
throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid parameters passed to vector similarity index. Error: {}", String(error.release()));
if (auto result = USearchIndex::make(metric, config); !result)
throw Exception(ErrorCodes::INCORRECT_DATA, "Could not create vector similarity index. Error: {}", String(result.error.release()));
else
@ -250,14 +248,47 @@ MergeTreeIndexGranulePtr MergeTreeIndexAggregatorVectorSimilarity::getGranuleAnd
return granule;
}
namespace
{
template <typename Column>
void updateImpl(const ColumnArray * column_array, const ColumnArray::Offsets & column_array_offsets, USearchIndexWithSerializationPtr & index, size_t dimensions, size_t rows)
{
const auto & column_array_data = column_array->getData();
const auto & column_array_data_float = typeid_cast<const Column &>(column_array_data);
const auto & column_array_data_float_data = column_array_data_float.getData();
/// Check all sizes are the same
for (size_t row = 0; row < rows - 1; ++row)
if (column_array_offsets[row + 1] - column_array_offsets[row] != dimensions)
throw Exception(ErrorCodes::INCORRECT_DATA, "All arrays in column with vector similarity index must have equal length");
/// Reserving space is mandatory
if (!index->try_reserve(roundUpToPowerOfTwoOrZero(index->size() + rows)))
throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not reserve memory for vector similarity index");
for (size_t row = 0; row < rows; ++row)
{
if (auto result = index->add(static_cast<USearchIndex::vector_key_t>(index->size()), &column_array_data_float_data[column_array_offsets[row - 1]]); !result)
throw Exception(ErrorCodes::INCORRECT_DATA, "Could not add data to vector similarity index. Error: {}", String(result.error.release()));
else
{
ProfileEvents::increment(ProfileEvents::USearchAddCount);
ProfileEvents::increment(ProfileEvents::USearchAddVisitedMembers, result.visited_members);
ProfileEvents::increment(ProfileEvents::USearchAddComputedDistances, result.computed_distances);
}
}
}
}
void MergeTreeIndexAggregatorVectorSimilarity::update(const Block & block, size_t * pos, size_t limit)
{
if (*pos >= block.rows())
throw Exception(
ErrorCodes::LOGICAL_ERROR,
"The provided position is not less than the number of block rows. Position: {}, Block rows: {}.",
*pos,
block.rows());
*pos, block.rows());
size_t rows_read = std::min(limit, block.rows() - *pos);
@ -271,16 +302,12 @@ void MergeTreeIndexAggregatorVectorSimilarity::update(const Block & block, size_
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected block with single column");
const String & index_column_name = index_sample_block.getByPosition(0).name;
ColumnPtr column_cut = block.getByName(index_column_name).column->cut(*pos, rows_read);
const ColumnPtr & index_column = block.getByName(index_column_name).column;
ColumnPtr column_cut = index_column->cut(*pos, rows_read);
if (const auto & column_array = typeid_cast<const ColumnArray *>(column_cut.get()))
{
const auto & column_array_data = column_array->getData();
const auto & column_array_data_float = typeid_cast<const ColumnFloat32 &>(column_array_data);
const auto & column_array_data_float_data = column_array_data_float.getData();
const auto & column_array_offsets = column_array->getOffsets();
const size_t num_rows = column_array_offsets.size();
const auto * column_array = typeid_cast<const ColumnArray *>(column_cut.get());
if (!column_array)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected Array(Float*) column");
if (column_array->empty())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Array is unexpectedly empty");
@ -291,43 +318,37 @@ void MergeTreeIndexAggregatorVectorSimilarity::update(const Block & block, size_
if (column_array->isDefaultAt(0))
throw Exception(ErrorCodes::INCORRECT_DATA, "The arrays in column '{}' must not be empty. Did you try to INSERT default values?", index_column_name);
/// Check all sizes are the same
const size_t dimensions = column_array_offsets[0];
for (size_t i = 0; i < num_rows - 1; ++i)
if (column_array_offsets[i + 1] - column_array_offsets[i] != dimensions)
throw Exception(ErrorCodes::INCORRECT_DATA, "All arrays in column '{}' must have equal length", index_column_name);
const size_t rows = column_array->size();
/// Also check that previously inserted blocks have the same size as this block.
/// Note that this guarantees consistency of dimension only within parts. We are unable to detect inconsistent dimensions across
/// parts - for this, a little help from the user is needed, e.g. CONSTRAINT cnstr CHECK length(array) = 42.
if (index && index->dimensions() != dimensions)
throw Exception(ErrorCodes::INCORRECT_DATA, "All arrays in column '{}' must have equal length", index_column_name);
const auto & column_array_offsets = column_array->getOffsets();
const size_t dimensions = column_array_offsets[0];
if (!index)
index = std::make_shared<USearchIndexWithSerialization>(dimensions, metric_kind, scalar_kind, usearch_hnsw_params);
/// Also check that previously inserted blocks have the same size as this block.
/// Note that this guarantees consistency of dimension only within parts. We are unable to detect inconsistent dimensions across
/// parts - for this, a little help from the user is needed, e.g. CONSTRAINT cnstr CHECK length(array) = 42.
if (index->dimensions() != dimensions)
throw Exception(ErrorCodes::INCORRECT_DATA, "All arrays in column with vector similarity index must have equal length");
/// We use Usearch's index_dense_t as index type which supports only 4 bio entries according to https://github.com/unum-cloud/usearch/tree/main/cpp
if (index->size() + num_rows > std::numeric_limits<UInt32>::max())
throw Exception(ErrorCodes::INCORRECT_DATA, "Size of vector similarity index in column {} would exceed 4 billion entries", index_column_name);
if (index->size() + rows > std::numeric_limits<UInt32>::max())
throw Exception(ErrorCodes::INCORRECT_DATA, "Size of vector similarity index would exceed 4 billion entries");
/// Reserving space is mandatory
if (!index->try_reserve(roundUpToPowerOfTwoOrZero(index->size() + num_rows)))
throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not reserve memory for vector similarity index");
DataTypePtr data_type = block.getDataTypes()[0];
const auto * data_type_array = typeid_cast<const DataTypeArray *>(data_type.get());
if (!data_type_array)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected data type Array(Float*)");
const TypeIndex nested_type_index = data_type_array->getNestedType()->getTypeId();
for (size_t row = 0; row < num_rows; ++row)
{
if (auto result = index->add(static_cast<UInt32>(index->size()), &column_array_data_float_data[column_array_offsets[row - 1]]); !result)
throw Exception(ErrorCodes::INCORRECT_DATA, "Could not add data to vector similarity index. Error: {}", String(result.error.release()));
if (WhichDataType(nested_type_index).isFloat32())
updateImpl<ColumnFloat32>(column_array, column_array_offsets, index, dimensions, rows);
else if (WhichDataType(nested_type_index).isFloat64())
updateImpl<ColumnFloat64>(column_array, column_array_offsets, index, dimensions, rows);
else
{
ProfileEvents::increment(ProfileEvents::USearchAddCount);
ProfileEvents::increment(ProfileEvents::USearchAddVisitedMembers, result.visited_members);
ProfileEvents::increment(ProfileEvents::USearchAddComputedDistances, result.computed_distances);
}
}
}
else
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected Array(Float32) column");
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected data type Array(Float*)");
*pos += rows_read;
}
@ -375,7 +396,7 @@ std::vector<size_t> MergeTreeIndexConditionVectorSimilarity::getUsefulRanges(Mer
"does not match the dimension in the index ({})",
vector_similarity_condition.getDimensions(), index->dimensions());
const std::vector<float> reference_vector = vector_similarity_condition.getReferenceVector();
const std::vector<Float64> reference_vector = vector_similarity_condition.getReferenceVector();
auto search_result = index->search(reference_vector.data(), limit);
if (!search_result)
@ -486,7 +507,7 @@ void vectorSimilarityIndexValidator(const IndexDescription & index, bool /* atta
if (!quantizationToScalarKind.contains(index.arguments[2].safeGet<String>()))
throw Exception(ErrorCodes::INCORRECT_DATA, "Third argument (quantization) of vector similarity index is not supported. Supported quantizations are: {}", joinByComma(quantizationToScalarKind));
/// Call Usearche's own parameter validation method for HNSW-specific parameters
/// Call Usearch's own parameter validation method for HNSW-specific parameters
UInt64 m = index.arguments[3].safeGet<UInt64>();
UInt64 ef_construction = index.arguments[4].safeGet<UInt64>();
UInt64 ef_search = index.arguments[5].safeGet<UInt64>();
@ -501,18 +522,14 @@ void vectorSimilarityIndexValidator(const IndexDescription & index, bool /* atta
if (index.column_names.size() != 1 || index.data_types.size() != 1)
throw Exception(ErrorCodes::INCORRECT_NUMBER_OF_COLUMNS, "Vector similarity indexes must be created on a single column");
/// Check data type of the indexed column:
/// Check that the data type is Array(Float*)
DataTypePtr data_type = index.sample_block.getDataTypes()[0];
if (const auto * data_type_array = typeid_cast<const DataTypeArray *>(data_type.get()))
{
const auto * data_type_array = typeid_cast<const DataTypeArray *>(data_type.get());
if (!data_type_array)
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Vector similarity indexes can only be created on columns of type Array(Float*)");
TypeIndex nested_type_index = data_type_array->getNestedType()->getTypeId();
if (!WhichDataType(nested_type_index).isFloat32())
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Vector similarity indexes can only be created on columns of type Array(Float32)");
}
else
{
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Vector similarity indexes can only be created on columns of type Array(Float32)");
}
if (!WhichDataType(nested_type_index).isFloat())
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Vector similarity indexes can only be created on columns of type Array(Float*)");
}
}

4
src/Storages/MergeTree/VectorSimilarityCondition.cpp
View File

@ -24,7 +24,7 @@ namespace
{
template <typename Literal>
void extractReferenceVectorFromLiteral(std::vector<Float32> & reference_vector, Literal literal)
void extractReferenceVectorFromLiteral(std::vector<Float64> & reference_vector, Literal literal)
{
Float64 float_element_of_reference_vector;
Int64 int_element_of_reference_vector;
@ -72,7 +72,7 @@ UInt64 VectorSimilarityCondition::getLimit() const
throw Exception(ErrorCodes::LOGICAL_ERROR, "No LIMIT section in query, not supported");
}
std::vector<float> VectorSimilarityCondition::getReferenceVector() const
std::vector<Float64> VectorSimilarityCondition::getReferenceVector() const
{
if (index_is_useful && query_information.has_value())
return query_information->reference_vector;

4
src/Storages/MergeTree/VectorSimilarityCondition.h
View File

@ -60,7 +60,7 @@ public:
L2
};
std::vector<Float32> reference_vector;
std::vector<Float64> reference_vector;
DistanceFunction distance_function;
String column_name;
UInt64 limit;
@ -70,7 +70,7 @@ public:
/// Returns false if query can be speeded up by an ANN index, true otherwise.
bool alwaysUnknownOrTrue(String distance_function) const;
std::vector<float> getReferenceVector() const;
std::vector<Float64> getReferenceVector() const;
size_t getDimensions() const;
String getColumnName() const;
Info::DistanceFunction getDistanceFunction() const;

2
src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp
View File

@ -332,6 +332,8 @@ struct DeltaLakeMetadataImpl
WhichDataType which(check_type->getTypeId());
if (which.isStringOrFixedString())
return value;
else if (isBool(check_type))
return parse<bool>(value);
else if (which.isInt8())
return parse<Int8>(value);
else if (which.isUInt8())

51
src/Storages/StorageExternalDistributed.cpp
View File

@ -1,4 +1,4 @@
#include "StorageExternalDistributed.h"
#include <Storages/StorageExternalDistributed.h>
#include <Core/Settings.h>
#include <Storages/StorageFactory.h>
@ -6,6 +6,8 @@
#include <Interpreters/InterpreterSelectQuery.h>
#include <Core/PostgreSQL/PoolWithFailover.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Common/parseAddress.h>
#include <Processors/QueryPlan/QueryPlan.h>
#include <Common/parseRemoteDescription.h>
@ -112,14 +114,39 @@ void registerStorageExternalDistributed(StorageFactory & factory)
std::unordered_set<StoragePtr> shards;
ASTs inner_engine_args(engine_args.begin() + 1, engine_args.end());
ASTPtr * address_arg = nullptr;
/// If there is a named collection argument, named `addresses_expr`
for (auto & node : inner_engine_args)
{
if (ASTFunction * func = node->as<ASTFunction>(); func && func->name == "equals" && func->arguments)
{
if (ASTExpressionList * func_args = func->arguments->as<ASTExpressionList>(); func_args && func_args->children.size() == 2)
{
if (ASTIdentifier * arg_name = func_args->children[0]->as<ASTIdentifier>(); arg_name && arg_name->name() == "addresses_expr")
{
address_arg = &func_args->children[1];
break;
}
}
}
}
/// Otherwise it is the first argument.
if (!address_arg)
address_arg = &inner_engine_args.at(0);
String addresses_expr = checkAndGetLiteralArgument<String>(*address_arg, "addresses");
Strings shards_addresses = get_addresses(addresses_expr);
auto engine_name = checkAndGetLiteralArgument<String>(engine_args[0], "engine_name");
if (engine_name == "URL")
{
auto configuration = StorageURL::getConfiguration(inner_engine_args, context);
auto shards_addresses = get_addresses(configuration.addresses_expr);
auto format_settings = StorageURL::getFormatSettingsFromArgs(args);
for (const auto & shard_address : shards_addresses)
{
*address_arg = std::make_shared<ASTLiteral>(shard_address);
auto configuration = StorageURL::getConfiguration(inner_engine_args, context);
auto uri_options = parseRemoteDescription(shard_address, 0, shard_address.size(), '|', max_addresses);
if (uri_options.size() > 1)
{
@ -140,13 +167,12 @@ void registerStorageExternalDistributed(StorageFactory & factory)
else if (engine_name == "MySQL")
{
MySQLSettings mysql_settings;
auto configuration = StorageMySQL::getConfiguration(inner_engine_args, context, mysql_settings);
auto shards_addresses = get_addresses(configuration.addresses_expr);
for (const auto & shard_address : shards_addresses)
{
auto current_configuration{configuration};
current_configuration.addresses = parseRemoteDescriptionForExternalDatabase(shard_address, max_addresses, 3306);
auto pool = createMySQLPoolWithFailover(current_configuration, mysql_settings);
*address_arg = std::make_shared<ASTLiteral>(shard_address);
auto configuration = StorageMySQL::getConfiguration(inner_engine_args, context, mysql_settings);
configuration.addresses = parseRemoteDescriptionForExternalDatabase(shard_address, max_addresses, 3306);
auto pool = createMySQLPoolWithFailover(configuration, mysql_settings);
shards.insert(std::make_shared<StorageMySQL>(
args.table_id, std::move(pool), configuration.database, configuration.table,
/* replace_query = */ false, /* on_duplicate_clause = */ "",
@ -157,14 +183,13 @@ void registerStorageExternalDistributed(StorageFactory & factory)
#if USE_LIBPQXX
else if (engine_name == "PostgreSQL")
{
auto configuration = StoragePostgreSQL::getConfiguration(inner_engine_args, context);
auto shards_addresses = get_addresses(configuration.addresses_expr);
for (const auto & shard_address : shards_addresses)
{
auto current_configuration{configuration};
current_configuration.addresses = parseRemoteDescriptionForExternalDatabase(shard_address, max_addresses, 5432);
*address_arg = std::make_shared<ASTLiteral>(shard_address);
auto configuration = StoragePostgreSQL::getConfiguration(inner_engine_args, context);
configuration.addresses = parseRemoteDescriptionForExternalDatabase(shard_address, max_addresses, 5432);
auto pool = std::make_shared<postgres::PoolWithFailover>(
current_configuration,
configuration,
settings.postgresql_connection_pool_size,
settings.postgresql_connection_pool_wait_timeout,
settings.postgresql_connection_pool_retries,

2
tests/integration/test_cgroup_limit/test.py
View File

@ -46,7 +46,7 @@ def test_cgroup_cpu_limit():
"clickhouse local -q \"select value from system.settings where name='max_threads'\"",
num_cpus,
)
expect_output = (r"\'auto({})\'".format(math.ceil(num_cpus))).encode()
expect_output = (r"auto({})".format(math.ceil(num_cpus))).encode()
assert (
result.strip() == expect_output
), f"fail for cpu limit={num_cpus}, result={result.strip()}, expect={expect_output}"

20
tests/queries/0_stateless/00366_multi_statements.sh
View File

@ -14,22 +14,22 @@ $CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2"
$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2;"
$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2; SELECT" 2>&1 | grep -o 'Syntax error'
$CLICKHOUSE_CLIENT -n --query="SELECT 1; S" 2>&1 | grep -o 'Syntax error'
$CLICKHOUSE_CLIENT -n --query="SELECT 1; SELECT 2"
$CLICKHOUSE_CLIENT -n --query="SELECT 1; SELECT 2;"
$CLICKHOUSE_CLIENT -n --query="SELECT 1; SELECT 2; SELECT" 2>&1 | grep -o 'Syntax error'
$CLICKHOUSE_CLIENT --query="SELECT 1; S" 2>&1 | grep -o 'Syntax error'
$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2"
$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2;"
$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2; SELECT" 2>&1 | grep -o 'Syntax error'
$CLICKHOUSE_CLIENT -n --query="DROP TABLE IF EXISTS t_00366; CREATE TABLE t_00366 (x UInt64) ENGINE = TinyLog;"
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS t_00366; CREATE TABLE t_00366 (x UInt64) ENGINE = TinyLog;"
$CLICKHOUSE_CLIENT --query="INSERT INTO t_00366 VALUES (1),(2),(3);"
$CLICKHOUSE_CLIENT --query="SELECT * FROM t_00366"
$CLICKHOUSE_CLIENT --query="INSERT INTO t_00366 VALUES" <<< "(4),(5),(6)"
$CLICKHOUSE_CLIENT --query="SELECT * FROM t_00366"
$CLICKHOUSE_CLIENT -n --query="INSERT INTO t_00366 VALUES (1),(2),(3);"
$CLICKHOUSE_CLIENT -n --query="SELECT * FROM t_00366"
$CLICKHOUSE_CLIENT -n --query="INSERT INTO t_00366 VALUES" <<< "(4),(5),(6)"
$CLICKHOUSE_CLIENT -n --query="SELECT * FROM t_00366"
$CLICKHOUSE_CLIENT --query="INSERT INTO t_00366 VALUES (1),(2),(3);"
$CLICKHOUSE_CLIENT --query="SELECT * FROM t_00366"
$CLICKHOUSE_CLIENT --query="INSERT INTO t_00366 VALUES" <<< "(4),(5),(6)"
$CLICKHOUSE_CLIENT --query="SELECT * FROM t_00366"
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT 1"
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT 1;"
@ -48,4 +48,4 @@ $CLICKHOUSE_CLIENT --query="SELECT * FROM t_00366"
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_00366+VALUES" -d "(7),(8),(9)"
$CLICKHOUSE_CLIENT --query="SELECT * FROM t_00366"
$CLICKHOUSE_CLIENT -n --query="DROP TABLE t_00366;"
$CLICKHOUSE_CLIENT --query="DROP TABLE t_00366;"

4
tests/queries/0_stateless/00443_preferred_block_size_bytes.sh
View File

@ -43,10 +43,10 @@ popd > /dev/null
#SCRIPTDIR=`dirname "$SCRIPTPATH"`
SCRIPTDIR=$SCRIPTPATH
cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=10 -n > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout
cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=10 > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout
cmp "$SCRIPTDIR"/00282_merging.reference "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout && echo PASSED || echo FAILED
cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=20 -n > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout
cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=20 > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout
cmp "$SCRIPTDIR"/00282_merging.reference "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout && echo PASSED || echo FAILED
rm "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout

20
tests/queries/0_stateless/00543_access_to_temporary_table_in_readonly_mode.sh
View File

@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP TABLE IF EXISTS test_readonly;
CREATE TABLE test_readonly (
ID Int
@ -16,7 +16,7 @@ $CLICKHOUSE_CLIENT -n --query="
################
# Try to create temporary table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 1;
CREATE TEMPORARY TABLE readonly (
ID Int
@ -26,7 +26,7 @@ CODE=$?;
[ "$CODE" -ne "164" ] && [ "$CODE" -ne "0" ] && echo "Fail" && exit $CODE;
# Try to insert into exists (non temporary) table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 1;
INSERT INTO test_readonly (ID) VALUES (1);
" 2> /dev/null;
@ -34,7 +34,7 @@ CODE=$?;
[ "$CODE" -ne "164" ] && [ "$CODE" -ne "0" ] && echo "Fail" && exit $CODE;
# Try to drop exists (non temporary) table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 1;
DROP TABLE test_readonly;
" 2> /dev/null;
@ -46,7 +46,7 @@ CODE=$?;
################
# Try to create temporary table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 2;
CREATE TEMPORARY TABLE readonly (
ID Int
@ -58,7 +58,7 @@ CODE=$?;
[ "$CODE" -ne "0" ] && echo "Fail" && exit $CODE;
# Try to insert into exists (non temporary) table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 2;
INSERT INTO test_readonly (ID) VALUES (1);
" 2> /dev/null;
@ -66,7 +66,7 @@ CODE=$?;
[ "$CODE" -ne "164" ] && [ "$CODE" -ne "0" ] && echo "Fail" && exit $CODE;
# Try to drop exists (non temporary) table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 2;
DROP TABLE test_readonly;
" 2> /dev/null;
@ -78,7 +78,7 @@ CODE=$?;
################
# Try to create temporary table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 0;
CREATE TEMPORARY TABLE readonly (
ID Int
@ -90,7 +90,7 @@ CODE=$?;
[ "$CODE" -ne "0" ] && echo "Fail" && exit $CODE;
# Try to insert into exists (non temporary) table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 0;
INSERT INTO test_readonly (ID) VALUES (1);
" 2> /dev/null;
@ -98,7 +98,7 @@ CODE=$?;
[ "$CODE" -ne "0" ] && echo "Fail" && exit $CODE;
# Try to drop exists (non temporary) table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 0;
DROP TABLE test_readonly;
" 2> /dev/null;

43
tests/queries/0_stateless/00600_replace_running_query.sh
View File

@ -6,41 +6,56 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
TEST_PREFIX=$RANDOM
TEST_PREFIX="${CLICKHOUSE_DATABASE}"
${CLICKHOUSE_CLIENT} -q "drop user if exists u_00600${TEST_PREFIX}"
${CLICKHOUSE_CLIENT} -q "create user u_00600${TEST_PREFIX} settings max_execution_time=60, readonly=1"
${CLICKHOUSE_CLIENT} -q "grant select on system.numbers to u_00600${TEST_PREFIX}"
function wait_for_query_to_start()
{
while [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() FROM system.processes WHERE query_id = '$1'") == 0 ]]; do sleep 0.1; done
while [[ 0 -eq $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() FROM system.processes WHERE query_id = '$1'") ]]
do
sleep 0.1
done
}
function wait_for_queries_to_finish()
{
while [[ 0 -ne $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() FROM system.processes WHERE current_database = '${CLICKHOUSE_DATABASE}' AND query NOT LIKE '%this query%'") ]]
do
sleep 0.1
done
}
$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL&query_id=hello&replace_running_query=1" -d 'SELECT 1, count() FROM system.numbers' > /dev/null 2>&1 &
wait_for_query_to_start 'hello'
$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL&query_id=${CLICKHOUSE_DATABASE}hello&replace_running_query=1" -d 'SELECT 1, count() FROM system.numbers' > /dev/null 2>&1 &
wait_for_query_to_start "${CLICKHOUSE_DATABASE}hello"
# Replace it
$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL&query_id=hello&replace_running_query=1" -d 'SELECT 0'
$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL&query_id=${CLICKHOUSE_DATABASE}hello&replace_running_query=1" -d 'SELECT 0'
# Wait for it to be replaced
wait
wait_for_queries_to_finish
${CLICKHOUSE_CLIENT_BINARY} --user=u_00600${TEST_PREFIX} --query_id=42 --query='SELECT 2, count() FROM system.numbers' 2>&1 | grep -cF 'was cancelled' &
wait_for_query_to_start '42'
${CLICKHOUSE_CLIENT_BINARY} --user=u_00600${TEST_PREFIX} --query_id="${CLICKHOUSE_DATABASE}42" --query='SELECT 2, count() FROM system.numbers' 2>&1 | grep -cF 'QUERY_WAS_CANCELLED' &
wait_for_query_to_start "${CLICKHOUSE_DATABASE}42"
# Trying to run another query with the same query_id
${CLICKHOUSE_CLIENT} --query_id=42 --query='SELECT 43' 2>&1 | grep -cF 'is already running by user'
${CLICKHOUSE_CLIENT} --query_id="${CLICKHOUSE_DATABASE}42" --query='SELECT 43' 2>&1 | grep -cF 'is already running by user'
# Trying to replace query of a different user
$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL&query_id=42&replace_running_query=1" -d 'SELECT 1' | grep -cF 'is already running by user'
$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL&query_id=${CLICKHOUSE_DATABASE}42&replace_running_query=1" -d 'SELECT 1' | grep -cF 'is already running by user'
$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "KILL QUERY WHERE query_id = '42' SYNC" > /dev/null
$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "KILL QUERY WHERE query_id = '${CLICKHOUSE_DATABASE}42' SYNC" > /dev/null
wait
wait_for_queries_to_finish
${CLICKHOUSE_CLIENT} --query_id=42 --query='SELECT 3, count() FROM system.numbers' 2>&1 | grep -cF 'was cancelled' &
wait_for_query_to_start '42'
${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --replace_running_query_max_wait_ms=500 --query='SELECT 43' 2>&1 | grep -F "can't be stopped" > /dev/null
${CLICKHOUSE_CLIENT} --query_id="${CLICKHOUSE_DATABASE}42" --query='SELECT 3, count() FROM system.numbers' 2>&1 | grep -cF 'QUERY_WAS_CANCELLED' &
wait_for_query_to_start "${CLICKHOUSE_DATABASE}42"
${CLICKHOUSE_CLIENT} --query_id="${CLICKHOUSE_DATABASE}42" --replace_running_query=1 --replace_running_query_max_wait_ms=500 --query='SELECT 43' 2>&1 | grep -F "can't be stopped" > /dev/null
wait
${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --query='SELECT 44'
wait_for_queries_to_finish
${CLICKHOUSE_CLIENT} --query_id="${CLICKHOUSE_DATABASE}42" --replace_running_query=1 --query='SELECT 44'
${CLICKHOUSE_CLIENT} -q "drop user u_00600${TEST_PREFIX}"

4
tests/queries/0_stateless/00634_performance_introspection_and_logging.sh
View File

@ -19,13 +19,13 @@ settings="$server_logs --log_queries=1 --log_query_threads=1 --log_profile_event
# Test insert logging on each block and checkPacket() method
$CLICKHOUSE_CLIENT $settings -n -q "
$CLICKHOUSE_CLIENT $settings -q "
DROP TABLE IF EXISTS null_00634;
CREATE TABLE null_00634 (i UInt8) ENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple();"
head -c 1000 /dev/zero | $CLICKHOUSE_CLIENT $settings --max_insert_block_size=10 --min_insert_block_size_rows=1 --min_insert_block_size_bytes=1 -q "INSERT INTO null_00634 FORMAT RowBinary"
$CLICKHOUSE_CLIENT $settings -n -q "
$CLICKHOUSE_CLIENT $settings -q "
SELECT count() FROM null_00634;
DROP TABLE null_00634;"

6
tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh
View File

@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/mergetree_mutations.lib
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
DROP TABLE IF EXISTS fetches_r1 SYNC;
DROP TABLE IF EXISTS fetches_r2 SYNC"
@ -17,7 +17,7 @@ ${CLICKHOUSE_CLIENT} --query="CREATE TABLE fetches_r2(x UInt32) ENGINE Replicate
SETTINGS prefer_fetch_merged_part_time_threshold=0, \
prefer_fetch_merged_part_size_threshold=0"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET insert_keeper_fault_injection_probability=0;
INSERT INTO fetches_r1 VALUES (1);
INSERT INTO fetches_r1 VALUES (2);
@ -51,6 +51,6 @@ ${CLICKHOUSE_CLIENT} --query="SYSTEM SYNC REPLICA fetches_r2"
${CLICKHOUSE_CLIENT} --query="SELECT '*** Check data after fetch/clone of mutated part ***'"
${CLICKHOUSE_CLIENT} --query="SELECT _part, * FROM fetches_r2 ORDER BY x"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
DROP TABLE fetches_r1 SYNC;
DROP TABLE fetches_r2 SYNC"

30
tests/queries/0_stateless/00754_distributed_optimize_skip_select_on_unused_shards.sh
View File

@ -25,83 +25,83 @@ ${CLICKHOUSE_CLIENT} --query "SELECT count(*) FROM distributed WHERE a = 0 AND b
| grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
# Should pass now
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b = 0;
"
# Should still fail because of matching unavailable shard
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 2 AND b = 2;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
# Try more complext expressions for constant folding - all should pass.
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 1 AND a = 0 AND b = 0;
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a IN (0, 1) AND b IN (0, 1);
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b = 0 OR a = 1 AND b = 1;
"
# TODO: should pass one day.
#${CLICKHOUSE_CLIENT} -n --query="
#${CLICKHOUSE_CLIENT} --query="
# SET optimize_skip_unused_shards = 1;
# SELECT count(*) FROM distributed WHERE a = 0 AND b >= 0 AND b <= 1;
#"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b = 0 AND c = 0;
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b = 0 AND c != 10;
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b = 0 AND (a+b)*b != 12;
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE (a = 0 OR a = 1) AND (b = 0 OR b = 1);
"
# These ones should fail.
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b <= 1;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND c = 0;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 OR a = 1 AND b = 0;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b = 0 OR a = 2 AND b = 2;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b = 0 OR c = 0;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'

26
tests/queries/0_stateless/00754_distributed_optimize_skip_select_on_unused_shards_with_prewhere.sh
View File

@ -30,73 +30,73 @@ ${CLICKHOUSE_CLIENT} --query "SELECT count(*) FROM distributed_00754 PREWHERE a
| grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
# Should pass now
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 0 AND b = 0;
"
# Should still fail because of matching unavailable shard
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 2 AND b = 2;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
# Try more complex expressions for constant folding - all should pass.
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 1 AND a = 0 WHERE b = 0;
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 1 WHERE b = 1 AND length(c) = 5;
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a IN (0, 1) AND b IN (0, 1) WHERE c LIKE '%l%';
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a IN (0, 1) WHERE b IN (0, 1) AND c LIKE '%l%';
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 0 AND b = 0 OR a = 1 AND b = 1 WHERE c LIKE '%l%';
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE (a = 0 OR a = 1) WHERE (b = 0 OR b = 1);
"
# These should fail.
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 0 AND b <= 1;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 0 WHERE c LIKE '%l%';
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 0 OR a = 1 AND b = 0;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 0 AND b = 0 OR a = 2 AND b = 2;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 0 AND b = 0 OR c LIKE '%l%';
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'

4
tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sh
View File

@ -10,7 +10,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
SHARD=$($CLICKHOUSE_CLIENT --query "Select getMacro('shard')")
REPLICA=$($CLICKHOUSE_CLIENT --query "Select getMacro('replica')")
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
DROP TABLE IF EXISTS part_header_r1;
DROP TABLE IF EXISTS part_header_r2;
@ -62,7 +62,7 @@ do
[[ $count1 == 1 && $count2 == 1 ]] && break
done
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
SELECT '*** Test part removal ***';
SELECT '*** replica 1 ***';

2
tests/queries/0_stateless/00837_minmax_index.sh
View File

@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE minmax_idx
(
u64 UInt64,

2
tests/queries/0_stateless/00838_unique_index.sh
View File

@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS set_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE set_idx
(
u64 UInt64,

2
tests/queries/0_stateless/00907_set_index_max_rows.sh
View File

@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS set_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE set_idx
(
u64 UInt64,

10
tests/queries/0_stateless/00908_bloom_filter_index.sh
View File

@ -12,7 +12,7 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS bloom_filter_idx3;"
# NGRAM BF
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE bloom_filter_idx
(
k UInt64,
@ -22,7 +22,7 @@ CREATE TABLE bloom_filter_idx
ORDER BY k
SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi';"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE bloom_filter_idx2
(
k UInt64,
@ -109,7 +109,7 @@ $CLICKHOUSE_CLIENT --optimize_or_like_chain 0 --query="SELECT count() FROM bloom
# TOKEN BF
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE bloom_filter_idx3
(
k UInt64,
@ -147,7 +147,7 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE bloom_filter_idx2"
$CLICKHOUSE_CLIENT --query="DROP TABLE bloom_filter_idx3"
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS bloom_filter_idx_na;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE bloom_filter_idx_na
(
na Array(Array(String)),
@ -156,7 +156,7 @@ CREATE TABLE bloom_filter_idx_na
ORDER BY na" 2>&1 | grep -c 'DB::Exception: Unexpected type Array(Array(String)) of bloom filter index'
# NGRAM BF with IPv6
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE bloom_filter_ipv6_idx
(
foo IPv6,

2
tests/queries/0_stateless/00942_mutate_index.sh
View File

@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE minmax_idx
(
u64 UInt64,

4
tests/queries/0_stateless/00943_materialize_index.sh
View File

@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE minmax_idx
(
u64 UInt64,
@ -34,7 +34,7 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO minmax_idx VALUES
$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;"
$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
ALTER TABLE minmax_idx ADD INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1 SETTINGS mutations_sync = 2;"
$CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx MATERIALIZE INDEX idx IN PARTITION 1 SETTINGS mutations_sync = 2;"

2
tests/queries/0_stateless/00944_clear_index_in_partition.sh
View File

@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE minmax_idx
(
u64 UInt64,

2
tests/queries/0_stateless/00964_bloom_index_string_functions.sh
View File

@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS bloom_filter_idx;"
# NGRAM BF
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE bloom_filter_idx
(
k UInt64,

2
tests/queries/0_stateless/00965_set_index_string_functions.sh
View File

@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS set_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE set_idx
(
k UInt64,

4
tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh
View File

@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS lowString;"
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS string;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
create table lowString
(
a LowCardinality(String),
@ -18,7 +18,7 @@ ENGINE = MergeTree()
PARTITION BY toYYYYMM(b)
ORDER BY (a)"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
create table string
(
a String,

2
tests/queries/0_stateless/00990_hasToken.sh
View File

@ -6,4 +6,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# We should have correct env vars from shell_config.sh to run this test
python3 "$CURDIR"/00990_hasToken.python | ${CLICKHOUSE_CLIENT} --max_query_size 1048576 -nm
python3 "$CURDIR"/00990_hasToken.python | ${CLICKHOUSE_CLIENT} --max_query_size 1048576 -m

6
tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh
View File

@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
R1=table_1013_1
R2=table_1013_2
${CLICKHOUSE_CLIENT} -n -q "
${CLICKHOUSE_CLIENT} -q "
DROP TABLE IF EXISTS $R1;
DROP TABLE IF EXISTS $R2;
@ -19,13 +19,13 @@ ${CLICKHOUSE_CLIENT} -n -q "
INSERT INTO $R1 VALUES (1)
"
timeout 10s ${CLICKHOUSE_CLIENT} -n -q "
timeout 10s ${CLICKHOUSE_CLIENT} -q "
SET receive_timeout=1;
SYSTEM SYNC REPLICA $R2
" 2>&1 | grep -F -q "Code: 159. DB::Exception" && echo 'OK' || echo 'Failed!'
# By dropping tables all related SYNC REPLICA queries would be terminated as well
${CLICKHOUSE_CLIENT} -n -q "
${CLICKHOUSE_CLIENT} -q "
DROP TABLE IF EXISTS $R2;
DROP TABLE IF EXISTS $R1;
"

4
tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh
View File

@ -9,7 +9,7 @@ R1=table_1017_1
R2=table_1017_2
T1=table_1017_merge
${CLICKHOUSE_CLIENT} -n -q "
${CLICKHOUSE_CLIENT} -q "
DROP DICTIONARY IF EXISTS dict1;
DROP TABLE IF EXISTS $R1;
DROP TABLE IF EXISTS $R2;
@ -68,7 +68,7 @@ ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $R1 DELETE WHERE dictHas('${CLICKHOUSE
${CLICKHOUSE_CLIENT} --query "ALTER TABLE $R1 DELETE WHERE dictHas('${CLICKHOUSE_DATABASE}.dict1', toUInt64(x))" --allow_nondeterministic_mutations=1 2>&1 \
&& echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n -q "
${CLICKHOUSE_CLIENT} -q "
DROP DICTIONARY IF EXISTS dict1;
DROP TABLE IF EXISTS $R2;
DROP TABLE IF EXISTS $R1;

6
tests/queries/0_stateless/01037_polygon_dicts_correctness_all.sh
View File

@ -14,7 +14,7 @@ declare -a SearchTypes=("POLYGON" "POLYGON_SIMPLE" "POLYGON_INDEX_EACH" "POLYGON
tar -xf "${CURDIR}"/01037_test_data_search.tar.gz -C "${DATA_DIR}"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP TABLE IF EXISTS points;
CREATE TABLE points (x Float64, y Float64) ENGINE = Memory;
"
@ -23,7 +23,7 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO points FORMAT TSV" --max_insert_block_si
rm "${DATA_DIR}"/01037_point_data
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP TABLE IF EXISTS polygons_array;
CREATE TABLE polygons_array
@ -43,7 +43,7 @@ for type in "${SearchTypes[@]}";
do
outputFile="${TMP_DIR}/results${type}.out"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP DICTIONARY IF EXISTS dict_array;
CREATE DICTIONARY dict_array

6
tests/queries/0_stateless/01037_polygon_dicts_correctness_fast.sh
View File

@ -14,7 +14,7 @@ declare -a SearchTypes=("POLYGON_INDEX_EACH" "POLYGON_INDEX_CELL")
tar -xf "${CURDIR}"/01037_test_data_perf.tar.gz -C "${DATA_DIR}"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE points (x Float64, y Float64) ENGINE = Memory;
"
@ -22,7 +22,7 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO points FORMAT TSV" --min_chunk_bytes_for
rm "${DATA_DIR}"/01037_point_data
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP TABLE IF EXISTS polygons_array;
CREATE TABLE polygons_array
@ -42,7 +42,7 @@ for type in "${SearchTypes[@]}";
do
outputFile="${TMP_DIR}/results${type}.out"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP DICTIONARY IF EXISTS dict_array;
CREATE DICTIONARY dict_array

6
tests/queries/0_stateless/01037_polygon_dicts_simple_functions.sh
View File

@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
TMP_DIR=${CLICKHOUSE_TMP}/tmp
mkdir -p $TMP_DIR
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP TABLE IF EXISTS polygons_array;
CREATE TABLE polygons_array (key Array(Array(Array(Array(Float64)))), name String, value UInt64) ENGINE = Memory;
@ -53,7 +53,7 @@ for type in "${SearchTypes[@]}";
do
outputFile="${TMP_DIR}/results${type}.out"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP DICTIONARY IF EXISTS dict_array;
CREATE DICTIONARY dict_array
(
@ -106,7 +106,7 @@ do
diff -q "${CURDIR}/01037_polygon_dicts_simple_functions.ans" "$outputFile"
done
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP DICTIONARY dict_array;
DROP DICTIONARY dict_tuple;
DROP TABLE polygons_array;

2
tests/queries/0_stateless/01055_minmax_index_compact_parts.sh
View File

@ -10,7 +10,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE minmax_idx
(
u64 UInt64,

10
tests/queries/0_stateless/01077_mutations_index_consistency.sh
View File

@ -7,13 +7,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS movement"
$CLICKHOUSE_CLIENT -n --query "CREATE TABLE movement (date DateTime('Asia/Istanbul')) Engine = MergeTree ORDER BY (toStartOfHour(date)) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';"
$CLICKHOUSE_CLIENT --query "CREATE TABLE movement (date DateTime('Asia/Istanbul')) Engine = MergeTree ORDER BY (toStartOfHour(date)) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';"
$CLICKHOUSE_CLIENT --query "insert into movement select toDateTime('2020-01-22 00:00:00', 'Asia/Istanbul') + number%(23*3600) from numbers(1000000);"
$CLICKHOUSE_CLIENT --query "OPTIMIZE TABLE movement FINAL"
$CLICKHOUSE_CLIENT -n --query "
$CLICKHOUSE_CLIENT --query "
SELECT
count(),
toStartOfHour(date) AS Hour
@ -26,7 +26,7 @@ ORDER BY Hour DESC
$CLICKHOUSE_CLIENT --query "alter table movement delete where date >= toDateTime('2020-01-22T16:00:00', 'Asia/Istanbul') and date < toDateTime('2020-01-22T17:00:00', 'Asia/Istanbul') SETTINGS mutations_sync = 2"
$CLICKHOUSE_CLIENT -n --query "
$CLICKHOUSE_CLIENT --query "
SELECT
count(),
toStartOfHour(date) AS Hour
@ -37,7 +37,7 @@ ORDER BY Hour DESC
" | grep "16:00:00" | wc -l
$CLICKHOUSE_CLIENT -n --query "
$CLICKHOUSE_CLIENT --query "
SELECT
count(),
toStartOfHour(date) AS Hour
@ -48,7 +48,7 @@ ORDER BY Hour DESC
" | grep "22:00:00" | cut -f1
$CLICKHOUSE_CLIENT -n --query "
$CLICKHOUSE_CLIENT --query "
SELECT
count(),
toStartOfHour(date) AS Hour

8
tests/queries/0_stateless/01187_set_profile_as_setting.sh
View File

@ -7,11 +7,11 @@ CLICKHOUSE_LOG_COMMENT=
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -n -m -q "select value, changed from system.settings where name='readonly';"
$CLICKHOUSE_CLIENT -n -m -q "set profile='default'; select value, changed from system.settings where name='readonly';"
$CLICKHOUSE_CLIENT -n -m -q "set profile='readonly'; select value, changed from system.settings where name='readonly';" 2>&1| grep -Fa "Cannot modify 'send_logs_level' setting in readonly mode" > /dev/null && echo "OK"
$CLICKHOUSE_CLIENT -m -q "select value, changed from system.settings where name='readonly';"
$CLICKHOUSE_CLIENT -m -q "set profile='default'; select value, changed from system.settings where name='readonly';"
$CLICKHOUSE_CLIENT -m -q "set profile='readonly'; select value, changed from system.settings where name='readonly';" 2>&1| grep -Fa "Cannot modify 'send_logs_level' setting in readonly mode" > /dev/null && echo "OK"
CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=fatal/g')
$CLICKHOUSE_CLIENT -n -m -q "set profile='readonly'; select value, changed from system.settings where name='readonly';"
$CLICKHOUSE_CLIENT -m -q "set profile='readonly'; select value, changed from system.settings where name='readonly';"
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=select+value,changed+from+system.settings+where+name='readonly'"
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&profile=default&query=select+value,changed+from+system.settings+where+name='readonly'"

2
tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh
View File

@ -10,7 +10,7 @@ set -o pipefail
# shellcheck disable=SC2120
function execute()
{
${CLICKHOUSE_CLIENT} -n "$@"
${CLICKHOUSE_CLIENT} "$@"
}
#

10
tests/queries/0_stateless/01307_multiple_leaders_zookeeper.sh
View File

@ -12,8 +12,8 @@ DATA_SIZE=200
SEQ=$(seq 0 $(($NUM_REPLICAS - 1)))
for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "DROP TABLE IF EXISTS r$REPLICA"; done
for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE r$REPLICA (x UInt64) ENGINE = ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table', 'r$REPLICA') ORDER BY x SETTINGS min_bytes_for_wide_part = '10M';"; done
for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS r$REPLICA"; done
for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT --query "CREATE TABLE r$REPLICA (x UInt64) ENGINE = ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table', 'r$REPLICA') ORDER BY x SETTINGS min_bytes_for_wide_part = '10M';"; done
function thread()
{
@ -30,6 +30,6 @@ done
wait
for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "SYSTEM SYNC REPLICA r$REPLICA"; done
for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "SELECT count(), sum(x) FROM r$REPLICA"; done
for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "DROP TABLE r$REPLICA"; done
for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA r$REPLICA"; done
for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT --query "SELECT count(), sum(x) FROM r$REPLICA"; done
for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT --query "DROP TABLE r$REPLICA"; done

2
tests/queries/0_stateless/01415_sticking_mutations.sh
View File

@ -11,7 +11,7 @@ $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS sticking_mutations"
function check_sticky_mutations()
{
$CLICKHOUSE_CLIENT -n --query "CREATE TABLE sticking_mutations (
$CLICKHOUSE_CLIENT --query "CREATE TABLE sticking_mutations (
date Date,
key UInt64,
value1 String,

2
tests/queries/0_stateless/01451_dist_logs.sh
View File

@ -10,4 +10,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# triggered not for the first query
for _ in {1..20}; do
echo "select * from remote('127.{2,3}', system.numbers) where number = 10 limit 1;"
done | ${CLICKHOUSE_CLIENT} -n 2>/dev/null
done | ${CLICKHOUSE_CLIENT} 2>/dev/null

4
tests/queries/0_stateless/01459_manual_write_to_replicas.sh
View File

@ -10,7 +10,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
NUM_REPLICAS=10
for i in $(seq 1 $NUM_REPLICAS); do
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
DROP TABLE IF EXISTS r$i SYNC;
CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', 'r$i') ORDER BY x;
"
@ -31,7 +31,7 @@ done
wait
for i in $(seq 1 $NUM_REPLICAS); do
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
SYSTEM SYNC REPLICA r$i;
SELECT count(), min(x), max(x), sum(x) FROM r$i;"
done

4
tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh
View File

@ -16,7 +16,7 @@ unset CLICKHOUSE_WRITE_COVERAGE
NUM_REPLICAS=10
for i in $(seq 1 $NUM_REPLICAS); do
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
DROP TABLE IF EXISTS r$i SYNC;
CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', 'r$i') ORDER BY x;
"
@ -39,7 +39,7 @@ done
wait
for i in $(seq 1 $NUM_REPLICAS); do
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
SYSTEM SYNC REPLICA r$i;
SELECT count(), min(x), max(x), sum(x) FROM r$i;"
done

4
tests/queries/0_stateless/01459_manual_write_to_replicas_quorum_detach_attach.sh
View File

@ -11,7 +11,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
NUM_REPLICAS=6
for i in $(seq 1 $NUM_REPLICAS); do
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
DROP TABLE IF EXISTS r$i SYNC;
CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', 'r$i') ORDER BY x;
"
@ -36,7 +36,7 @@ done
wait
for i in $(seq 1 $NUM_REPLICAS); do
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
SYSTEM SYNC REPLICA r$i;
SELECT count(), min(x), max(x), sum(x) FROM r$i;"
done

2
tests/queries/0_stateless/01507_clickhouse_server_start_with_embedded_config.sh
View File

@ -34,7 +34,7 @@ done
# Check access rights
$CLICKHOUSE_CLIENT -n --query "
$CLICKHOUSE_CLIENT --query "
DROP DATABASE IF EXISTS test;
CREATE DATABASE test;
USE test;

4
tests/queries/0_stateless/01508_format_regexp_raw.sh
View File

@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} -n --query "
${CLICKHOUSE_CLIENT} --query "
DROP TABLE IF EXISTS t;
CREATE TABLE t (a String, b String) ENGINE = Memory;
"
@ -12,7 +12,7 @@ CREATE TABLE t (a String, b String) ENGINE = Memory;
${CLICKHOUSE_CLIENT} --format_regexp_escaping_rule 'Raw' --format_regexp '^(.+?) separator (.+?)$' --query '
INSERT INTO t FORMAT Regexp abc\ separator Hello, world!'
${CLICKHOUSE_CLIENT} -n --query "
${CLICKHOUSE_CLIENT} --query "
SELECT * FROM t;
DROP TABLE t;
"

2
tests/queries/0_stateless/01509_dictionary_preallocate.sh
View File

@ -15,7 +15,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# PREALLOCATE attribute (and also for the history/greppability, that it was
# such).
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
DROP TABLE IF EXISTS data_01509;
DROP DICTIONARY IF EXISTS dict_01509;
CREATE TABLE data_01509

4
tests/queries/0_stateless/01510_format_regexp_raw_low_cardinality.sh
View File

@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} -n --query "
${CLICKHOUSE_CLIENT} --query "
DROP TABLE IF EXISTS t;
CREATE TABLE t (a String, b LowCardinality(Nullable(String))) ENGINE = Memory;
"
@ -12,7 +12,7 @@ CREATE TABLE t (a String, b LowCardinality(Nullable(String))) ENGINE = Memory;
${CLICKHOUSE_CLIENT} --format_regexp_escaping_rule 'Raw' --format_regexp '^(.+?) separator (.+?)$' --query '
INSERT INTO t FORMAT Regexp abc\ separator Hello, world!'
${CLICKHOUSE_CLIENT} -n --query "
${CLICKHOUSE_CLIENT} --query "
SELECT * FROM t;
DROP TABLE t;
"

2
tests/queries/0_stateless/01526_initial_query_id.sh
View File

@ -15,7 +15,7 @@ ${CLICKHOUSE_CURL} \
--get \
--data-urlencode "query=select 1 format Null"
${CLICKHOUSE_CLIENT} -n -q "
${CLICKHOUSE_CLIENT} -q "
system flush logs;
select interface, initial_query_id = query_id
from system.query_log

2
tests/queries/0_stateless/01599_mutation_query_params.sh
View File

@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -n --query "
$CLICKHOUSE_CLIENT --query "
DROP TABLE IF EXISTS test;
CREATE TABLE test

4
tests/queries/0_stateless/01600_quota_by_forwarded_ip.sh
View File

@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -n --query "
$CLICKHOUSE_CLIENT --query "
CREATE USER quoted_by_ip_${CLICKHOUSE_DATABASE};
CREATE USER quoted_by_forwarded_ip_${CLICKHOUSE_DATABASE};
@ -57,7 +57,7 @@ ${CLICKHOUSE_CURL} -H 'X-Forwarded-For: 5.6.7.8, 1.2.3.4' -sS "${CLICKHOUSE_URL}
${CLICKHOUSE_CURL} -H 'X-Forwarded-For: 1.2.3.4, 5.6.7.8' -sS "${CLICKHOUSE_URL}&user=quoted_by_forwarded_ip_${CLICKHOUSE_DATABASE}" -d "SELECT count() FROM numbers(10)"
$CLICKHOUSE_CLIENT -n --query "
$CLICKHOUSE_CLIENT --query "
DROP QUOTA IF EXISTS quota_by_ip_${CLICKHOUSE_DATABASE};
DROP QUOTA IF EXISTS quota_by_forwarded_ip;

2
tests/queries/0_stateless/01684_ssd_cache_dictionary_simple_key.sh
View File

@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP DATABASE IF EXISTS 01684_database_for_cache_dictionary;
CREATE DATABASE 01684_database_for_cache_dictionary;

2
tests/queries/0_stateless/01685_ssd_cache_dictionary_complex_key.sh
View File

@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE complex_key_simple_attributes_source_table
(
id UInt64,

2
tests/queries/0_stateless/01691_parser_data_type_exponential.sh
View File

File diff suppressed because one or more lines are too long

2
tests/queries/0_stateless/01710_projections_optimize_aggregation_in_order.sh
View File

@ -4,7 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
DROP TABLE IF EXISTS in_order_agg_01710;
CREATE TABLE in_order_agg_01710

2
tests/queries/0_stateless/01710_projections_partial_optimize_aggregation_in_order.sh
View File

@ -9,7 +9,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
DROP TABLE IF EXISTS in_order_agg_partial_01710;
CREATE TABLE in_order_agg_partial_01710

2
tests/queries/0_stateless/01753_optimize_aggregation_in_order.sh
View File

@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT --optimize_aggregation_in_order=1 -nm -q "
$CLICKHOUSE_CLIENT --optimize_aggregation_in_order=1 -m -q "
drop table if exists data_01753;
create table data_01753 (key Int) engine=MergeTree() order by key as select * from numbers(8);
select * from data_01753 group by key settings max_block_size=1;

8
tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference
View File

@ -49,7 +49,7 @@ order by query;
tuple(2)
select 'optimize_skip_unused_shards_rewrite_in(2,)';
optimize_skip_unused_shards_rewrite_in(2,)
with (select currentDatabase()) as id_2 select *, ignore(id_2) from dist_01756 where dummy in (2,);
with (select currentDatabase()) as id_2 select *, ignore(id_2) from dist_01756 where dummy in (2);
system flush logs;
select splitByString('IN', query)[-1] from system.query_log where
event_date >= yesterday() and
@ -59,10 +59,10 @@ select splitByString('IN', query)[-1] from system.query_log where
query like concat('%', currentDatabase(), '%AS%id_2%') and
type = 'QueryFinish'
order by query;
tuple(2)
(2)
select 'optimize_skip_unused_shards_rewrite_in(0,)';
optimize_skip_unused_shards_rewrite_in(0,)
with (select currentDatabase()) as id_00 select *, ignore(id_00) from dist_01756 where dummy in (0,);
with (select currentDatabase()) as id_00 select *, ignore(id_00) from dist_01756 where dummy in (0);
0 0
system flush logs;
select splitByString('IN', query)[-1] from system.query_log where
@ -73,7 +73,7 @@ select splitByString('IN', query)[-1] from system.query_log where
query like concat('%', currentDatabase(), '%AS%id_00%') and
type = 'QueryFinish'
order by query;
tuple(0)
(0)
-- signed column
select 'signed column';
signed column

4
tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql
View File

@ -63,7 +63,7 @@ select splitByString('IN', query)[-1] from system.query_log where
order by query;
select 'optimize_skip_unused_shards_rewrite_in(2,)';
with (select currentDatabase()) as id_2 select *, ignore(id_2) from dist_01756 where dummy in (2,);
with (select currentDatabase()) as id_2 select *, ignore(id_2) from dist_01756 where dummy in (2);
system flush logs;
select splitByString('IN', query)[-1] from system.query_log where
event_date >= yesterday() and
@ -75,7 +75,7 @@ select splitByString('IN', query)[-1] from system.query_log where
order by query;
select 'optimize_skip_unused_shards_rewrite_in(0,)';
with (select currentDatabase()) as id_00 select *, ignore(id_00) from dist_01756 where dummy in (0,);
with (select currentDatabase()) as id_00 select *, ignore(id_00) from dist_01756 where dummy in (0);
system flush logs;
select splitByString('IN', query)[-1] from system.query_log where
event_date >= yesterday() and

2
tests/queries/0_stateless/01758_optimize_skip_unused_shards_once.sh
View File

@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT --optimize_skip_unused_shards=1 -nm -q "
$CLICKHOUSE_CLIENT --optimize_skip_unused_shards=1 -m -q "
create table dist_01758 as system.one engine=Distributed(test_cluster_two_shards, system, one, dummy);
select * from dist_01758 where dummy = 0 format Null;
" |& grep -o "StorageDistributed (dist_01758).*"

2
tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.sh
View File

@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 -nm -q "
$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 -m -q "
DROP TABLE IF EXISTS tmp_01683;
DROP TABLE IF EXISTS dist_01683;

6
tests/queries/0_stateless/01814_distributed_push_down_limit.sh
View File

@ -13,7 +13,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
function setup()
{
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
drop table if exists data_01814;
drop table if exists dist_01814;
@ -24,7 +24,7 @@ function setup()
function cleanup()
{
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
drop table data_01814;
drop table dist_01814;
"
@ -67,7 +67,7 @@ function test_distributed_push_down_limit_with_query_log()
$CLICKHOUSE_CLIENT "${settings_and_opts[@]}" -q "select * from $table group by key limit $offset, 10"
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
system flush logs;
select read_rows from system.query_log
where

8
tests/queries/0_stateless/01853_dictionary_cache_duplicates.sh
View File

@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
function run_test_once()
{
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
DROP TABLE IF EXISTS simple_key_source_table_01863;
CREATE TABLE simple_key_source_table_01863
(
@ -29,13 +29,13 @@ function run_test_once()
LIFETIME(MIN 0 MAX 1000);
"
prev=$($CLICKHOUSE_CLIENT -nm -q "SELECT value FROM system.events WHERE event = 'DictCacheKeysRequestedMiss' SETTINGS system_events_show_zero_values=1")
curr=$($CLICKHOUSE_CLIENT -nm -q "
prev=$($CLICKHOUSE_CLIENT -m -q "SELECT value FROM system.events WHERE event = 'DictCacheKeysRequestedMiss' SETTINGS system_events_show_zero_values=1")
curr=$($CLICKHOUSE_CLIENT -m -q "
SELECT toUInt64(1) as key, dictGet('simple_key_cache_dictionary_01863', 'value', key) FORMAT Null;
SELECT value FROM system.events WHERE event = 'DictCacheKeysRequestedMiss' SETTINGS system_events_show_zero_values=1
")
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
DROP DICTIONARY simple_key_cache_dictionary_01863;
DROP TABLE simple_key_source_table_01863;
"

2
tests/queries/0_stateless/01872_initial_query_start_time.sh
View File

@ -13,7 +13,7 @@ ${CLICKHOUSE_CLIENT} -q "create table m (dummy UInt8) ENGINE = Distributed('test
query_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))")
${CLICKHOUSE_CLIENT} -q "select * from m format Null" "--query_id=$query_id"
${CLICKHOUSE_CLIENT} -n -q "
${CLICKHOUSE_CLIENT} -q "
system flush logs;
select
anyIf(initial_query_start_time, is_initial_query) = anyIf(initial_query_start_time, not is_initial_query),

2
tests/queries/0_stateless/01875_ssd_cache_dictionary_decimal256_type.sh
View File

@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET allow_experimental_bigint_types = 1;
DROP TABLE IF EXISTS dictionary_decimal_source_table;

2
tests/queries/0_stateless/01890_materialized_distributed_join.sh
View File

@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
drop table if exists test_distributed;
drop table if exists test_source;
drop table if exists test_shard;

2
tests/queries/0_stateless/01903_ssd_cache_dictionary_array_type.sh
View File

@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP TABLE IF EXISTS dictionary_array_source_table;
CREATE TABLE dictionary_array_source_table
(

2
tests/queries/0_stateless/01904_ssd_cache_dictionary_default_nullable_type.sh
View File

@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP TABLE IF EXISTS dictionary_nullable_source_table;
CREATE TABLE dictionary_nullable_source_table
(

4
tests/queries/0_stateless/01927_query_views_log_matview_exceptions.sh
View File

@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
function cleanup()
{
${CLICKHOUSE_CLIENT} -n -q "
${CLICKHOUSE_CLIENT} -q "
DROP TABLE IF EXISTS matview_exception_a_to_c;
DROP TABLE IF EXISTS matview_exception_a_to_b;
DROP TABLE IF EXISTS table_exception_c;
@ -17,7 +17,7 @@ function cleanup()
function setup()
{
${CLICKHOUSE_CLIENT} -n -q "
${CLICKHOUSE_CLIENT} -q "
CREATE TABLE table_exception_a (a String, b Int64) ENGINE = MergeTree ORDER BY b;
CREATE TABLE table_exception_b (a Float64, b Int64) ENGINE = MergeTree ORDER BY tuple();
CREATE TABLE table_exception_c (a Float64) ENGINE = MergeTree ORDER BY a;

4
tests/queries/0_stateless/01947_multiple_pipe_read.sh
View File

@ -11,7 +11,7 @@ cat "$SAMPLE_FILE"
echo '******************'
echo 'Read twice from a regular file'
${CLICKHOUSE_LOCAL} --structure 'x UInt64, s String' -n -q 'select * from table; select * from table;' --file "$SAMPLE_FILE"
${CLICKHOUSE_LOCAL} --structure 'x UInt64, s String' -q 'select * from table; select * from table;' --file "$SAMPLE_FILE"
echo '---'
${CLICKHOUSE_LOCAL} --structure 'x UInt64, s String' -q 'select * from table WHERE x IN (select x from table);' --file "$SAMPLE_FILE"
echo '---'
@ -19,7 +19,7 @@ ${CLICKHOUSE_LOCAL} --structure 'x UInt64, s String' -q 'select * from table UNI
echo '******************'
echo 'Read twice from file descriptor that corresponds to a regular file'
${CLICKHOUSE_LOCAL} --structure 'x UInt64, s String' -n -q 'select * from table; select * from table;' < "$SAMPLE_FILE"
${CLICKHOUSE_LOCAL} --structure 'x UInt64, s String' -q 'select * from table; select * from table;' < "$SAMPLE_FILE"
echo '---'
${CLICKHOUSE_LOCAL} --structure 'x UInt64, s String' -q 'select * from table WHERE x IN (select x from table);' < "$SAMPLE_FILE"
echo '---'

32
tests/queries/0_stateless/02003_memory_limit_in_client.sh
View File

@ -4,21 +4,21 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT --max_memory_usage_in_client=1 -n -q "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000) -- { clientError MEMORY_LIMIT_EXCEEDED }"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client=0 -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client=1 -q "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000) -- { clientError MEMORY_LIMIT_EXCEEDED }"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client=0 -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='5K' -n -q "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000) -- { clientError MEMORY_LIMIT_EXCEEDED }"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='5k' -n -q "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000) -- { clientError MEMORY_LIMIT_EXCEEDED }"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='1M' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='23G' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='11T' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='5K' -q "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000) -- { clientError MEMORY_LIMIT_EXCEEDED }"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='5k' -q "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000) -- { clientError MEMORY_LIMIT_EXCEEDED }"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='1M' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='23G' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='11T' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='2P' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='2.1p' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='10E' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='10.2e' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='-1.1T' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_NUMBER"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='-1' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_NUMBER"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='1m' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='14g' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='11t' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='2P' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='2.1p' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='10E' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='10.2e' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='-1.1T' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_NUMBER"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='-1' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_NUMBER"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='1m' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='14g' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='11t' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"

2
tests/queries/0_stateless/02021_create_database_with_comment.sh
View File

@ -20,7 +20,7 @@ function test_db_comments()
local ENGINE_NAME="$1"
echo "engine : ${ENGINE_NAME}"
$CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -nm <<EOF
$CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -m <<EOF
DROP DATABASE IF EXISTS ${DB_NAME};
CREATE DATABASE ${DB_NAME} ENGINE = ${ENGINE_NAME} COMMENT 'Test DB with comment';
EOF

8
tests/queries/0_stateless/02050_client_profile_events.sh
View File

@ -11,14 +11,14 @@ echo 'print only last (and also number of rows to provide more info in case of f
$CLICKHOUSE_CLIENT --max_block_size=65505 --print-profile-events --profile-events-delay-ms=-1 -q 'select * from numbers(1e5)' |& grep -o -e '\[ 0 \] SelectedRows: .*$' -e Exception
echo 'regression test for incorrect filtering out snapshots'
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -n -q 'select 1; select 1' >& /dev/null
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q 'select 1; select 1' >& /dev/null
echo $?
echo 'regression test for overlap profile events snapshots between queries'
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -n -q 'select 1; select 1' |& grep -F -o '[ 0 ] SelectedRows: 1 (increment)'
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q 'select 1; select 1' |& grep -F -o '[ 0 ] SelectedRows: 1 (increment)'
echo 'regression test for overlap profile events snapshots between queries (clickhouse-local)'
$CLICKHOUSE_LOCAL --print-profile-events --profile-events-delay-ms=-1 -n -q 'select 1; select 1' |& grep -F -o '[ 0 ] SelectedRows: 1 (increment)'
$CLICKHOUSE_LOCAL --print-profile-events --profile-events-delay-ms=-1 -q 'select 1; select 1' |& grep -F -o '[ 0 ] SelectedRows: 1 (increment)'
echo 'print everything'
profile_events="$(
@ -35,5 +35,5 @@ profile_events="$(
test "$profile_events" -gt 1 && echo OK || echo "FAIL ($profile_events)"
echo 'check that ProfileEvents is new for each query'
sleep_function_calls=$($CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -n -q 'select sleep(1); select 1' |& grep -c 'SleepFunctionCalls')
sleep_function_calls=$($CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q 'select sleep(1); select 1' |& grep -c 'SleepFunctionCalls')
test "$sleep_function_calls" -eq 1 && echo OK || echo "FAIL ($sleep_function_calls)"

2
tests/queries/0_stateless/02210_toColumnTypeName_toLowCardinality_const.reference
View File

@ -1 +1 @@
Const(ColumnLowCardinality)
Const(LowCardinality(UInt8))

Some files were not shown because too many files have changed in this diff Show More