mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 01:25:21 +00:00
Merge branch 'master' into fix-fuzzer-html
This commit is contained in:
Alexey Milovidov
GitHub
commit
44de7195d1
@ -377,15 +377,15 @@ set (DEBUG_INFO_FLAGS "-g -gdwarf-4")
|
||||
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS}")
|
||||
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
|
||||
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
|
||||
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
|
||||
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${CMAKE_C_FLAGS_ADD}")
|
||||
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}")
|
||||
set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_C_FLAGS_ADD}")
|
||||
set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}")
|
||||
|
||||
set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} ${COMPILER_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
|
||||
set (CMAKE_ASM_FLAGS_RELWITHDEBINFO "${CMAKE_ASM_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
|
||||
set (CMAKE_ASM_FLAGS_DEBUG "${CMAKE_ASM_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_ASM_FLAGS_ADD}")
|
||||
set (CMAKE_ASM_FLAGS_DEBUG "${CMAKE_ASM_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
|
||||
|
||||
if (COMPILER_CLANG)
|
||||
if (OS_DARWIN)
|
||||
|
||||
@ -402,10 +402,11 @@ ReplxxLineReader::ReplxxLineReader(
|
||||
words.push_back(hs.get().text());
|
||||
}
|
||||
|
||||
std::string current_query(rx.get_state().text());
|
||||
std::string new_query;
|
||||
try
|
||||
{
|
||||
new_query = std::string(skim(words));
|
||||
new_query = std::string(skim(current_query, words));
|
||||
}
|
||||
catch (const std::exception & e)
|
||||
{
|
||||
|
||||
@ -43,7 +43,10 @@ set_target_properties(unwind PROPERTIES FOLDER "contrib/libunwind-cmake")
|
||||
|
||||
target_include_directories(unwind SYSTEM BEFORE PUBLIC $<BUILD_INTERFACE:${LIBUNWIND_SOURCE_DIR}/include>)
|
||||
target_compile_definitions(unwind PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY)
|
||||
target_compile_options(unwind PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all $<$<COMPILE_LANGUAGE:CXX>:-nostdinc++ -fno-rtti>)
|
||||
|
||||
# We should enable optimizations (otherwise it will be too slow in debug)
|
||||
# and disable sanitizers (otherwise infinite loop may happen)
|
||||
target_compile_options(unwind PRIVATE -O3 -fno-exceptions -funwind-tables -fno-sanitize=all $<$<COMPILE_LANGUAGE:CXX>:-nostdinc++ -fno-rtti>)
|
||||
|
||||
check_c_compiler_flag(-Wunused-but-set-variable HAVE_WARNING_UNUSED_BUT_SET_VARIABLE)
|
||||
if (HAVE_WARNING_UNUSED_BUT_SET_VARIABLE)
|
||||
|
||||
2
contrib/sysroot
vendored
2
contrib/sysroot
vendored
@ -1 +1 @@
|
||||
Subproject commit e9fb375d0a1e5ebfd74c043f088f2342552103f8
|
||||
Subproject commit 0f41651860fa4a530ecd68b93a15b8fd77397adf
|
||||
@ -2,6 +2,7 @@
|
||||
<profiles>
|
||||
<default>
|
||||
<max_execution_time>10</max_execution_time>
|
||||
|
||||
<!--
|
||||
Don't let the fuzzer change this setting (I've actually seen it
|
||||
do this before).
|
||||
@ -14,6 +15,11 @@
|
||||
<max_memory_usage>
|
||||
<max>10G</max>
|
||||
</max_memory_usage>
|
||||
|
||||
<!-- Analyzer is unstable, not ready for testing. -->
|
||||
<allow_experimental_analyzer>
|
||||
<readonly/>
|
||||
</allow_experimental_analyzer>
|
||||
</constraints>
|
||||
</default>
|
||||
</profiles>
|
||||
|
||||
15
docker/test/fuzzer/run-fuzzer.sh
Executable file → Normal file
15
docker/test/fuzzer/run-fuzzer.sh
Executable file → Normal file
@ -257,12 +257,21 @@ quit
|
||||
if [ "$server_died" == 1 ]
|
||||
then
|
||||
# The server has died.
|
||||
task_exit_code=210
|
||||
echo "failure" > status.txt
|
||||
if ! zgrep --text -ao "Received signal.*\|Logical error.*\|Assertion.*failed\|Failed assertion.*\|.*runtime error: .*\|.*is located.*\|SUMMARY: AddressSanitizer:.*\|SUMMARY: MemorySanitizer:.*\|SUMMARY: ThreadSanitizer:.*\|.*_LIBCPP_ASSERT.*" server.log.gz > description.txt
|
||||
then
|
||||
echo "Lost connection to server. See the logs." > description.txt
|
||||
fi
|
||||
|
||||
if grep -F --text 'Sanitizer: out-of-memory' description.txt
|
||||
then
|
||||
# OOM of sanitizer is not a problem we can handle - treat it as success, but preserve the description.
|
||||
task_exit_code=0
|
||||
echo "success" > status.txt
|
||||
else
|
||||
task_exit_code=210
|
||||
echo "failure" > status.txt
|
||||
fi
|
||||
|
||||
elif [ "$fuzzer_exit_code" == "143" ] || [ "$fuzzer_exit_code" == "0" ]
|
||||
then
|
||||
# Variants of a normal run:
|
||||
@ -355,7 +364,7 @@ th { cursor: pointer; }
|
||||
|
||||
<h1>AST Fuzzer for PR <a href="https://github.com/ClickHouse/ClickHouse/pull/${PR_TO_TEST}">#${PR_TO_TEST}</a> @ ${SHA_TO_TEST}</h1>
|
||||
<p class="links">
|
||||
<a href="runlog.log">runlog.log</a>
|
||||
<a href="run.log">run.log</a>
|
||||
<a href="fuzzer.log">fuzzer.log</a>
|
||||
<a href="server.log.gz">server.log.gz</a>
|
||||
<a href="main.log">main.log</a>
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
Allow to run simple ClickHouse stress test in Docker from debian packages.
|
||||
Allows to run simple ClickHouse stress test in Docker from debian packages.
|
||||
Actually it runs multiple copies of clickhouse-test (functional tests).
|
||||
This allows to find problems like segmentation fault which cause shutdown of server.
|
||||
This allows to find problems like failed assertions and memory safety issues.
|
||||
|
||||
Usage:
|
||||
```
|
||||
|
||||
@ -123,6 +123,22 @@ EOL
|
||||
<core_path>$PWD</core_path>
|
||||
</clickhouse>
|
||||
EOL
|
||||
|
||||
# Analyzer is not yet ready for testing
|
||||
cat > /etc/clickhouse-server/users.d/no_analyzer.xml <<EOL
|
||||
<clickhouse>
|
||||
<profiles>
|
||||
<default>
|
||||
<constraints>
|
||||
<allow_experimental_analyzer>
|
||||
<readonly/>
|
||||
</allow_experimental_analyzer>
|
||||
</constraints>
|
||||
</default>
|
||||
</profiles>
|
||||
</clickhouse>
|
||||
EOL
|
||||
|
||||
}
|
||||
|
||||
function stop()
|
||||
@ -334,219 +350,221 @@ zgrep -Fa "########################################" /test_output/* > /dev/null
|
||||
zgrep -Fa " received signal " /test_output/gdb.log > /dev/null \
|
||||
&& echo -e 'Found signal in gdb.log\tFAIL' >> /test_output/test_results.tsv
|
||||
|
||||
echo -e "Backward compatibility check\n"
|
||||
if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
|
||||
echo -e "Backward compatibility check\n"
|
||||
|
||||
echo "Get previous release tag"
|
||||
previous_release_tag=$(clickhouse-client --version | grep -o "[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*" | get_previous_release_tag)
|
||||
echo $previous_release_tag
|
||||
echo "Get previous release tag"
|
||||
previous_release_tag=$(clickhouse-client --version | grep -o "[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*" | get_previous_release_tag)
|
||||
echo $previous_release_tag
|
||||
|
||||
echo "Clone previous release repository"
|
||||
git clone https://github.com/ClickHouse/ClickHouse.git --no-tags --progress --branch=$previous_release_tag --no-recurse-submodules --depth=1 previous_release_repository
|
||||
echo "Clone previous release repository"
|
||||
git clone https://github.com/ClickHouse/ClickHouse.git --no-tags --progress --branch=$previous_release_tag --no-recurse-submodules --depth=1 previous_release_repository
|
||||
|
||||
echo "Download previous release server"
|
||||
mkdir previous_release_package_folder
|
||||
echo "Download previous release server"
|
||||
mkdir previous_release_package_folder
|
||||
|
||||
echo $previous_release_tag | download_release_packets && echo -e 'Download script exit code\tOK' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Download script failed\tFAIL' >> /test_output/test_results.tsv
|
||||
echo $previous_release_tag | download_release_packets && echo -e 'Download script exit code\tOK' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Download script failed\tFAIL' >> /test_output/test_results.tsv
|
||||
|
||||
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.clean.log
|
||||
for table in query_log trace_log
|
||||
do
|
||||
clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.tsv.gz ||:
|
||||
done
|
||||
|
||||
tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
|
||||
|
||||
# Check if we cloned previous release repository successfully
|
||||
if ! [ "$(ls -A previous_release_repository/tests/queries)" ]
|
||||
then
|
||||
echo -e "Backward compatibility check: Failed to clone previous release tests\tFAIL" >> /test_output/test_results.tsv
|
||||
elif ! [ "$(ls -A previous_release_package_folder/clickhouse-common-static_*.deb && ls -A previous_release_package_folder/clickhouse-server_*.deb)" ]
|
||||
then
|
||||
echo -e "Backward compatibility check: Failed to download previous release packets\tFAIL" >> /test_output/test_results.tsv
|
||||
else
|
||||
echo -e "Successfully cloned previous release tests\tOK" >> /test_output/test_results.tsv
|
||||
echo -e "Successfully downloaded previous release packets\tOK" >> /test_output/test_results.tsv
|
||||
|
||||
# Uninstall current packages
|
||||
dpkg --remove clickhouse-client
|
||||
dpkg --remove clickhouse-server
|
||||
dpkg --remove clickhouse-common-static-dbg
|
||||
dpkg --remove clickhouse-common-static
|
||||
|
||||
rm -rf /var/lib/clickhouse/*
|
||||
|
||||
# Make BC check more funny by forcing Ordinary engine for system database
|
||||
mkdir /var/lib/clickhouse/metadata
|
||||
echo "ATTACH DATABASE system ENGINE=Ordinary" > /var/lib/clickhouse/metadata/system.sql
|
||||
|
||||
# Install previous release packages
|
||||
install_packages previous_release_package_folder
|
||||
|
||||
# Start server from previous release
|
||||
# Previous version may not be ready for fault injections
|
||||
export ZOOKEEPER_FAULT_INJECTION=0
|
||||
configure
|
||||
|
||||
# Avoid "Setting s3_check_objects_after_upload is neither a builtin setting..."
|
||||
rm -f /etc/clickhouse-server/users.d/enable_blobs_check.xml ||:
|
||||
rm -f /etc/clickhouse-server/users.d/marks.xml ||:
|
||||
|
||||
# Remove s3 related configs to avoid "there is no disk type `cache`"
|
||||
rm -f /etc/clickhouse-server/config.d/storage_conf.xml ||:
|
||||
rm -f /etc/clickhouse-server/config.d/azure_storage_conf.xml ||:
|
||||
|
||||
# Turn on after 22.12
|
||||
rm -f /etc/clickhouse-server/config.d/compressed_marks_and_index.xml ||:
|
||||
# it uses recently introduced settings which previous versions may not have
|
||||
rm -f /etc/clickhouse-server/users.d/insert_keeper_retries.xml ||:
|
||||
|
||||
start
|
||||
|
||||
clickhouse-client --query="SELECT 'Server version: ', version()"
|
||||
|
||||
# Install new package before running stress test because we should use new
|
||||
# clickhouse-client and new clickhouse-test.
|
||||
#
|
||||
# But we should leave old binary in /usr/bin/ and debug symbols in
|
||||
# /usr/lib/debug/usr/bin (if any) for gdb and internal DWARF parser, so it
|
||||
# will print sane stacktraces and also to avoid possible crashes.
|
||||
#
|
||||
# FIXME: those files can be extracted directly from debian package, but
|
||||
# actually better solution will be to use different PATH instead of playing
|
||||
# games with files from packages.
|
||||
mv /usr/bin/clickhouse previous_release_package_folder/
|
||||
mv /usr/lib/debug/usr/bin/clickhouse.debug previous_release_package_folder/
|
||||
install_packages package_folder
|
||||
mv /usr/bin/clickhouse package_folder/
|
||||
mv /usr/lib/debug/usr/bin/clickhouse.debug package_folder/
|
||||
mv previous_release_package_folder/clickhouse /usr/bin/
|
||||
mv previous_release_package_folder/clickhouse.debug /usr/lib/debug/usr/bin/clickhouse.debug
|
||||
|
||||
mkdir tmp_stress_output
|
||||
|
||||
./stress --test-cmd="/usr/bin/clickhouse-test --queries=\"previous_release_repository/tests/queries\"" --backward-compatibility-check --output-folder tmp_stress_output --global-time-limit=1200 \
|
||||
&& echo -e 'Backward compatibility check: Test script exit code\tOK' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: Test script failed\tFAIL' >> /test_output/test_results.tsv
|
||||
rm -rf tmp_stress_output
|
||||
|
||||
clickhouse-client --query="SELECT 'Tables count:', count() FROM system.tables"
|
||||
|
||||
stop 1
|
||||
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.stress.log
|
||||
|
||||
# Start new server
|
||||
mv package_folder/clickhouse /usr/bin/
|
||||
mv package_folder/clickhouse.debug /usr/lib/debug/usr/bin/clickhouse.debug
|
||||
export ZOOKEEPER_FAULT_INJECTION=1
|
||||
configure
|
||||
start 500
|
||||
clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK'" >> /test_output/test_results.tsv \
|
||||
|| (echo -e 'Backward compatibility check: Server failed to start\tFAIL' >> /test_output/test_results.tsv \
|
||||
&& grep -a "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log >> /test_output/bc_check_application_errors.txt)
|
||||
|
||||
clickhouse-client --query="SELECT 'Server version: ', version()"
|
||||
|
||||
# Let the server run for a while before checking log.
|
||||
sleep 60
|
||||
|
||||
stop
|
||||
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.clean.log
|
||||
|
||||
# Error messages (we should ignore some errors)
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/38643 ("Unknown index: idx.")
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 ("Cannot parse string 'Hello' as UInt64")
|
||||
# FIXME Not sure if it's expected, but some tests from BC check may not be finished yet when we restarting server.
|
||||
# Let's just ignore all errors from queries ("} <Error> TCPHandler: Code:", "} <Error> executeQuery: Code:")
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39197 ("Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'")
|
||||
# NOTE Incompatibility was introduced in https://github.com/ClickHouse/ClickHouse/pull/39263, it's expected
|
||||
# ("This engine is deprecated and is not supported in transactions", "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part")
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 - bad mutation does not indicate backward incompatibility
|
||||
echo "Check for Error messages in server log:"
|
||||
zgrep -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \
|
||||
-e "Code: 236. DB::Exception: Cancelled mutating parts" \
|
||||
-e "REPLICA_IS_ALREADY_ACTIVE" \
|
||||
-e "REPLICA_ALREADY_EXISTS" \
|
||||
-e "ALL_REPLICAS_LOST" \
|
||||
-e "DDLWorker: Cannot parse DDL task query" \
|
||||
-e "RaftInstance: failed to accept a rpc connection due to error 125" \
|
||||
-e "UNKNOWN_DATABASE" \
|
||||
-e "NETWORK_ERROR" \
|
||||
-e "UNKNOWN_TABLE" \
|
||||
-e "ZooKeeperClient" \
|
||||
-e "KEEPER_EXCEPTION" \
|
||||
-e "DirectoryMonitor" \
|
||||
-e "TABLE_IS_READ_ONLY" \
|
||||
-e "Code: 1000, e.code() = 111, Connection refused" \
|
||||
-e "UNFINISHED" \
|
||||
-e "NETLINK_ERROR" \
|
||||
-e "Renaming unexpected part" \
|
||||
-e "PART_IS_TEMPORARILY_LOCKED" \
|
||||
-e "and a merge is impossible: we didn't find" \
|
||||
-e "found in queue and some source parts for it was lost" \
|
||||
-e "is lost forever." \
|
||||
-e "Unknown index: idx." \
|
||||
-e "Cannot parse string 'Hello' as UInt64" \
|
||||
-e "} <Error> TCPHandler: Code:" \
|
||||
-e "} <Error> executeQuery: Code:" \
|
||||
-e "Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'" \
|
||||
-e "This engine is deprecated and is not supported in transactions" \
|
||||
-e "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part" \
|
||||
-e "The set of parts restored in place of" \
|
||||
-e "(ReplicatedMergeTreeAttachThread): Initialization failed. Error" \
|
||||
-e "Code: 269. DB::Exception: Destination table is myself" \
|
||||
-e "Coordination::Exception: Connection loss" \
|
||||
-e "MutateFromLogEntryTask" \
|
||||
-e "No connection to ZooKeeper, cannot get shared table ID" \
|
||||
-e "Session expired" \
|
||||
/var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
|
||||
&& echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
# Remove file bc_check_error_messages.txt if it's empty
|
||||
[ -s /test_output/bc_check_error_messages.txt ] || rm /test_output/bc_check_error_messages.txt
|
||||
|
||||
# Sanitizer asserts
|
||||
zgrep -Fa "==================" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
|
||||
zgrep -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
|
||||
zgrep -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \
|
||||
&& echo -e 'Backward compatibility check: Sanitizer assert (in stderr.log)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No sanitizer asserts\tOK' >> /test_output/test_results.tsv
|
||||
rm -f /test_output/tmp
|
||||
|
||||
# OOM
|
||||
zgrep -Fa " <Fatal> Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
|
||||
&& echo -e 'Backward compatibility check: OOM killer (or signal 9) in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
# Logical errors
|
||||
echo "Check for Logical errors in server log:"
|
||||
zgrep -Fa -A20 "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_logical_errors.txt \
|
||||
&& echo -e 'Backward compatibility check: Logical error thrown (see clickhouse-server.log or bc_check_logical_errors.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No logical errors\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
# Remove file bc_check_logical_errors.txt if it's empty
|
||||
[ -s /test_output/bc_check_logical_errors.txt ] || rm /test_output/bc_check_logical_errors.txt
|
||||
|
||||
# Crash
|
||||
zgrep -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
|
||||
&& echo -e 'Backward compatibility check: Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: Not crashed\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
# It also checks for crash without stacktrace (printed by watchdog)
|
||||
echo "Check for Fatal message in server log:"
|
||||
zgrep -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_fatal_messages.txt \
|
||||
&& echo -e 'Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
# Remove file bc_check_fatal_messages.txt if it's empty
|
||||
[ -s /test_output/bc_check_fatal_messages.txt ] || rm /test_output/bc_check_fatal_messages.txt
|
||||
|
||||
tar -chf /test_output/coordination.backward.tar /var/lib/clickhouse/coordination ||:
|
||||
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.clean.log
|
||||
for table in query_log trace_log
|
||||
do
|
||||
clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.backward.tsv.gz ||:
|
||||
clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.tsv.gz ||:
|
||||
done
|
||||
|
||||
tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
|
||||
|
||||
# Check if we cloned previous release repository successfully
|
||||
if ! [ "$(ls -A previous_release_repository/tests/queries)" ]
|
||||
then
|
||||
echo -e "Backward compatibility check: Failed to clone previous release tests\tFAIL" >> /test_output/test_results.tsv
|
||||
elif ! [ "$(ls -A previous_release_package_folder/clickhouse-common-static_*.deb && ls -A previous_release_package_folder/clickhouse-server_*.deb)" ]
|
||||
then
|
||||
echo -e "Backward compatibility check: Failed to download previous release packets\tFAIL" >> /test_output/test_results.tsv
|
||||
else
|
||||
echo -e "Successfully cloned previous release tests\tOK" >> /test_output/test_results.tsv
|
||||
echo -e "Successfully downloaded previous release packets\tOK" >> /test_output/test_results.tsv
|
||||
|
||||
# Uninstall current packages
|
||||
dpkg --remove clickhouse-client
|
||||
dpkg --remove clickhouse-server
|
||||
dpkg --remove clickhouse-common-static-dbg
|
||||
dpkg --remove clickhouse-common-static
|
||||
|
||||
rm -rf /var/lib/clickhouse/*
|
||||
|
||||
# Make BC check more funny by forcing Ordinary engine for system database
|
||||
mkdir /var/lib/clickhouse/metadata
|
||||
echo "ATTACH DATABASE system ENGINE=Ordinary" > /var/lib/clickhouse/metadata/system.sql
|
||||
|
||||
# Install previous release packages
|
||||
install_packages previous_release_package_folder
|
||||
|
||||
# Start server from previous release
|
||||
# Previous version may not be ready for fault injections
|
||||
export ZOOKEEPER_FAULT_INJECTION=0
|
||||
configure
|
||||
|
||||
# Avoid "Setting s3_check_objects_after_upload is neither a builtin setting..."
|
||||
rm -f /etc/clickhouse-server/users.d/enable_blobs_check.xml ||:
|
||||
rm -f /etc/clickhouse-server/users.d/marks.xml ||:
|
||||
|
||||
# Remove s3 related configs to avoid "there is no disk type `cache`"
|
||||
rm -f /etc/clickhouse-server/config.d/storage_conf.xml ||:
|
||||
rm -f /etc/clickhouse-server/config.d/azure_storage_conf.xml ||:
|
||||
|
||||
# Turn on after 22.12
|
||||
rm -f /etc/clickhouse-server/config.d/compressed_marks_and_index.xml ||:
|
||||
# it uses recently introduced settings which previous versions may not have
|
||||
rm -f /etc/clickhouse-server/users.d/insert_keeper_retries.xml ||:
|
||||
|
||||
start
|
||||
|
||||
clickhouse-client --query="SELECT 'Server version: ', version()"
|
||||
|
||||
# Install new package before running stress test because we should use new
|
||||
# clickhouse-client and new clickhouse-test.
|
||||
#
|
||||
# But we should leave old binary in /usr/bin/ and debug symbols in
|
||||
# /usr/lib/debug/usr/bin (if any) for gdb and internal DWARF parser, so it
|
||||
# will print sane stacktraces and also to avoid possible crashes.
|
||||
#
|
||||
# FIXME: those files can be extracted directly from debian package, but
|
||||
# actually better solution will be to use different PATH instead of playing
|
||||
# games with files from packages.
|
||||
mv /usr/bin/clickhouse previous_release_package_folder/
|
||||
mv /usr/lib/debug/usr/bin/clickhouse.debug previous_release_package_folder/
|
||||
install_packages package_folder
|
||||
mv /usr/bin/clickhouse package_folder/
|
||||
mv /usr/lib/debug/usr/bin/clickhouse.debug package_folder/
|
||||
mv previous_release_package_folder/clickhouse /usr/bin/
|
||||
mv previous_release_package_folder/clickhouse.debug /usr/lib/debug/usr/bin/clickhouse.debug
|
||||
|
||||
mkdir tmp_stress_output
|
||||
|
||||
./stress --test-cmd="/usr/bin/clickhouse-test --queries=\"previous_release_repository/tests/queries\"" --backward-compatibility-check --output-folder tmp_stress_output --global-time-limit=1200 \
|
||||
&& echo -e 'Backward compatibility check: Test script exit code\tOK' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: Test script failed\tFAIL' >> /test_output/test_results.tsv
|
||||
rm -rf tmp_stress_output
|
||||
|
||||
clickhouse-client --query="SELECT 'Tables count:', count() FROM system.tables"
|
||||
|
||||
stop 1
|
||||
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.stress.log
|
||||
|
||||
# Start new server
|
||||
mv package_folder/clickhouse /usr/bin/
|
||||
mv package_folder/clickhouse.debug /usr/lib/debug/usr/bin/clickhouse.debug
|
||||
export ZOOKEEPER_FAULT_INJECTION=1
|
||||
configure
|
||||
start 500
|
||||
clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK'" >> /test_output/test_results.tsv \
|
||||
|| (echo -e 'Backward compatibility check: Server failed to start\tFAIL' >> /test_output/test_results.tsv \
|
||||
&& grep -a "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log >> /test_output/bc_check_application_errors.txt)
|
||||
|
||||
clickhouse-client --query="SELECT 'Server version: ', version()"
|
||||
|
||||
# Let the server run for a while before checking log.
|
||||
sleep 60
|
||||
|
||||
stop
|
||||
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.clean.log
|
||||
|
||||
# Error messages (we should ignore some errors)
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/38643 ("Unknown index: idx.")
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 ("Cannot parse string 'Hello' as UInt64")
|
||||
# FIXME Not sure if it's expected, but some tests from BC check may not be finished yet when we restarting server.
|
||||
# Let's just ignore all errors from queries ("} <Error> TCPHandler: Code:", "} <Error> executeQuery: Code:")
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39197 ("Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'")
|
||||
# NOTE Incompatibility was introduced in https://github.com/ClickHouse/ClickHouse/pull/39263, it's expected
|
||||
# ("This engine is deprecated and is not supported in transactions", "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part")
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 - bad mutation does not indicate backward incompatibility
|
||||
echo "Check for Error messages in server log:"
|
||||
zgrep -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \
|
||||
-e "Code: 236. DB::Exception: Cancelled mutating parts" \
|
||||
-e "REPLICA_IS_ALREADY_ACTIVE" \
|
||||
-e "REPLICA_ALREADY_EXISTS" \
|
||||
-e "ALL_REPLICAS_LOST" \
|
||||
-e "DDLWorker: Cannot parse DDL task query" \
|
||||
-e "RaftInstance: failed to accept a rpc connection due to error 125" \
|
||||
-e "UNKNOWN_DATABASE" \
|
||||
-e "NETWORK_ERROR" \
|
||||
-e "UNKNOWN_TABLE" \
|
||||
-e "ZooKeeperClient" \
|
||||
-e "KEEPER_EXCEPTION" \
|
||||
-e "DirectoryMonitor" \
|
||||
-e "TABLE_IS_READ_ONLY" \
|
||||
-e "Code: 1000, e.code() = 111, Connection refused" \
|
||||
-e "UNFINISHED" \
|
||||
-e "NETLINK_ERROR" \
|
||||
-e "Renaming unexpected part" \
|
||||
-e "PART_IS_TEMPORARILY_LOCKED" \
|
||||
-e "and a merge is impossible: we didn't find" \
|
||||
-e "found in queue and some source parts for it was lost" \
|
||||
-e "is lost forever." \
|
||||
-e "Unknown index: idx." \
|
||||
-e "Cannot parse string 'Hello' as UInt64" \
|
||||
-e "} <Error> TCPHandler: Code:" \
|
||||
-e "} <Error> executeQuery: Code:" \
|
||||
-e "Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'" \
|
||||
-e "This engine is deprecated and is not supported in transactions" \
|
||||
-e "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part" \
|
||||
-e "The set of parts restored in place of" \
|
||||
-e "(ReplicatedMergeTreeAttachThread): Initialization failed. Error" \
|
||||
-e "Code: 269. DB::Exception: Destination table is myself" \
|
||||
-e "Coordination::Exception: Connection loss" \
|
||||
-e "MutateFromLogEntryTask" \
|
||||
-e "No connection to ZooKeeper, cannot get shared table ID" \
|
||||
-e "Session expired" \
|
||||
/var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
|
||||
&& echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
# Remove file bc_check_error_messages.txt if it's empty
|
||||
[ -s /test_output/bc_check_error_messages.txt ] || rm /test_output/bc_check_error_messages.txt
|
||||
|
||||
# Sanitizer asserts
|
||||
zgrep -Fa "==================" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
|
||||
zgrep -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
|
||||
zgrep -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \
|
||||
&& echo -e 'Backward compatibility check: Sanitizer assert (in stderr.log)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No sanitizer asserts\tOK' >> /test_output/test_results.tsv
|
||||
rm -f /test_output/tmp
|
||||
|
||||
# OOM
|
||||
zgrep -Fa " <Fatal> Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
|
||||
&& echo -e 'Backward compatibility check: OOM killer (or signal 9) in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
# Logical errors
|
||||
echo "Check for Logical errors in server log:"
|
||||
zgrep -Fa -A20 "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_logical_errors.txt \
|
||||
&& echo -e 'Backward compatibility check: Logical error thrown (see clickhouse-server.log or bc_check_logical_errors.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No logical errors\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
# Remove file bc_check_logical_errors.txt if it's empty
|
||||
[ -s /test_output/bc_check_logical_errors.txt ] || rm /test_output/bc_check_logical_errors.txt
|
||||
|
||||
# Crash
|
||||
zgrep -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
|
||||
&& echo -e 'Backward compatibility check: Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: Not crashed\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
# It also checks for crash without stacktrace (printed by watchdog)
|
||||
echo "Check for Fatal message in server log:"
|
||||
zgrep -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_fatal_messages.txt \
|
||||
&& echo -e 'Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
# Remove file bc_check_fatal_messages.txt if it's empty
|
||||
[ -s /test_output/bc_check_fatal_messages.txt ] || rm /test_output/bc_check_fatal_messages.txt
|
||||
|
||||
tar -chf /test_output/coordination.backward.tar /var/lib/clickhouse/coordination ||:
|
||||
for table in query_log trace_log
|
||||
do
|
||||
clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.backward.tsv.gz ||:
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
dmesg -T > /test_output/dmesg.log
|
||||
|
||||
@ -825,6 +825,23 @@ Setting fields:
|
||||
The `table` or `where` fields cannot be used together with the `query` field. And either one of the `table` or `query` fields must be declared.
|
||||
:::
|
||||
|
||||
## Null
|
||||
|
||||
A special source that can be used to create dummy (empty) dictionaries. Such dictionaries can useful for tests or with setups with separated data and query nodes at nodes with Distributed tables.
|
||||
|
||||
``` sql
|
||||
CREATE DICTIONARY null_dict (
|
||||
id UInt64,
|
||||
val UInt8,
|
||||
default_val UInt8 DEFAULT 123,
|
||||
nullable_val Nullable(UInt8)
|
||||
)
|
||||
PRIMARY KEY id
|
||||
SOURCE(NULL())
|
||||
LAYOUT(FLAT())
|
||||
LIFETIME(0);
|
||||
```
|
||||
|
||||
## Related Content
|
||||
|
||||
- [Using dictionaries to accelerate queries](https://clickhouse.com/blog/faster-queries-dictionaries-clickhouse)
|
||||
- [Using dictionaries to accelerate queries](https://clickhouse.com/blog/faster-queries-dictionaries-clickhouse)
|
||||
|
||||
@ -1044,8 +1044,8 @@ try
|
||||
bool continue_if_corrupted = config().getBool("merge_tree_metadata_cache.continue_if_corrupted", false);
|
||||
try
|
||||
{
|
||||
LOG_DEBUG(
|
||||
log, "Initializing merge tree metadata cache lru_cache_size:{} continue_if_corrupted:{}", size, continue_if_corrupted);
|
||||
LOG_DEBUG(log, "Initializing MergeTree metadata cache, lru_cache_size: {} continue_if_corrupted: {}",
|
||||
ReadableSize(size), continue_if_corrupted);
|
||||
global_context->initializeMergeTreeMetadataCache(path_str + "/" + "rocksdb", size);
|
||||
}
|
||||
catch (...)
|
||||
|
||||
1
programs/server/config.d/graphite.xml
Symbolic link
1
programs/server/config.d/graphite.xml
Symbolic link
@ -0,0 +1 @@
|
||||
../../../tests/config/config.d/graphite.xml
|
||||
@ -87,4 +87,4 @@ private:
|
||||
} // namespace cxxbridge1
|
||||
} // namespace rust
|
||||
|
||||
::rust::String skim(::std::vector<::std::string> const &words);
|
||||
::rust::String skim(::std::string const &prefix, ::std::vector<::std::string> const &words);
|
||||
|
||||
@ -5,7 +5,7 @@ use cxx::{CxxString, CxxVector};
|
||||
#[cxx::bridge]
|
||||
mod ffi {
|
||||
extern "Rust" {
|
||||
fn skim(words: &CxxVector<CxxString>) -> Result<String>;
|
||||
fn skim(prefix: &CxxString, words: &CxxVector<CxxString>) -> Result<String>;
|
||||
}
|
||||
}
|
||||
|
||||
@ -18,7 +18,7 @@ impl SkimItem for Item {
|
||||
}
|
||||
}
|
||||
|
||||
fn skim(words: &CxxVector<CxxString>) -> Result<String, String> {
|
||||
fn skim(prefix: &CxxString, words: &CxxVector<CxxString>) -> Result<String, String> {
|
||||
// Let's check is terminal available. To avoid panic.
|
||||
if let Err(err) = TermInfo::from_env() {
|
||||
return Err(format!("{}", err));
|
||||
@ -26,6 +26,7 @@ fn skim(words: &CxxVector<CxxString>) -> Result<String, String> {
|
||||
|
||||
let options = SkimOptionsBuilder::default()
|
||||
.height(Some("30%"))
|
||||
.query(Some(prefix.to_str().unwrap()))
|
||||
.tac(true)
|
||||
.tiebreak(Some("-score".to_string()))
|
||||
.build()
|
||||
|
||||
@ -344,7 +344,7 @@ set_source_files_properties(
|
||||
Common/Elf.cpp
|
||||
Common/Dwarf.cpp
|
||||
Common/SymbolIndex.cpp
|
||||
PROPERTIES COMPILE_FLAGS "-O3 ${WITHOUT_COVERAGE}")
|
||||
PROPERTIES COMPILE_FLAGS "-O2 ${WITHOUT_COVERAGE}")
|
||||
|
||||
target_link_libraries (clickhouse_common_io
|
||||
PRIVATE
|
||||
|
||||
@ -2,12 +2,7 @@
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/setThreadName.h>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/filesystemHelpers.h>
|
||||
#include <Common/getCurrentProcessFDCount.h>
|
||||
#include <Common/getMaxFileDescriptorCount.h>
|
||||
#include <Interpreters/Cache/FileCache.h>
|
||||
#include <Server/ProtocolServerAdapter.h>
|
||||
#include <IO/UncompressedCache.h>
|
||||
#include <IO/MMappedFileCache.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
|
||||
@ -565,6 +565,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
|
||||
M(UInt64, max_distributed_depth, 5, "Maximum distributed query depth", 0) \
|
||||
M(Bool, database_replicated_always_detach_permanently, false, "Execute DETACH TABLE as DETACH TABLE PERMANENTLY if database engine is Replicated", 0) \
|
||||
M(Bool, database_replicated_allow_only_replicated_engine, false, "Allow to create only Replicated tables in database with engine Replicated", 0) \
|
||||
M(Bool, database_replicated_allow_replicated_engine_arguments, true, "Allow to create only Replicated tables in database with engine Replicated with explicit arguments", 0) \
|
||||
M(DistributedDDLOutputMode, distributed_ddl_output_mode, DistributedDDLOutputMode::THROW, "Format of distributed DDL query result", 0) \
|
||||
M(UInt64, distributed_ddl_entry_format_version, 3, "Compatibility version of distributed DDL (ON CLUSTER) queries", 0) \
|
||||
\
|
||||
|
||||
@ -178,12 +178,9 @@ __attribute__((__weak__)) void collectCrashLog(
|
||||
class SignalListener : public Poco::Runnable
|
||||
{
|
||||
public:
|
||||
enum Signals : int
|
||||
{
|
||||
StdTerminate = -1,
|
||||
StopThread = -2,
|
||||
SanitizerTrap = -3,
|
||||
};
|
||||
static constexpr int StdTerminate = -1;
|
||||
static constexpr int StopThread = -2;
|
||||
static constexpr int SanitizerTrap = -3;
|
||||
|
||||
explicit SignalListener(BaseDaemon & daemon_)
|
||||
: log(&Poco::Logger::get("BaseDaemon"))
|
||||
@ -208,7 +205,7 @@ public:
|
||||
// Don't use strsignal here, because it's not thread-safe.
|
||||
LOG_TRACE(log, "Received signal {}", sig);
|
||||
|
||||
if (sig == Signals::StopThread)
|
||||
if (sig == StopThread)
|
||||
{
|
||||
LOG_INFO(log, "Stop SignalListener thread");
|
||||
break;
|
||||
@ -219,7 +216,7 @@ public:
|
||||
BaseDaemon::instance().closeLogs(BaseDaemon::instance().logger());
|
||||
LOG_INFO(log, "Opened new log file after received signal.");
|
||||
}
|
||||
else if (sig == Signals::StdTerminate)
|
||||
else if (sig == StdTerminate)
|
||||
{
|
||||
UInt32 thread_num;
|
||||
std::string message;
|
||||
@ -909,7 +906,7 @@ void BaseDaemon::initializeTerminationAndSignalProcessing()
|
||||
|
||||
void BaseDaemon::logRevision() const
|
||||
{
|
||||
Poco::Logger::root().information("Starting " + std::string{VERSION_FULL}
|
||||
logger().information("Starting " + std::string{VERSION_FULL}
|
||||
+ " (revision: " + std::to_string(ClickHouseRevision::getVersionRevision())
|
||||
+ ", git hash: " + (git_hash.empty() ? "<unknown>" : git_hash)
|
||||
+ ", build id: " + (build_id.empty() ? "<unknown>" : build_id) + ")"
|
||||
|
||||
@ -259,7 +259,7 @@ void SerializationBool::deserializeTextCSV(IColumn & column, ReadBuffer & istr,
|
||||
if (istr.eof())
|
||||
throw Exception("Expected boolean value but get EOF.", ErrorCodes::CANNOT_PARSE_BOOL);
|
||||
|
||||
deserializeImpl(column, istr, settings, [&](ReadBuffer & buf){ return buf.eof() || *buf.position() == settings.csv.delimiter || *buf.position() == '\n'; });
|
||||
deserializeImpl(column, istr, settings, [&](ReadBuffer & buf){ return buf.eof() || *buf.position() == settings.csv.delimiter || *buf.position() == '\n' || *buf.position() == '\r'; });
|
||||
}
|
||||
|
||||
void SerializationBool::serializeTextRaw(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
|
||||
@ -584,7 +584,14 @@ void DatabaseReplicated::checkQueryValid(const ASTPtr & query, ContextPtr query_
|
||||
bool enable_functional_tests_helper = getContext()->getConfigRef().has("_functional_tests_helper_database_replicated_replace_args_macros");
|
||||
|
||||
if (!enable_functional_tests_helper)
|
||||
LOG_WARNING(log, "It's not recommended to explicitly specify zookeeper_path and replica_name in ReplicatedMergeTree arguments");
|
||||
{
|
||||
if (query_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments)
|
||||
LOG_WARNING(log, "It's not recommended to explicitly specify zookeeper_path and replica_name in ReplicatedMergeTree arguments");
|
||||
else
|
||||
throw Exception(ErrorCodes::INCORRECT_QUERY,
|
||||
"It's not allowed to specify explicit zookeeper_path and replica_name for ReplicatedMergeTree arguments in Replicated database. "
|
||||
"If you really want to specify them explicitly, enable setting database_replicated_allow_replicated_engine_arguments.");
|
||||
}
|
||||
|
||||
if (maybe_shard_macros && maybe_replica_macros)
|
||||
return;
|
||||
|
||||
48
src/Dictionaries/NullDictionarySource.cpp
Normal file
48
src/Dictionaries/NullDictionarySource.cpp
Normal file
@ -0,0 +1,48 @@
|
||||
#include "NullDictionarySource.h"
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Processors/Sources/NullSource.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include "DictionarySourceFactory.h"
|
||||
#include "DictionarySourceHelpers.h"
|
||||
#include "DictionaryStructure.h"
|
||||
#include "registerDictionaries.h"
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
NullDictionarySource::NullDictionarySource(Block & sample_block_) : sample_block(sample_block_)
|
||||
{
|
||||
}
|
||||
|
||||
NullDictionarySource::NullDictionarySource(const NullDictionarySource & other) : sample_block(other.sample_block)
|
||||
{
|
||||
}
|
||||
|
||||
QueryPipeline NullDictionarySource::loadAll()
|
||||
{
|
||||
LOG_TRACE(&Poco::Logger::get("NullDictionarySource"), "loadAll {}", toString());
|
||||
return QueryPipeline(std::make_shared<NullSource>(sample_block));
|
||||
}
|
||||
|
||||
|
||||
std::string NullDictionarySource::toString() const
|
||||
{
|
||||
return "Null";
|
||||
}
|
||||
|
||||
|
||||
void registerDictionarySourceNull(DictionarySourceFactory & factory)
|
||||
{
|
||||
auto create_table_source
|
||||
= [=](const DictionaryStructure & /* dict_struct */,
|
||||
const Poco::Util::AbstractConfiguration & /* config */,
|
||||
const std::string & /* config_prefix */,
|
||||
Block & sample_block,
|
||||
ContextPtr /* global_context */,
|
||||
const std::string & /* default_database */,
|
||||
bool /* created_from_ddl*/) -> DictionarySourcePtr { return std::make_unique<NullDictionarySource>(sample_block); };
|
||||
|
||||
factory.registerSource("null", create_table_source);
|
||||
}
|
||||
|
||||
}
|
||||
53
src/Dictionaries/NullDictionarySource.h
Normal file
53
src/Dictionaries/NullDictionarySource.h
Normal file
@ -0,0 +1,53 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Block.h>
|
||||
#include "IDictionarySource.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
/// Allows creating empty dictionary
|
||||
class NullDictionarySource final : public IDictionarySource
|
||||
{
|
||||
public:
|
||||
NullDictionarySource(Block & sample_block_);
|
||||
|
||||
NullDictionarySource(const NullDictionarySource & other);
|
||||
|
||||
QueryPipeline loadAll() override;
|
||||
|
||||
QueryPipeline loadUpdatedAll() override
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method loadUpdatedAll is unsupported for NullDictionarySource");
|
||||
}
|
||||
|
||||
QueryPipeline loadIds(const std::vector<UInt64> & /*ids*/) override
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method loadIds is unsupported for NullDictionarySource");
|
||||
}
|
||||
|
||||
QueryPipeline loadKeys(const Columns & /*key_columns*/, const std::vector<size_t> & /*requested_rows*/) override
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method loadKeys is unsupported for NullDictionarySource");
|
||||
}
|
||||
|
||||
bool isModified() const override { return false; }
|
||||
|
||||
bool supportsSelectiveLoad() const override { return false; }
|
||||
|
||||
///Not supported for NullDictionarySource
|
||||
bool hasUpdateField() const override { return false; }
|
||||
|
||||
DictionarySourcePtr clone() const override { return std::make_shared<NullDictionarySource>(*this); }
|
||||
|
||||
std::string toString() const override;
|
||||
|
||||
private:
|
||||
Block sample_block;
|
||||
};
|
||||
|
||||
}
|
||||
@ -6,6 +6,7 @@ namespace DB
|
||||
|
||||
class DictionarySourceFactory;
|
||||
|
||||
void registerDictionarySourceNull(DictionarySourceFactory & factory);
|
||||
void registerDictionarySourceFile(DictionarySourceFactory & source_factory);
|
||||
void registerDictionarySourceMysql(DictionarySourceFactory & source_factory);
|
||||
void registerDictionarySourceClickHouse(DictionarySourceFactory & source_factory);
|
||||
@ -36,6 +37,7 @@ void registerDictionaries()
|
||||
{
|
||||
{
|
||||
auto & source_factory = DictionarySourceFactory::instance();
|
||||
registerDictionarySourceNull(source_factory);
|
||||
registerDictionarySourceFile(source_factory);
|
||||
registerDictionarySourceMysql(source_factory);
|
||||
registerDictionarySourceClickHouse(source_factory);
|
||||
|
||||
@ -126,7 +126,7 @@ private:
|
||||
|
||||
size_t total_values = 0;
|
||||
size_t pre_values = 0;
|
||||
std::vector<size_t> row_length(input_rows_count);
|
||||
PODArray<size_t> row_length(input_rows_count);
|
||||
|
||||
for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx)
|
||||
{
|
||||
@ -138,6 +138,8 @@ private:
|
||||
row_length[row_idx] = (static_cast<__int128_t>(end_data[row_idx]) - static_cast<__int128_t>(start) - 1) / static_cast<__int128_t>(step) + 1;
|
||||
else if (start > end_data[row_idx] && step < 0)
|
||||
row_length[row_idx] = (static_cast<__int128_t>(end_data[row_idx]) - static_cast<__int128_t>(start) + 1) / static_cast<__int128_t>(step) + 1;
|
||||
else
|
||||
row_length[row_idx] = 0;
|
||||
|
||||
pre_values += row_length[row_idx];
|
||||
|
||||
@ -161,8 +163,11 @@ private:
|
||||
IColumn::Offset offset{};
|
||||
for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx)
|
||||
{
|
||||
for (size_t idx = 0; idx < row_length[row_idx]; idx++)
|
||||
out_data[offset++] = static_cast<T>(start + offset * step);
|
||||
for (size_t idx = 0; idx < row_length[row_idx]; ++idx)
|
||||
{
|
||||
out_data[offset] = static_cast<T>(start + offset * step);
|
||||
++offset;
|
||||
}
|
||||
out_offsets[row_idx] = offset;
|
||||
}
|
||||
|
||||
@ -183,7 +188,7 @@ private:
|
||||
|
||||
size_t total_values = 0;
|
||||
size_t pre_values = 0;
|
||||
std::vector<size_t> row_length(input_rows_count);
|
||||
PODArray<size_t> row_length(input_rows_count);
|
||||
|
||||
for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx)
|
||||
{
|
||||
@ -195,7 +200,8 @@ private:
|
||||
row_length[row_idx] = (static_cast<__int128_t>(end_data[row_idx]) - static_cast<__int128_t>(start_data[row_idx]) - 1) / static_cast<__int128_t>(step) + 1;
|
||||
else if (start_data[row_idx] > end_data[row_idx] && step < 0)
|
||||
row_length[row_idx] = (static_cast<__int128_t>(end_data[row_idx]) - static_cast<__int128_t>(start_data[row_idx]) + 1) / static_cast<__int128_t>(step) + 1;
|
||||
|
||||
else
|
||||
row_length[row_idx] = 0;
|
||||
|
||||
pre_values += row_length[row_idx];
|
||||
|
||||
@ -241,7 +247,7 @@ private:
|
||||
|
||||
size_t total_values = 0;
|
||||
size_t pre_values = 0;
|
||||
std::vector<size_t> row_length(input_rows_count);
|
||||
PODArray<size_t> row_length(input_rows_count);
|
||||
|
||||
for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx)
|
||||
{
|
||||
@ -253,6 +259,8 @@ private:
|
||||
row_length[row_idx] = (static_cast<__int128_t>(end_data[row_idx]) - static_cast<__int128_t>(start) - 1) / static_cast<__int128_t>(step_data[row_idx]) + 1;
|
||||
else if (start > end_data[row_idx] && step_data[row_idx] < 0)
|
||||
row_length[row_idx] = (static_cast<__int128_t>(end_data[row_idx]) - static_cast<__int128_t>(start) + 1) / static_cast<__int128_t>(step_data[row_idx]) + 1;
|
||||
else
|
||||
row_length[row_idx] = 0;
|
||||
|
||||
pre_values += row_length[row_idx];
|
||||
|
||||
@ -301,7 +309,7 @@ private:
|
||||
|
||||
size_t total_values = 0;
|
||||
size_t pre_values = 0;
|
||||
std::vector<size_t> row_length(input_rows_count);
|
||||
PODArray<size_t> row_length(input_rows_count);
|
||||
|
||||
for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx)
|
||||
{
|
||||
@ -312,6 +320,8 @@ private:
|
||||
row_length[row_idx] = (static_cast<__int128_t>(end_start[row_idx]) - static_cast<__int128_t>(start_data[row_idx]) - 1) / static_cast<__int128_t>(step_data[row_idx]) + 1;
|
||||
else if (start_data[row_idx] > end_start[row_idx] && step_data[row_idx] < 0)
|
||||
row_length[row_idx] = (static_cast<__int128_t>(end_start[row_idx]) - static_cast<__int128_t>(start_data[row_idx]) + 1) / static_cast<__int128_t>(step_data[row_idx]) + 1;
|
||||
else
|
||||
row_length[row_idx] = 0;
|
||||
|
||||
pre_values += row_length[row_idx];
|
||||
|
||||
|
||||
@ -1098,6 +1098,25 @@ inline String toString(const T & x)
|
||||
return buf.str();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline String toStringWithFinalSeparator(const std::vector<T> & x, const String & final_sep)
|
||||
{
|
||||
WriteBufferFromOwnString buf;
|
||||
for (auto it = x.begin(); it != x.end(); ++it)
|
||||
{
|
||||
if (it != x.begin())
|
||||
{
|
||||
if (std::next(it) == x.end())
|
||||
writeString(final_sep, buf);
|
||||
else
|
||||
writeString(", ", buf);
|
||||
}
|
||||
writeQuoted(*it, buf);
|
||||
}
|
||||
|
||||
return buf.str();
|
||||
}
|
||||
|
||||
inline void writeNullTerminatedString(const String & s, WriteBuffer & buffer)
|
||||
{
|
||||
/// c_str is guaranteed to return zero-terminated string
|
||||
|
||||
32
src/IO/tests/gtest_WriteHelpers.cpp
Normal file
32
src/IO/tests/gtest_WriteHelpers.cpp
Normal file
@ -0,0 +1,32 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
using namespace DB;
|
||||
|
||||
|
||||
TEST(WriteHelpersTest, ToStringWithFinalSeparatorTest)
|
||||
{
|
||||
{
|
||||
std::vector<std::string> v;
|
||||
EXPECT_EQ(toStringWithFinalSeparator(v, " or "), "");
|
||||
}
|
||||
{
|
||||
std::vector<std::string> v = {"AAA"};
|
||||
EXPECT_EQ(toStringWithFinalSeparator(v, " or "), "'AAA'");
|
||||
}
|
||||
{
|
||||
std::vector<std::string> v = {"AAA", "BBB"};
|
||||
EXPECT_EQ(toStringWithFinalSeparator(v, " or "), "'AAA' or 'BBB'");
|
||||
}
|
||||
{
|
||||
std::vector<std::string> v = {"AAA", "BBB", "CCC"};
|
||||
EXPECT_EQ(toStringWithFinalSeparator(v, " or "), "'AAA', 'BBB' or 'CCC'");
|
||||
}
|
||||
{
|
||||
std::vector<std::string> v = {"AAA", "BBB", "CCC", "DDD"};
|
||||
EXPECT_EQ(toStringWithFinalSeparator(v, " or "), "'AAA', 'BBB', 'CCC' or 'DDD'");
|
||||
}
|
||||
}
|
||||
@ -404,6 +404,8 @@ ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns)
|
||||
column_declaration->children.push_back(column_declaration->default_expression);
|
||||
}
|
||||
|
||||
column_declaration->ephemeral_default = column.default_desc.ephemeral_default;
|
||||
|
||||
if (!column.comment.empty())
|
||||
{
|
||||
column_declaration->comment = std::make_shared<ASTLiteral>(Field(column.comment));
|
||||
@ -540,11 +542,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(
|
||||
final_column_name));
|
||||
|
||||
default_expr_list->children.emplace_back(
|
||||
setAlias(
|
||||
col_decl.default_specifier == "EPHEMERAL" ? /// can be ASTLiteral::value NULL
|
||||
std::make_shared<ASTLiteral>(data_type_ptr->getDefault()) :
|
||||
col_decl.default_expression->clone(),
|
||||
tmp_column_name));
|
||||
setAlias(col_decl.default_expression->clone(), tmp_column_name));
|
||||
}
|
||||
else
|
||||
default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), col_decl.name));
|
||||
@ -590,10 +588,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(
|
||||
visitor.visit(col_decl.default_expression);
|
||||
}
|
||||
|
||||
ASTPtr default_expr =
|
||||
col_decl.default_specifier == "EPHEMERAL" && col_decl.default_expression->as<ASTLiteral>()->value.isNull() ?
|
||||
std::make_shared<ASTLiteral>(DataTypeFactory::instance().get(col_decl.type)->getDefault()) :
|
||||
col_decl.default_expression->clone();
|
||||
ASTPtr default_expr = col_decl.default_expression->clone();
|
||||
|
||||
if (col_decl.type)
|
||||
column.type = name_type_it->type;
|
||||
@ -607,6 +602,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(
|
||||
|
||||
column.default_desc.kind = columnDefaultKindFromString(col_decl.default_specifier);
|
||||
column.default_desc.expression = default_expr;
|
||||
column.default_desc.ephemeral_default = col_decl.ephemeral_default;
|
||||
}
|
||||
else if (col_decl.type)
|
||||
column.type = name_type_it->type;
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
|
||||
#include <Core/Settings.h>
|
||||
#include <Core/NamesAndTypes.h>
|
||||
@ -1229,16 +1230,24 @@ void TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select
|
||||
if (storage)
|
||||
{
|
||||
std::vector<String> hint_name{};
|
||||
std::set<String> helper_hint_name{};
|
||||
for (const auto & name : columns_context.requiredColumns())
|
||||
{
|
||||
auto hints = storage->getHints(name);
|
||||
hint_name.insert(hint_name.end(), hints.begin(), hints.end());
|
||||
for (const auto & hint : hints)
|
||||
{
|
||||
// We want to preserve the ordering of the hints
|
||||
// (as they are ordered by Levenshtein distance)
|
||||
auto [_, inserted] = helper_hint_name.insert(hint);
|
||||
if (inserted)
|
||||
hint_name.push_back(hint);
|
||||
}
|
||||
}
|
||||
|
||||
if (!hint_name.empty())
|
||||
{
|
||||
ss << ", maybe you meant: ";
|
||||
ss << toString(hint_name);
|
||||
ss << toStringWithFinalSeparator(hint_name, " or ");
|
||||
}
|
||||
}
|
||||
else
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
#include <Common/quoteString.h>
|
||||
#include <IO/Operators.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <DataTypes/DataTypeFactory.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -78,7 +79,7 @@ void ASTColumnDeclaration::formatImpl(const FormatSettings & settings, FormatSta
|
||||
if (default_expression)
|
||||
{
|
||||
settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << default_specifier << (settings.hilite ? hilite_none : "");
|
||||
if (default_specifier != "EPHEMERAL" || !default_expression->as<ASTLiteral>()->value.isNull())
|
||||
if (!ephemeral_default)
|
||||
{
|
||||
settings.ostr << ' ';
|
||||
default_expression->formatImpl(settings, state, frame);
|
||||
|
||||
@ -16,6 +16,7 @@ public:
|
||||
std::optional<bool> null_modifier;
|
||||
String default_specifier;
|
||||
ASTPtr default_expression;
|
||||
bool ephemeral_default;
|
||||
ASTPtr comment;
|
||||
ASTPtr codec;
|
||||
ASTPtr ttl;
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTColumnDeclaration.h>
|
||||
#include <Parsers/ASTIdentifier_fwd.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
@ -170,6 +171,7 @@ bool IParserColumnDeclaration<NameParser>::parseImpl(Pos & pos, ASTPtr & node, E
|
||||
ASTPtr type;
|
||||
String default_specifier;
|
||||
std::optional<bool> null_modifier;
|
||||
bool ephemeral_default = false;
|
||||
ASTPtr default_expression;
|
||||
ASTPtr comment_expression;
|
||||
ASTPtr codec_expression;
|
||||
@ -235,8 +237,16 @@ bool IParserColumnDeclaration<NameParser>::parseImpl(Pos & pos, ASTPtr & node, E
|
||||
else if (s_ephemeral.ignore(pos, expected))
|
||||
{
|
||||
default_specifier = s_ephemeral.getName();
|
||||
if (!literal_parser.parse(pos, default_expression, expected) && type)
|
||||
default_expression = std::make_shared<ASTLiteral>(Field());
|
||||
if (!expr_parser.parse(pos, default_expression, expected) && type)
|
||||
{
|
||||
ephemeral_default = true;
|
||||
|
||||
auto default_function = std::make_shared<ASTFunction>();
|
||||
default_function->name = "defaultValueOfTypeName";
|
||||
default_function->arguments = std::make_shared<ASTExpressionList>();
|
||||
default_function->arguments->children.emplace_back(std::make_shared<ASTLiteral>(type->as<ASTFunction>()->formatWithSecretsHidden()));
|
||||
default_expression = default_function;
|
||||
}
|
||||
|
||||
if (!default_expression && !type)
|
||||
return false;
|
||||
@ -302,6 +312,7 @@ bool IParserColumnDeclaration<NameParser>::parseImpl(Pos & pos, ASTPtr & node, E
|
||||
column_declaration->default_specifier = default_specifier;
|
||||
if (default_expression)
|
||||
{
|
||||
column_declaration->ephemeral_default = ephemeral_default;
|
||||
column_declaration->default_expression = default_expression;
|
||||
column_declaration->children.push_back(std::move(default_expression));
|
||||
}
|
||||
|
||||
@ -332,8 +332,7 @@ std::string buildTaggedRegex(std::string regexp_str)
|
||||
* </default>
|
||||
* </graphite_rollup>
|
||||
*/
|
||||
static const Pattern &
|
||||
appendGraphitePattern(
|
||||
static const Pattern & appendGraphitePattern(
|
||||
const Poco::Util::AbstractConfiguration & config,
|
||||
const String & config_element, Patterns & patterns,
|
||||
bool default_rule,
|
||||
|
||||
@ -9,6 +9,12 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
|
||||
static GraphiteRollupSortedAlgorithm::ColumnsDefinition defineColumns(
|
||||
const Block & header, const Graphite::Params & params)
|
||||
{
|
||||
@ -26,6 +32,9 @@ static GraphiteRollupSortedAlgorithm::ColumnsDefinition defineColumns(
|
||||
if (i != def.time_column_num && i != def.value_column_num && i != def.version_column_num)
|
||||
def.unmodified_column_numbers.push_back(i);
|
||||
|
||||
if (!WhichDataType(header.getByPosition(def.value_column_num).type).isFloat64())
|
||||
throw Exception("Only `Float64` data type is allowed for the value column of GraphiteMergeTree", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
return def;
|
||||
}
|
||||
|
||||
|
||||
@ -26,6 +26,7 @@ struct ColumnDefault
|
||||
{
|
||||
ColumnDefaultKind kind = ColumnDefaultKind::Default;
|
||||
ASTPtr expression;
|
||||
bool ephemeral_default = false;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -123,6 +123,7 @@ void ColumnDescription::readText(ReadBuffer & buf)
|
||||
{
|
||||
default_desc.kind = columnDefaultKindFromString(col_ast->default_specifier);
|
||||
default_desc.expression = std::move(col_ast->default_expression);
|
||||
default_desc.ephemeral_default = col_ast->ephemeral_default;
|
||||
}
|
||||
|
||||
if (col_ast->comment)
|
||||
|
||||
@ -1,34 +1,23 @@
|
||||
#include <Storages/MergeTree/MergeTreeData.h>
|
||||
|
||||
#include <Backups/BackupEntriesCollector.h>
|
||||
#include <Backups/BackupEntryFromImmutableFile.h>
|
||||
#include <Backups/BackupEntryFromSmallFile.h>
|
||||
#include <Backups/BackupEntryWrappedWith.h>
|
||||
#include <Backups/IBackup.h>
|
||||
#include <Backups/RestorerFromBackup.h>
|
||||
#include <Compression/CompressedReadBuffer.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeEnum.h>
|
||||
#include <DataTypes/DataTypeLowCardinality.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <DataTypes/DataTypeUUID.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/NestedUtils.h>
|
||||
#include <DataTypes/DataTypeObject.h>
|
||||
#include <DataTypes/ObjectUtils.h>
|
||||
#include <Columns/ColumnObject.h>
|
||||
#include <DataTypes/hasNullable.h>
|
||||
#include <Disks/createVolume.h>
|
||||
#include <Disks/ObjectStorages/DiskObjectStorage.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <IO/ConcatReadBuffer.h>
|
||||
#include <IO/Operators.h>
|
||||
#include <IO/ReadBufferFromMemory.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <IO/S3Common.h>
|
||||
#include <Interpreters/Aggregator.h>
|
||||
#include <Interpreters/ExpressionAnalyzer.h>
|
||||
#include <Interpreters/PartLog.h>
|
||||
@ -56,7 +45,6 @@
|
||||
#include <Storages/MergeTree/MergeTreeDataPartWide.h>
|
||||
#include <Storages/MergeTree/DataPartStorageOnDisk.h>
|
||||
#include <Storages/MergeTree/checkDataPart.h>
|
||||
#include <Storages/MergeTree/localBackup.h>
|
||||
#include <Storages/StorageMergeTree.h>
|
||||
#include <Storages/StorageReplicatedMergeTree.h>
|
||||
#include <Storages/VirtualColumnUtils.h>
|
||||
@ -70,22 +58,17 @@
|
||||
#include <Common/quoteString.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/noexcept_scope.h>
|
||||
#include <Processors/QueryPlan/ReadFromMergeTree.h>
|
||||
#include <Processors/Formats/IInputFormat.h>
|
||||
#include <AggregateFunctions/AggregateFunctionCount.h>
|
||||
#include <Common/scope_guard_safe.h>
|
||||
|
||||
#include <boost/range/adaptor/filtered.hpp>
|
||||
#include <boost/range/algorithm_ext/erase.hpp>
|
||||
#include <boost/algorithm/string/join.hpp>
|
||||
#include <boost/algorithm/string/replace.hpp>
|
||||
|
||||
#include <base/insertAtEnd.h>
|
||||
#include <base/sort.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <iomanip>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <thread>
|
||||
@ -1030,7 +1013,7 @@ void MergeTreeData::loadDataPartsFromDisk(
|
||||
size_t suspicious_broken_parts_bytes = 0;
|
||||
std::atomic<bool> has_adaptive_parts = false;
|
||||
std::atomic<bool> has_non_adaptive_parts = false;
|
||||
std::atomic<bool> has_lightweight_in_parts = false;
|
||||
std::atomic<bool> has_lightweight_deletes_in_parts = false;
|
||||
|
||||
std::mutex mutex;
|
||||
auto load_part = [&](const String & part_name, const DiskPtr & part_disk_ptr)
|
||||
@ -1122,7 +1105,7 @@ void MergeTreeData::loadDataPartsFromDisk(
|
||||
|
||||
/// Check if there is lightweight delete in part
|
||||
if (part->hasLightweightDelete())
|
||||
has_lightweight_in_parts.store(true, std::memory_order_relaxed);
|
||||
has_lightweight_deletes_in_parts.store(true, std::memory_order_relaxed);
|
||||
|
||||
part->modification_time = part_disk_ptr->getLastModified(fs::path(relative_data_path) / part_name).epochTime();
|
||||
/// Assume that all parts are Active, covered parts will be detected and marked as Outdated later
|
||||
@ -1206,7 +1189,7 @@ void MergeTreeData::loadDataPartsFromDisk(
|
||||
|
||||
has_non_adaptive_index_granularity_parts = has_non_adaptive_parts;
|
||||
|
||||
if (has_lightweight_in_parts)
|
||||
if (has_lightweight_deletes_in_parts)
|
||||
has_lightweight_delete_parts.store(true);
|
||||
|
||||
if (suspicious_broken_parts > settings->max_suspicious_broken_parts && !skip_sanity_checks)
|
||||
@ -5999,20 +5982,25 @@ std::optional<ProjectionCandidate> MergeTreeData::getQueryProcessingStageWithAgg
|
||||
if (select_query->interpolate() && !select_query->interpolate()->children.empty())
|
||||
return std::nullopt;
|
||||
|
||||
// Currently projections don't support GROUPING SET yet.
|
||||
if (select_query->group_by_with_grouping_sets)
|
||||
// Projections don't support grouping sets yet.
|
||||
if (select_query->group_by_with_grouping_sets
|
||||
|| select_query->group_by_with_totals
|
||||
|| select_query->group_by_with_rollup
|
||||
|| select_query->group_by_with_cube)
|
||||
return std::nullopt;
|
||||
|
||||
auto query_options = SelectQueryOptions(
|
||||
QueryProcessingStage::WithMergeableState,
|
||||
/* depth */ 1,
|
||||
/* is_subquery_= */ true
|
||||
).ignoreProjections().ignoreAlias();
|
||||
).ignoreProjections().ignoreAlias();
|
||||
|
||||
InterpreterSelectQuery select(
|
||||
query_ptr,
|
||||
query_context,
|
||||
query_options,
|
||||
query_info.prepared_sets);
|
||||
|
||||
const auto & analysis_result = select.getAnalysisResult();
|
||||
|
||||
query_info.prepared_sets = select.getQueryAnalyzer()->getPreparedSets();
|
||||
|
||||
@ -110,7 +110,7 @@ Granules getGranulesToWrite(const MergeTreeIndexGranularity & index_granularity,
|
||||
.is_complete = (rows_left_in_block >= expected_rows_in_mark)
|
||||
});
|
||||
current_row += result.back().rows_to_write;
|
||||
current_mark++;
|
||||
++current_mark;
|
||||
}
|
||||
|
||||
return result;
|
||||
@ -146,6 +146,7 @@ void MergeTreeDataPartWriterCompact::write(const Block & block, const IColumn::P
|
||||
if (compute_granularity)
|
||||
{
|
||||
size_t index_granularity_for_block = computeIndexGranularity(block);
|
||||
assert(index_granularity_for_block >= 1);
|
||||
fillIndexGranularity(index_granularity_for_block, block.rows());
|
||||
}
|
||||
|
||||
|
||||
@ -135,7 +135,9 @@ static size_t computeIndexGranularityImpl(
|
||||
size_t rows_in_block = block.rows();
|
||||
size_t index_granularity_for_block;
|
||||
if (!can_use_adaptive_index_granularity)
|
||||
{
|
||||
index_granularity_for_block = fixed_index_granularity_rows;
|
||||
}
|
||||
else
|
||||
{
|
||||
size_t block_size_in_memory = block.bytes();
|
||||
@ -152,11 +154,13 @@ static size_t computeIndexGranularityImpl(
|
||||
index_granularity_for_block = index_granularity_bytes / size_of_row_in_bytes;
|
||||
}
|
||||
}
|
||||
if (index_granularity_for_block == 0) /// very rare case when index granularity bytes less then single row
|
||||
index_granularity_for_block = 1;
|
||||
|
||||
/// We should be less or equal than fixed index granularity
|
||||
index_granularity_for_block = std::min(fixed_index_granularity_rows, index_granularity_for_block);
|
||||
|
||||
/// very rare case when index granularity bytes less then single row
|
||||
if (index_granularity_for_block == 0)
|
||||
index_granularity_for_block = 1;
|
||||
|
||||
return index_granularity_for_block;
|
||||
}
|
||||
|
||||
|
||||
@ -99,6 +99,15 @@ void MergeTreeSettings::sanityCheck(size_t background_pool_tasks) const
|
||||
background_pool_tasks);
|
||||
}
|
||||
|
||||
// Zero index_granularity is nonsensical.
|
||||
if (index_granularity < 1)
|
||||
{
|
||||
throw Exception(
|
||||
ErrorCodes::BAD_ARGUMENTS,
|
||||
"index_granularity: value {} makes no sense",
|
||||
index_granularity);
|
||||
}
|
||||
|
||||
// The min_index_granularity_bytes value is 1024 b and index_granularity_bytes is 10 mb by default.
|
||||
// If index_granularity_bytes is not disabled i.e > 0 b, then always ensure that it's greater than
|
||||
// min_index_granularity_bytes. This is mainly a safeguard against accidents whereby a really low
|
||||
|
||||
@ -17,7 +17,7 @@ struct ReplicatedMergeTreeLogEntryData;
|
||||
/// (so instead of doing exactly the same merge cluster-wise you can do merge once and fetch ready part)
|
||||
/// Fetches may be desirable for other operational reasons (backup replica without lot of CPU resources).
|
||||
///
|
||||
/// That class allow to take a decisions about preferred strategy for a concreate merge.
|
||||
/// That class allow to take a decisions about preferred strategy for a concrete merge.
|
||||
///
|
||||
/// Since that code is used in shouldExecuteLogEntry we need to be able to:
|
||||
/// 1) make decision fast
|
||||
|
||||
@ -476,7 +476,7 @@ static StoragePtr create(const StorageFactory::Arguments & args)
|
||||
{
|
||||
String graphite_config_name;
|
||||
String error_msg
|
||||
= "Last parameter of GraphiteMergeTree must be name (in single quotes) of element in configuration file with Graphite options";
|
||||
= "Last parameter of GraphiteMergeTree must be the name (in single quotes) of the element in configuration file with the Graphite options";
|
||||
error_msg += getMergeTreeVerboseHelp(is_extended_storage_def);
|
||||
|
||||
if (const auto * ast = engine_args[arg_cnt - 1]->as<ASTLiteral>())
|
||||
|
||||
@ -171,7 +171,6 @@ struct ProjectionCandidate
|
||||
*/
|
||||
struct SelectQueryInfo
|
||||
{
|
||||
|
||||
SelectQueryInfo()
|
||||
: prepared_sets(std::make_shared<PreparedSets>())
|
||||
{}
|
||||
|
||||
@ -95,7 +95,7 @@ if __name__ == "__main__":
|
||||
)
|
||||
logging.info("Going to run %s", run_command)
|
||||
|
||||
run_log_path = os.path.join(temp_path, "runlog.log")
|
||||
run_log_path = os.path.join(temp_path, "run.log")
|
||||
with open(run_log_path, "w", encoding="utf-8") as log:
|
||||
with subprocess.Popen(
|
||||
run_command, shell=True, stderr=log, stdout=log
|
||||
@ -113,7 +113,7 @@ if __name__ == "__main__":
|
||||
)
|
||||
s3_prefix = f"{pr_info.number}/{pr_info.sha}/fuzzer_{check_name_lower}/"
|
||||
paths = {
|
||||
"runlog.log": run_log_path,
|
||||
"run.log": run_log_path,
|
||||
"main.log": os.path.join(workspace_path, "main.log"),
|
||||
"server.log.gz": os.path.join(workspace_path, "server.log.gz"),
|
||||
"fuzzer.log": os.path.join(workspace_path, "fuzzer.log"),
|
||||
@ -130,8 +130,8 @@ if __name__ == "__main__":
|
||||
paths[f] = ""
|
||||
|
||||
report_url = GITHUB_RUN_URL
|
||||
if paths["runlog.log"]:
|
||||
report_url = paths["runlog.log"]
|
||||
if paths["run.log"]:
|
||||
report_url = paths["run.log"]
|
||||
if paths["main.log"]:
|
||||
report_url = paths["main.log"]
|
||||
if paths["server.log.gz"]:
|
||||
|
||||
@ -57,7 +57,7 @@ if __name__ == "__main__":
|
||||
|
||||
logging.info("Going to run codebrowser: %s", run_command)
|
||||
|
||||
run_log_path = os.path.join(TEMP_PATH, "runlog.log")
|
||||
run_log_path = os.path.join(TEMP_PATH, "run.log")
|
||||
|
||||
with TeePopen(run_command, run_log_path) as process:
|
||||
retcode = process.wait()
|
||||
|
||||
@ -82,7 +82,7 @@ if __name__ == "__main__":
|
||||
f"{docker_image}"
|
||||
)
|
||||
|
||||
run_log_path = os.path.join(test_output, "runlog.log")
|
||||
run_log_path = os.path.join(test_output, "run.log")
|
||||
logging.info("Running command: '%s'", cmd)
|
||||
|
||||
with TeePopen(cmd, run_log_path) as process:
|
||||
|
||||
@ -60,7 +60,7 @@ if __name__ == "__main__":
|
||||
else:
|
||||
user = f"{os.geteuid()}:{os.getegid()}"
|
||||
|
||||
run_log_path = os.path.join(test_output, "runlog.log")
|
||||
run_log_path = os.path.join(test_output, "run.log")
|
||||
|
||||
with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"):
|
||||
cmd = (
|
||||
|
||||
@ -155,7 +155,7 @@ if __name__ == "__main__":
|
||||
if not os.path.exists(logs_path):
|
||||
os.makedirs(logs_path)
|
||||
|
||||
run_log_path = os.path.join(logs_path, "runlog.log")
|
||||
run_log_path = os.path.join(logs_path, "run.log")
|
||||
with TeePopen(run_cmd, run_log_path, timeout=40 * 60) as process:
|
||||
retcode = process.wait()
|
||||
if retcode == 0:
|
||||
|
||||
@ -292,7 +292,7 @@ if __name__ == "__main__":
|
||||
if not os.path.exists(result_path):
|
||||
os.makedirs(result_path)
|
||||
|
||||
run_log_path = os.path.join(result_path, "runlog.log")
|
||||
run_log_path = os.path.join(result_path, "run.log")
|
||||
|
||||
additional_envs = get_additional_envs(
|
||||
check_name, run_by_hash_num, run_by_hash_total
|
||||
|
||||
@ -251,7 +251,7 @@ if __name__ == "__main__":
|
||||
)
|
||||
logging.info("Going to run jepsen: %s", cmd)
|
||||
|
||||
run_log_path = os.path.join(TEMP_PATH, "runlog.log")
|
||||
run_log_path = os.path.join(TEMP_PATH, "run.log")
|
||||
|
||||
with TeePopen(cmd, run_log_path) as process:
|
||||
retcode = process.wait()
|
||||
|
||||
@ -176,7 +176,7 @@ if __name__ == "__main__":
|
||||
)
|
||||
logging.info("Going to run command %s", run_command)
|
||||
|
||||
run_log_path = os.path.join(temp_path, "runlog.log")
|
||||
run_log_path = os.path.join(temp_path, "run.log")
|
||||
|
||||
popen_env = os.environ.copy()
|
||||
popen_env.update(env_extra)
|
||||
@ -198,7 +198,7 @@ if __name__ == "__main__":
|
||||
"all-query-metrics.tsv": os.path.join(
|
||||
result_path, "report/all-query-metrics.tsv"
|
||||
),
|
||||
"runlog.log": run_log_path,
|
||||
"run.log": run_log_path,
|
||||
}
|
||||
|
||||
s3_prefix = f"{pr_info.number}/{pr_info.sha}/{check_name_prefix}/"
|
||||
@ -253,8 +253,8 @@ if __name__ == "__main__":
|
||||
|
||||
report_url = GITHUB_RUN_URL
|
||||
|
||||
if uploaded["runlog.log"]:
|
||||
report_url = uploaded["runlog.log"]
|
||||
if uploaded["run.log"]:
|
||||
report_url = uploaded["run.log"]
|
||||
|
||||
if uploaded["compare.log"]:
|
||||
report_url = uploaded["compare.log"]
|
||||
|
||||
@ -95,7 +95,7 @@ if __name__ == "__main__":
|
||||
run_command = get_run_command(build_url, workspace_path, docker_image)
|
||||
logging.info("Going to run %s", run_command)
|
||||
|
||||
run_log_path = os.path.join(workspace_path, "runlog.log")
|
||||
run_log_path = os.path.join(workspace_path, "run.log")
|
||||
with open(run_log_path, "w", encoding="utf-8") as log:
|
||||
with subprocess.Popen(
|
||||
run_command, shell=True, stderr=log, stdout=log
|
||||
|
||||
@ -34,6 +34,7 @@ def get_run_command(
|
||||
"docker run --cap-add=SYS_PTRACE "
|
||||
# a static link, don't use S3_URL or S3_DOWNLOAD
|
||||
"-e S3_URL='https://s3.amazonaws.com/clickhouse-datasets' "
|
||||
f"-e DISABLE_BC_CHECK={os.environ.get('DISABLE_BC_CHECK', '0')} "
|
||||
# For dmesg and sysctl
|
||||
"--privileged "
|
||||
f"--volume={build_path}:/package_folder "
|
||||
@ -138,7 +139,7 @@ if __name__ == "__main__":
|
||||
if not os.path.exists(result_path):
|
||||
os.makedirs(result_path)
|
||||
|
||||
run_log_path = os.path.join(temp_path, "runlog.log")
|
||||
run_log_path = os.path.join(temp_path, "run.log")
|
||||
|
||||
run_command = get_run_command(
|
||||
packages_path, result_path, repo_tests_path, server_log_path, docker_image
|
||||
|
||||
@ -140,7 +140,7 @@ if __name__ == "__main__":
|
||||
|
||||
run_command = f"docker run --cap-add=SYS_PTRACE --volume={tests_binary_path}:/unit_tests_dbms --volume={test_output}:/test_output {docker_image}"
|
||||
|
||||
run_log_path = os.path.join(test_output, "runlog.log")
|
||||
run_log_path = os.path.join(test_output, "run.log")
|
||||
|
||||
logging.info("Going to run func tests: %s", run_command)
|
||||
|
||||
|
||||
@ -6,6 +6,12 @@ from helpers.test_tools import TSV
|
||||
from helpers.network import _NetworkManager
|
||||
|
||||
|
||||
# This is a workaround for a problem with logging in pytest [1].
|
||||
#
|
||||
# [1]: https://github.com/pytest-dev/pytest/issues/5502
|
||||
logging.raiseExceptions = False
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="session")
|
||||
def cleanup_environment():
|
||||
try:
|
||||
|
||||
@ -413,5 +413,11 @@ if __name__ == "__main__":
|
||||
subprocess.check_call(f"docker kill {' '.join(containers)}", shell=True)
|
||||
print(f"Containers {containers} killed")
|
||||
|
||||
# Avoid overlaps with previous runs
|
||||
subprocess.check_call("dmesg --clear", shell=True)
|
||||
|
||||
print(("Running pytest container as: '" + cmd + "'."))
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
|
||||
# Dump dmesg (to capture possible OOMs)
|
||||
subprocess.check_call("dmesg -T", shell=True)
|
||||
|
||||
@ -1,42 +0,0 @@
|
||||
<clickhouse>
|
||||
<max_concurrent_queries>1000</max_concurrent_queries>
|
||||
|
||||
<remote_servers>
|
||||
<one_shard>
|
||||
<shard>
|
||||
<internal_replication>true</internal_replication>
|
||||
<replica>
|
||||
<host>node1_r1</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
<replica>
|
||||
<host>node1_r2</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</one_shard>
|
||||
<two_shards>
|
||||
<shard>
|
||||
<internal_replication>true</internal_replication>
|
||||
<replica>
|
||||
<host>node1_r1</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
<replica>
|
||||
<host>node1_r2</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
<shard>
|
||||
<replica>
|
||||
<host>node2_r1</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
<replica>
|
||||
<host>node2_r2</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</two_shards>
|
||||
</remote_servers>
|
||||
</clickhouse>
|
||||
@ -1,121 +0,0 @@
|
||||
# pylint: disable=redefined-outer-name
|
||||
# pylint: disable=unused-argument
|
||||
# pylint: disable=line-too-long
|
||||
|
||||
import shlex
|
||||
import itertools
|
||||
import pytest
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
node1_r1 = cluster.add_instance("node1_r1", main_configs=["configs/remote_servers.xml"])
|
||||
node2_r1 = cluster.add_instance("node2_r1", main_configs=["configs/remote_servers.xml"])
|
||||
node1_r2 = cluster.add_instance("node1_r2", main_configs=["configs/remote_servers.xml"])
|
||||
node2_r2 = cluster.add_instance("node2_r2", main_configs=["configs/remote_servers.xml"])
|
||||
|
||||
|
||||
def run_benchmark(payload, settings):
|
||||
node1_r1.exec_in_container(
|
||||
[
|
||||
"bash",
|
||||
"-c",
|
||||
"echo {} | ".format(shlex.quote(payload.strip()))
|
||||
+ " ".join(
|
||||
[
|
||||
"clickhouse",
|
||||
"benchmark",
|
||||
"--concurrency=100",
|
||||
"--cumulative",
|
||||
"--delay=0",
|
||||
# NOTE: with current matrix even 3 seconds it huge...
|
||||
"--timelimit=3",
|
||||
# tune some basic timeouts
|
||||
"--hedged_connection_timeout_ms=200",
|
||||
"--connect_timeout_with_failover_ms=200",
|
||||
"--connections_with_failover_max_tries=5",
|
||||
*settings,
|
||||
]
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
for _, instance in cluster.instances.items():
|
||||
instance.query(
|
||||
"""
|
||||
create table if not exists data (
|
||||
key Int,
|
||||
/* just to increase block size */
|
||||
v1 UInt64,
|
||||
v2 UInt64,
|
||||
v3 UInt64,
|
||||
v4 UInt64,
|
||||
v5 UInt64,
|
||||
v6 UInt64,
|
||||
v7 UInt64,
|
||||
v8 UInt64,
|
||||
v9 UInt64,
|
||||
v10 UInt64,
|
||||
v11 UInt64,
|
||||
v12 UInt64
|
||||
) Engine=MergeTree() order by key partition by key%5;
|
||||
insert into data (key) select * from numbers(10);
|
||||
|
||||
create table if not exists dist_one as data engine=Distributed(one_shard, currentDatabase(), data, key);
|
||||
create table if not exists dist_one_over_dist as data engine=Distributed(one_shard, currentDatabase(), dist_one, kostikConsistentHash(key, 2));
|
||||
|
||||
create table if not exists dist_two as data engine=Distributed(two_shards, currentDatabase(), data, key);
|
||||
create table if not exists dist_two_over_dist as data engine=Distributed(two_shards, currentDatabase(), dist_two, kostikConsistentHash(key, 2));
|
||||
"""
|
||||
)
|
||||
yield cluster
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"table,settings",
|
||||
itertools.product(
|
||||
[ # tables
|
||||
"dist_one",
|
||||
"dist_one_over_dist",
|
||||
"dist_two",
|
||||
"dist_two_over_dist",
|
||||
],
|
||||
[ # settings
|
||||
*list(
|
||||
itertools.combinations(
|
||||
[
|
||||
"", # defaults
|
||||
"--prefer_localhost_replica=0",
|
||||
"--async_socket_for_remote=0",
|
||||
"--use_hedged_requests=0",
|
||||
"--optimize_skip_unused_shards=1",
|
||||
"--distributed_group_by_no_merge=2",
|
||||
"--optimize_distributed_group_by_sharding_key=1",
|
||||
# TODO: enlarge test matrix (but first those values to accept ms):
|
||||
#
|
||||
# - sleep_in_send_tables_status
|
||||
# - sleep_in_send_data
|
||||
],
|
||||
2,
|
||||
)
|
||||
)
|
||||
# TODO: more combinations that just 2
|
||||
],
|
||||
),
|
||||
)
|
||||
def test_stress_distributed(table, settings, started_cluster):
|
||||
payload = f"""
|
||||
select * from {table} where key = 0;
|
||||
select * from {table} where key = 1;
|
||||
select * from {table} where key = 2;
|
||||
select * from {table} where key = 3;
|
||||
select * from {table};
|
||||
"""
|
||||
run_benchmark(payload, settings)
|
||||
@ -213,9 +213,9 @@ def test_attach_detach_partition(cluster):
|
||||
assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(4096)"
|
||||
wait_for_delete_empty_parts(node, "hdfs_test")
|
||||
wait_for_delete_inactive_parts(node, "hdfs_test")
|
||||
|
||||
hdfs_objects = fs.listdir("/clickhouse")
|
||||
assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2
|
||||
wait_for_delete_hdfs_objects(
|
||||
cluster, FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2
|
||||
)
|
||||
|
||||
node.query("ALTER TABLE hdfs_test ATTACH PARTITION '2020-01-03'")
|
||||
assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(8192)"
|
||||
@ -227,9 +227,7 @@ def test_attach_detach_partition(cluster):
|
||||
assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(4096)"
|
||||
wait_for_delete_empty_parts(node, "hdfs_test")
|
||||
wait_for_delete_inactive_parts(node, "hdfs_test")
|
||||
|
||||
hdfs_objects = fs.listdir("/clickhouse")
|
||||
assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE
|
||||
wait_for_delete_hdfs_objects(cluster, FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE)
|
||||
|
||||
node.query("ALTER TABLE hdfs_test DETACH PARTITION '2020-01-04'")
|
||||
node.query(
|
||||
@ -239,9 +237,7 @@ def test_attach_detach_partition(cluster):
|
||||
assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(0)"
|
||||
wait_for_delete_empty_parts(node, "hdfs_test")
|
||||
wait_for_delete_inactive_parts(node, "hdfs_test")
|
||||
|
||||
hdfs_objects = fs.listdir("/clickhouse")
|
||||
assert len(hdfs_objects) == FILES_OVERHEAD
|
||||
wait_for_delete_hdfs_objects(cluster, FILES_OVERHEAD)
|
||||
|
||||
|
||||
def test_move_partition_to_another_disk(cluster):
|
||||
@ -307,9 +303,7 @@ def test_table_manipulations(cluster):
|
||||
assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(0)"
|
||||
wait_for_delete_empty_parts(node, "hdfs_test")
|
||||
wait_for_delete_inactive_parts(node, "hdfs_test")
|
||||
|
||||
hdfs_objects = fs.listdir("/clickhouse")
|
||||
assert len(hdfs_objects) == FILES_OVERHEAD
|
||||
wait_for_delete_hdfs_objects(cluster, FILES_OVERHEAD)
|
||||
|
||||
|
||||
def test_move_replace_partition_to_another_table(cluster):
|
||||
@ -376,7 +370,6 @@ def test_move_replace_partition_to_another_table(cluster):
|
||||
assert node.query("SELECT count(*) FROM hdfs_clone FORMAT Values") == "(8192)"
|
||||
|
||||
# Wait for outdated partitions deletion.
|
||||
print(1)
|
||||
wait_for_delete_hdfs_objects(
|
||||
cluster, FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4
|
||||
)
|
||||
|
||||
@ -3,4 +3,5 @@ SELECT * FROM (
|
||||
FROM system.numbers
|
||||
ANY LEFT JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 10) js2
|
||||
USING n LIMIT 10
|
||||
) ORDER BY n;
|
||||
) ORDER BY n
|
||||
SETTINGS join_algorithm = 'hash'; -- the query does not finish with merge join
|
||||
|
||||
@ -64,7 +64,7 @@ function alter_table()
|
||||
if [ -z "$table" ]; then continue; fi
|
||||
$CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=0 -q \
|
||||
"alter table $table update n = n + (select max(n) from merge(REGEXP('${CLICKHOUSE_DATABASE}.*'), '.*')) where 1 settings allow_nondeterministic_mutations=1" \
|
||||
2>&1| grep -Fa "Exception: " | grep -Fv "Cannot enqueue query" | grep -Fv "ZooKeeper session expired" | grep -Fv UNKNOWN_DATABASE | grep -Fv UNKNOWN_TABLE | grep -Fv TABLE_IS_READ_ONLY
|
||||
2>&1| grep -Fa "Exception: " | grep -Fv "Cannot enqueue query" | grep -Fv "ZooKeeper session expired" | grep -Fv UNKNOWN_DATABASE | grep -Fv UNKNOWN_TABLE | grep -Fv TABLE_IS_READ_ONLY | grep -Fv TABLE_IS_DROPPED
|
||||
sleep 0.$RANDOM
|
||||
done
|
||||
}
|
||||
@ -75,7 +75,7 @@ function insert()
|
||||
table=$($CLICKHOUSE_CLIENT -q "select database || '.' || name from system.tables where database like '${CLICKHOUSE_DATABASE}%' order by rand() limit 1")
|
||||
if [ -z "$table" ]; then continue; fi
|
||||
$CLICKHOUSE_CLIENT -q \
|
||||
"insert into $table values ($RANDOM)" 2>&1| grep -Fa "Exception: " | grep -Fv UNKNOWN_DATABASE | grep -Fv UNKNOWN_TABLE | grep -Fv TABLE_IS_READ_ONLY
|
||||
"insert into $table values ($RANDOM)" 2>&1| grep -Fa "Exception: " | grep -Fv UNKNOWN_DATABASE | grep -Fv UNKNOWN_TABLE | grep -Fv TABLE_IS_READ_ONLY | grep -Fv TABLE_IS_DROPPED
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ z UInt32 DEFAULT 5
|
||||
7 5
|
||||
21 5
|
||||
x UInt32 DEFAULT y
|
||||
y UInt32 EPHEMERAL 0
|
||||
y UInt32 EPHEMERAL defaultValueOfTypeName(\'UInt32\')
|
||||
z UInt32 DEFAULT 5
|
||||
1 2
|
||||
0 2
|
||||
|
||||
@ -16,3 +16,5 @@ ${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none -n --query "CREATE TABLE
|
||||
${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" -n --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_rmt (x UInt32) engine = ReplicatedMergeTree order by x;"
|
||||
${CLICKHOUSE_CLIENT} --query "DROP DATABASE ${CLICKHOUSE_DATABASE}_db"
|
||||
${CLICKHOUSE_CLIENT} -q "DROP USER user_${CLICKHOUSE_DATABASE}"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "drop table mute_stylecheck"
|
||||
|
||||
@ -0,0 +1,2 @@
|
||||
CREATE TABLE default.test\n(\n `a` UInt8,\n `b` String EPHEMERAL\n)\nENGINE = Memory
|
||||
CREATE TABLE default.test\n(\n `a` UInt8,\n `b` String EPHEMERAL 1 + 2\n)\nENGINE = Memory
|
||||
@ -1,10 +1,13 @@
|
||||
DROP TABLE IF EXISTS test;
|
||||
|
||||
CREATE TABLE test(a UInt8, b String EPHEMERAL) Engine=Memory();
|
||||
|
||||
SHOW CREATE TABLE test;
|
||||
DROP TABLE test;
|
||||
|
||||
CREATE TABLE test(a UInt8, b EPHEMERAL String) Engine=Memory(); -- { clientError SYNTAX_ERROR }
|
||||
CREATE TABLE test(a UInt8, b EPHEMERAL String) Engine=Memory(); -- { serverError UNKNOWN_IDENTIFIER }
|
||||
CREATE TABLE test(a UInt8, b EPHEMERAL 'a' String) Engine=Memory(); -- { clientError SYNTAX_ERROR }
|
||||
CREATE TABLE test(a UInt8, b String EPHEMERAL test) Engine=Memory(); -- { clientError SYNTAX_ERROR }
|
||||
CREATE TABLE test(a UInt8, b String EPHEMERAL 1+2) Engine=Memory(); -- { clientError SYNTAX_ERROR }
|
||||
CREATE TABLE test(a UInt8, b String EPHEMERAL test) Engine=Memory(); -- { serverError UNKNOWN_IDENTIFIER }
|
||||
|
||||
CREATE TABLE test(a UInt8, b String EPHEMERAL 1+2) Engine=Memory();
|
||||
SHOW CREATE TABLE test;
|
||||
DROP TABLE test;
|
||||
|
||||
@ -11,7 +11,7 @@ s String
|
||||
create table, several columns with different default specifiers
|
||||
di UInt8 DEFAULT 1
|
||||
id Int32
|
||||
s String EPHEMERAL \'\'
|
||||
s String EPHEMERAL defaultValueOfTypeName(\'String\')
|
||||
create table failed, column +type +DEFAULT +AUTO_INCREMENT
|
||||
create table failed, column -type +DEFAULT +AUTO_INCREMENT
|
||||
create table failed, column +type +AUTO_INCREMENT +DEFAULT
|
||||
|
||||
@ -0,0 +1,4 @@
|
||||
-3600001
|
||||
-1
|
||||
1970-01-01 00:59:59.999
|
||||
1969-12-31 23:59:59.999
|
||||
@ -0,0 +1,4 @@
|
||||
select toUnixTimestamp64Milli(toDateTime64('1969-12-31 23:59:59.999', 3, 'Europe/Amsterdam'));
|
||||
select toUnixTimestamp64Milli(toDateTime64('1969-12-31 23:59:59.999', 3, 'UTC'));
|
||||
select fromUnixTimestamp64Milli(toInt64(-1), 'Europe/Amsterdam');
|
||||
select fromUnixTimestamp64Milli(toInt64(-1), 'UTC');
|
||||
@ -0,0 +1 @@
|
||||
-1293882467
|
||||
@ -0,0 +1,2 @@
|
||||
SELECT toUnixTimestamp(toDateTime64('1928-12-31 12:12:12.123', 3, 'UTC')); -- { serverError DECIMAL_OVERFLOW }
|
||||
SELECT toInt64(toDateTime64('1928-12-31 12:12:12.123', 3, 'UTC'));
|
||||
6
tests/queries/0_stateless/02508_bad_graphite.sql
Normal file
6
tests/queries/0_stateless/02508_bad_graphite.sql
Normal file
@ -0,0 +1,6 @@
|
||||
DROP TABLE IF EXISTS test_graphite;
|
||||
create table test_graphite (key UInt32, Path String, Time DateTime('UTC'), Value UInt8, Version UInt32, col UInt64)
|
||||
engine = GraphiteMergeTree('graphite_rollup') order by key;
|
||||
|
||||
INSERT INTO test_graphite (key) VALUES (0); -- { serverError BAD_ARGUMENTS }
|
||||
DROP TABLE test_graphite;
|
||||
@ -0,0 +1,11 @@
|
||||
4c36abda-8bd8-11eb-8204-005056aa8bf6 2021-03-24 01:04:27 1
|
||||
4c408902-8bd8-11eb-8204-005056aa8bf6 2021-03-24 01:04:27 1
|
||||
4c5bf20a-8bd8-11eb-8204-005056aa8bf6 2021-03-24 01:04:27 1
|
||||
4c61623a-8bd8-11eb-8204-005056aa8bf6 2021-03-24 01:04:27 1
|
||||
4c6efab2-8bd8-11eb-a952-005056aa8bf6 2021-03-24 01:04:27 1
|
||||
---
|
||||
4c36abda-8bd8-11eb-8204-005056aa8bf6 2021-03-24 01:04:27 1
|
||||
4c408902-8bd8-11eb-8204-005056aa8bf6 2021-03-24 01:04:27 1
|
||||
4c5bf20a-8bd8-11eb-8204-005056aa8bf6 2021-03-24 01:04:27 1
|
||||
4c61623a-8bd8-11eb-8204-005056aa8bf6 2021-03-24 01:04:27 1
|
||||
4c6efab2-8bd8-11eb-a952-005056aa8bf6 2021-03-24 01:04:27 1
|
||||
@ -0,0 +1,10 @@
|
||||
DROP TABLE IF EXISTS table;
|
||||
CREATE TABLE table (uid UUID, date DateTime('Asia/Kamchatka')) ENGINE = MergeTree ORDER BY date;
|
||||
|
||||
INSERT INTO `table` VALUES ('4c36abda-8bd8-11eb-8204-005056aa8bf6', '2021-03-24 01:04:27'), ('4c408902-8bd8-11eb-8204-005056aa8bf6', '2021-03-24 01:04:27'), ('4c5bf20a-8bd8-11eb-8204-005056aa8bf6', '2021-03-24 01:04:27'), ('4c61623a-8bd8-11eb-8204-005056aa8bf6', '2021-03-24 01:04:27'), ('4c6efab2-8bd8-11eb-a952-005056aa8bf6', '2021-03-24 01:04:27');
|
||||
|
||||
SELECT uid, date, toDate(date) = toDate('2021-03-24') AS res FROM table WHERE res = 1 ORDER BY uid, date;
|
||||
SELECT '---';
|
||||
SELECT uid, date, toDate(date) = toDate('2021-03-24') AS res FROM table WHERE toDate(date) = toDate('2021-03-24') ORDER BY uid, date;
|
||||
|
||||
DROP TABLE table;
|
||||
23
tests/queries/0_stateless/02509_h3_arguments.reference
Normal file
23
tests/queries/0_stateless/02509_h3_arguments.reference
Normal file
@ -0,0 +1,23 @@
|
||||
583031433791012863
|
||||
583031433791012863
|
||||
587531185127686143
|
||||
614047082918969343
|
||||
614047153853038591
|
||||
614048195802038271
|
||||
614054260742553599
|
||||
614054419345965055
|
||||
614552348391374847
|
||||
614553222213795839
|
||||
614554538768072703
|
||||
614555412668088319
|
||||
614790495813500927
|
||||
614047082918969343
|
||||
614047153853038591
|
||||
614048195802038271
|
||||
614054260742553599
|
||||
614054419345965055
|
||||
614552348391374847
|
||||
614553222213795839
|
||||
614554538768072703
|
||||
614555412668088319
|
||||
614790495813500927
|
||||
13
tests/queries/0_stateless/02509_h3_arguments.sql
Normal file
13
tests/queries/0_stateless/02509_h3_arguments.sql
Normal file
@ -0,0 +1,13 @@
|
||||
-- Tags: no-fasttest
|
||||
|
||||
select h3ToParent(641573946153969375, 1);
|
||||
select h3ToParent(641573946153969375, arrayJoin([1,2]));
|
||||
|
||||
DROP TABLE IF EXISTS data_table;
|
||||
|
||||
CREATE TABLE data_table (id UInt64, longitude Float64, latitude Float64) ENGINE=MergeTree ORDER BY id;
|
||||
INSERT INTO data_table SELECT number, number, number FROM numbers(10);
|
||||
SELECT geoToH3(longitude, latitude, toUInt8(8)) AS h3Index FROM data_table ORDER BY 1;
|
||||
SELECT geoToH3(longitude, latitude, toUInt8(longitude - longitude + 8)) AS h3Index FROM data_table ORDER BY 1;
|
||||
|
||||
DROP TABLE data_table;
|
||||
@ -0,0 +1 @@
|
||||
1 6
|
||||
25
tests/queries/0_stateless/02510_group_by_prewhere_null.sql
Normal file
25
tests/queries/0_stateless/02510_group_by_prewhere_null.sql
Normal file
@ -0,0 +1,25 @@
|
||||
DROP TABLE IF EXISTS table1;
|
||||
|
||||
create table table1 (
|
||||
col1 Int32,
|
||||
col2 Int32
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
partition by tuple()
|
||||
order by col1;
|
||||
|
||||
INSERT INTO table1 VALUES (1, 2), (1, 4);
|
||||
|
||||
with NULL as pid
|
||||
select a.col1, sum(a.col2) as summ
|
||||
from table1 a
|
||||
prewhere (pid is null or a.col2 = pid)
|
||||
group by a.col1;
|
||||
|
||||
with 123 as pid
|
||||
select a.col1, sum(a.col2) as summ
|
||||
from table1 a
|
||||
prewhere (pid is null or a.col2 = pid)
|
||||
group by a.col1;
|
||||
|
||||
DROP TABLE table1;
|
||||
@ -0,0 +1,2 @@
|
||||
true
|
||||
false
|
||||
7
tests/queries/0_stateless/02513_csv_bool_allow_crlf.sh
Executable file
7
tests/queries/0_stateless/02513_csv_bool_allow_crlf.sh
Executable file
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
echo -ne "True\r\nFalse\r\n" | $CLICKHOUSE_LOCAL --structure='x Bool' --input-format=CSV -q "select * from table";
|
||||
@ -0,0 +1,2 @@
|
||||
1
|
||||
1
|
||||
29
tests/queries/0_stateless/02513_insert_without_materialized_columns.sh
Executable file
29
tests/queries/0_stateless/02513_insert_without_materialized_columns.sh
Executable file
@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-parallel
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
|
||||
FILE_NAME="${CLICKHOUSE_DATABASE}_test.native.zstd"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query "CREATE TABLE test (a Int64, b Int64 MATERIALIZED a) ENGINE = MergeTree() PRIMARY KEY tuple()"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query "INSERT INTO test VALUES (1)"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query "SELECT * FROM test"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query "SELECT * FROM test INTO OUTFILE '${CLICKHOUSE_TMP}/${FILE_NAME}' FORMAT Native"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query "TRUNCATE TABLE test"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query "INSERT INTO test FROM INFILE '${CLICKHOUSE_TMP}/${FILE_NAME}'"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query "SELECT * FROM test"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query "DROP TABLE test"
|
||||
|
||||
rm -f "${CLICKHOUSE_TMP}/${FILE_NAME}"
|
||||
@ -0,0 +1,7 @@
|
||||
CREATE TABLE t
|
||||
(
|
||||
id Int64,
|
||||
d String,
|
||||
p Map(String, String)
|
||||
)
|
||||
ENGINE = ReplacingMergeTree order by id settings index_granularity = 0; -- { serverError BAD_ARGUMENTS }
|
||||
21
tests/queries/0_stateless/02514_database_replicated_no_arguments_for_rmt.sh
Executable file
21
tests/queries/0_stateless/02514_database_replicated_no_arguments_for_rmt.sh
Executable file
@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: replica, no-replicated-database
|
||||
# I don't understand why this test fails in ReplicatedDatabase run
|
||||
# but too many magic included in it, so I just disabled it for ReplicatedDatabase run becase
|
||||
# here we explicitely create it and check is alright.
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "create table mute_stylecheck (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/root', '1') order by x"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE USER user_${CLICKHOUSE_DATABASE} settings database_replicated_allow_replicated_engine_arguments=0"
|
||||
${CLICKHOUSE_CLIENT} -q "GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}_db.* TO user_${CLICKHOUSE_DATABASE}"
|
||||
${CLICKHOUSE_CLIENT} --allow_experimental_database_replicated=1 --query "CREATE DATABASE ${CLICKHOUSE_DATABASE}_db engine = Replicated('/clickhouse/databases/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_DATABASE}_db', '{shard}', '{replica}')"
|
||||
${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" -n --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_rmt_ok (x UInt32) engine = ReplicatedMergeTree order by x;"
|
||||
${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" -n --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_rmt_fail (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/root/{shard}', '{replica}') order by x; -- { serverError 80 }"
|
||||
${CLICKHOUSE_CLIENT} --query "DROP DATABASE ${CLICKHOUSE_DATABASE}_db"
|
||||
${CLICKHOUSE_CLIENT} -q "DROP USER user_${CLICKHOUSE_DATABASE}"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "drop table mute_stylecheck"
|
||||
@ -0,0 +1,4 @@
|
||||
0 \N 111 0 111
|
||||
123 \N 111 123 111
|
||||
\N \N 111
|
||||
77
|
||||
48
tests/queries/0_stateless/02514_null_dictionary_source.sql
Normal file
48
tests/queries/0_stateless/02514_null_dictionary_source.sql
Normal file
@ -0,0 +1,48 @@
|
||||
-- Tags: no-parallel
|
||||
|
||||
DROP DICTIONARY IF EXISTS null_dict;
|
||||
CREATE DICTIONARY null_dict (
|
||||
id UInt64,
|
||||
val UInt8,
|
||||
default_val UInt8 DEFAULT 123,
|
||||
nullable_val Nullable(UInt8)
|
||||
)
|
||||
PRIMARY KEY id
|
||||
SOURCE(NULL())
|
||||
LAYOUT(FLAT())
|
||||
LIFETIME(0);
|
||||
|
||||
SELECT
|
||||
dictGet('null_dict', 'val', 1337),
|
||||
dictGetOrNull('null_dict', 'val', 1337),
|
||||
dictGetOrDefault('null_dict', 'val', 1337, 111),
|
||||
dictGetUInt8('null_dict', 'val', 1337),
|
||||
dictGetUInt8OrDefault('null_dict', 'val', 1337, 111);
|
||||
|
||||
SELECT
|
||||
dictGet('null_dict', 'default_val', 1337),
|
||||
dictGetOrNull('null_dict', 'default_val', 1337),
|
||||
dictGetOrDefault('null_dict', 'default_val', 1337, 111),
|
||||
dictGetUInt8('null_dict', 'default_val', 1337),
|
||||
dictGetUInt8OrDefault('null_dict', 'default_val', 1337, 111);
|
||||
|
||||
SELECT
|
||||
dictGet('null_dict', 'nullable_val', 1337),
|
||||
dictGetOrNull('null_dict', 'nullable_val', 1337),
|
||||
dictGetOrDefault('null_dict', 'nullable_val', 1337, 111);
|
||||
|
||||
SELECT val, nullable_val FROM null_dict;
|
||||
|
||||
DROP DICTIONARY IF EXISTS null_ip_dict;
|
||||
CREATE DICTIONARY null_ip_dict (
|
||||
network String,
|
||||
val UInt8 DEFAULT 77
|
||||
)
|
||||
PRIMARY KEY network
|
||||
SOURCE(NULL())
|
||||
LAYOUT(IP_TRIE())
|
||||
LIFETIME(0);
|
||||
|
||||
SELECT dictGet('null_ip_dict', 'val', toIPv4('127.0.0.1'));
|
||||
|
||||
SELECT network, val FROM null_ip_dict;
|
||||
@ -0,0 +1,3 @@
|
||||
0
|
||||
|
||||
0
|
||||
@ -0,0 +1,6 @@
|
||||
DROP TABLE IF EXISTS t;
|
||||
CREATE TABLE t (x UInt8, PROJECTION p (SELECT x GROUP BY x)) ENGINE = MergeTree ORDER BY ();
|
||||
INSERT INTO t VALUES (0);
|
||||
SET group_by_overflow_mode = 'any', max_rows_to_group_by = 1000, totals_mode = 'after_having_auto';
|
||||
SELECT x FROM t GROUP BY x WITH TOTALS;
|
||||
DROP TABLE t;
|
||||
120
tests/queries/0_stateless/02516_projections_with_rollup.sql
Normal file
120
tests/queries/0_stateless/02516_projections_with_rollup.sql
Normal file
@ -0,0 +1,120 @@
|
||||
DROP TABLE IF EXISTS video_log;
|
||||
DROP TABLE IF EXISTS video_log_result__fuzz_0;
|
||||
DROP TABLE IF EXISTS rng;
|
||||
|
||||
CREATE TABLE video_log
|
||||
(
|
||||
`datetime` DateTime,
|
||||
`user_id` UInt64,
|
||||
`device_id` UInt64,
|
||||
`domain` LowCardinality(String),
|
||||
`bytes` UInt64,
|
||||
`duration` UInt64
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(datetime)
|
||||
ORDER BY (user_id, device_id);
|
||||
|
||||
CREATE TABLE video_log_result__fuzz_0
|
||||
(
|
||||
`hour` Nullable(DateTime),
|
||||
`sum_bytes` UInt64,
|
||||
`avg_duration` Float64
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(hour)
|
||||
ORDER BY sum_bytes
|
||||
SETTINGS allow_nullable_key = 1;
|
||||
|
||||
CREATE TABLE rng
|
||||
(
|
||||
`user_id_raw` UInt64,
|
||||
`device_id_raw` UInt64,
|
||||
`domain_raw` UInt64,
|
||||
`bytes_raw` UInt64,
|
||||
`duration_raw` UInt64
|
||||
)
|
||||
ENGINE = GenerateRandom(1024);
|
||||
|
||||
INSERT INTO video_log SELECT
|
||||
toUnixTimestamp('2022-07-22 01:00:00') + (rowNumberInAllBlocks() / 20000),
|
||||
user_id_raw % 100000000 AS user_id,
|
||||
device_id_raw % 200000000 AS device_id,
|
||||
domain_raw % 100,
|
||||
(bytes_raw % 1024) + 128,
|
||||
(duration_raw % 300) + 100
|
||||
FROM rng
|
||||
LIMIT 1728000;
|
||||
|
||||
INSERT INTO video_log SELECT
|
||||
toUnixTimestamp('2022-07-22 01:00:00') + (rowNumberInAllBlocks() / 20000),
|
||||
user_id_raw % 100000000 AS user_id,
|
||||
100 AS device_id,
|
||||
domain_raw % 100,
|
||||
(bytes_raw % 1024) + 128,
|
||||
(duration_raw % 300) + 100
|
||||
FROM rng
|
||||
LIMIT 10;
|
||||
|
||||
ALTER TABLE video_log
|
||||
ADD PROJECTION p_norm
|
||||
(
|
||||
SELECT
|
||||
datetime,
|
||||
device_id,
|
||||
bytes,
|
||||
duration
|
||||
ORDER BY device_id
|
||||
);
|
||||
|
||||
ALTER TABLE video_log
|
||||
MATERIALIZE PROJECTION p_norm
|
||||
SETTINGS mutations_sync = 1;
|
||||
|
||||
ALTER TABLE video_log
|
||||
ADD PROJECTION p_agg
|
||||
(
|
||||
SELECT
|
||||
toStartOfHour(datetime) AS hour,
|
||||
domain,
|
||||
sum(bytes),
|
||||
avg(duration)
|
||||
GROUP BY
|
||||
hour,
|
||||
domain
|
||||
);
|
||||
|
||||
ALTER TABLE video_log
|
||||
MATERIALIZE PROJECTION p_agg
|
||||
SETTINGS mutations_sync = 1;
|
||||
|
||||
-- We are not interested in the result of this query, but it should not produce a logical error.
|
||||
SELECT
|
||||
avg_duration1,
|
||||
avg_duration1 = avg_duration2
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
sum(bytes),
|
||||
hour,
|
||||
toStartOfHour(datetime) AS hour,
|
||||
avg(duration) AS avg_duration1
|
||||
FROM video_log
|
||||
GROUP BY hour
|
||||
WITH ROLLUP
|
||||
WITH TOTALS
|
||||
)
|
||||
LEFT JOIN
|
||||
(
|
||||
SELECT
|
||||
hour,
|
||||
sum_bytes AS sum_bytes2,
|
||||
avg_duration AS avg_duration2
|
||||
FROM video_log_result__fuzz_0
|
||||
) USING (hour)
|
||||
SETTINGS joined_subquery_requires_alias = 0
|
||||
FORMAT Null;
|
||||
|
||||
DROP TABLE video_log;
|
||||
DROP TABLE video_log_result__fuzz_0;
|
||||
DROP TABLE rng;
|
||||
Loading…
Reference in New Issue
Block a user