thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-18 04:12:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

filelog engine tests improve

Browse Source 
fix
This commit is contained in:
feng lv 2021-10-29 02:39:27 +00:00
parent fab8da1c64
commit 13a305d648
7 changed files with 57 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmake/find/filelog.cmake
View File

@ -1,10 +1,3 @@
option (ENABLE_FILELOG "Enable FILELOG" ON)
if (NOT ENABLE_FILELOG)
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use StorageFileLog with ENABLE_FILELOG=OFF")
return()
endif()
# StorageFileLog only support Linux platform # StorageFileLog only support Linux platform
if (OS_LINUX) if (OS_LINUX)
set (USE_FILELOG 1) set (USE_FILELOG 1)

3
src/Storages/FileLog/DirectoryWatcherBase.cpp
View File

@ -35,7 +35,7 @@ DirectoryWatcherBase::DirectoryWatcherBase(
fd = inotify_init(); fd = inotify_init();
if (fd == -1) if (fd == -1)
throw Exception("Cannot initialize inotify", ErrorCodes::IO_SETUP_ERROR); throwFromErrno("Cannot initialize inotify", ErrorCodes::IO_SETUP_ERROR);
watch_task = getContext()->getSchedulePool().createTask("directory_watch", [this] { watchFunc(); }); watch_task = getContext()->getSchedulePool().createTask("directory_watch", [this] { watchFunc(); });
start(); start();
@ -59,6 +59,7 @@ void DirectoryWatcherBase::watchFunc()
if (wd == -1) if (wd == -1)
{ {
owner.onError(Exception(ErrorCodes::IO_SETUP_ERROR, "Watch directory {} failed", path)); owner.onError(Exception(ErrorCodes::IO_SETUP_ERROR, "Watch directory {} failed", path));
throwFromErrnoWithPath("Watch directory {} failed", path, ErrorCodes::IO_SETUP_ERROR);
} }
std::string buffer; std::string buffer;

10
tests/queries/0_stateless/02022_storage_filelog_one_file.sh
View File

@ -14,23 +14,23 @@ user_files_path=$(clickhouse-client --query "select _path,_file from file('nonex
for i in {1..20} for i in {1..20}
do do
echo $i, $i >> ${user_files_path}/a.txt echo $i, $i >> ${user_files_path}/02022_storage_filelog_one_file.txt
done done
${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;" ${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;"
${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/a.txt', 'CSV');" ${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/02022_storage_filelog_one_file.txt', 'CSV');"
${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;"
for i in {100..120} for i in {100..120}
do do
echo $i, $i >> ${user_files_path}/a.txt echo $i, $i >> ${user_files_path}/02022_storage_filelog_one_file.txt
done done
${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;"
# touch does not change file content, no event # touch does not change file content, no event
touch ${user_files_path}/a.txt touch ${user_files_path}/02022_storage_filelog_one_file.txt
${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;"
${CLICKHOUSE_CLIENT} --query "detach table file_log;" ${CLICKHOUSE_CLIENT} --query "detach table file_log;"
@ -39,4 +39,4 @@ ${CLICKHOUSE_CLIENT} --query "attach table file_log;"
# should no records return # should no records return
${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;"
rm -rf ${user_files_path}/a.txt rm -rf ${user_files_path}/02022_storage_filelog_one_file.txt

38
tests/queries/0_stateless/02023_storage_filelog.sh
View File

@ -12,52 +12,52 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# "insert into function file('exist.txt', 'CSV', 'val1 char') values ('aaaa'); select _path from file('exist.txt', 'CSV', 'val1 char')" # "insert into function file('exist.txt', 'CSV', 'val1 char') values ('aaaa'); select _path from file('exist.txt', 'CSV', 'val1 char')"
user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}')
mkdir -p ${user_files_path}/logs/ mkdir -p ${user_files_path}/02023_storage_filelog/
rm -rf ${user_files_path}/logs/* rm -rf ${user_files_path}/02023_storage_filelog/*
for i in {1..20} for i in {1..20}
do do
echo $i, $i >> ${user_files_path}/logs/a.txt echo $i, $i >> ${user_files_path}/02023_storage_filelog/a.txt
done done
${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;" ${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;"
${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/logs/', 'CSV');" ${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/02023_storage_filelog/', 'CSV');"
${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;"
cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/b.txt cp ${user_files_path}/02023_storage_filelog/a.txt ${user_files_path}/02023_storage_filelog/b.txt
${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;"
for i in {100..120} for i in {100..120}
do do
echo $i, $i >> ${user_files_path}/logs/a.txt echo $i, $i >> ${user_files_path}/02023_storage_filelog/a.txt
done done
# touch does not change file content, no event # touch does not change file content, no event
touch ${user_files_path}/logs/a.txt touch ${user_files_path}/02023_storage_filelog/a.txt
cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/c.txt cp ${user_files_path}/02023_storage_filelog/a.txt ${user_files_path}/02023_storage_filelog/c.txt
cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/d.txt cp ${user_files_path}/02023_storage_filelog/a.txt ${user_files_path}/02023_storage_filelog/d.txt
cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/e.txt cp ${user_files_path}/02023_storage_filelog/a.txt ${user_files_path}/02023_storage_filelog/e.txt
mv ${user_files_path}/logs/b.txt ${user_files_path}/logs/j.txt mv ${user_files_path}/02023_storage_filelog/b.txt ${user_files_path}/02023_storage_filelog/j.txt
rm ${user_files_path}/logs/d.txt rm ${user_files_path}/02023_storage_filelog/d.txt
${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;"
${CLICKHOUSE_CLIENT} --query "detach table file_log;" ${CLICKHOUSE_CLIENT} --query "detach table file_log;"
cp ${user_files_path}/logs/e.txt ${user_files_path}/logs/f.txt cp ${user_files_path}/02023_storage_filelog/e.txt ${user_files_path}/02023_storage_filelog/f.txt
mv ${user_files_path}/logs/e.txt ${user_files_path}/logs/g.txt mv ${user_files_path}/02023_storage_filelog/e.txt ${user_files_path}/02023_storage_filelog/g.txt
mv ${user_files_path}/logs/c.txt ${user_files_path}/logs/h.txt mv ${user_files_path}/02023_storage_filelog/c.txt ${user_files_path}/02023_storage_filelog/h.txt
for i in {150..200} for i in {150..200}
do do
echo $i, $i >> ${user_files_path}/logs/h.txt echo $i, $i >> ${user_files_path}/02023_storage_filelog/h.txt
done done
for i in {200..250} for i in {200..250}
do do
echo $i, $i >> ${user_files_path}/logs/i.txt echo $i, $i >> ${user_files_path}/02023_storage_filelog/i.txt
done done
${CLICKHOUSE_CLIENT} --query "attach table file_log;" ${CLICKHOUSE_CLIENT} --query "attach table file_log;"
@ -69,11 +69,11 @@ ${CLICKHOUSE_CLIENT} --query "attach table file_log;"
# should no records return # should no records return
${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;"
truncate ${user_files_path}/logs/a.txt --size 0 truncate ${user_files_path}/02023_storage_filelog/a.txt --size 0
# exception happend # exception happend
${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" 2>&1 | grep -q "Code: 33" && echo 'OK' || echo 'FAIL' ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" 2>&1 | grep -q "Code: 33" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} --query "drop table file_log;" ${CLICKHOUSE_CLIENT} --query "drop table file_log;"
rm -rf ${user_files_path}/logs rm -rf ${user_files_path}/02023_storage_filelog

20
tests/queries/0_stateless/02024_storage_filelog_mv.sh
View File

@ -12,16 +12,16 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# "insert into function file('exist.txt', 'CSV', 'val1 char') values ('aaaa'); select _path from file('exist.txt', 'CSV', 'val1 char')" # "insert into function file('exist.txt', 'CSV', 'val1 char') values ('aaaa'); select _path from file('exist.txt', 'CSV', 'val1 char')"
user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}')
mkdir -p ${user_files_path}/logs/ mkdir -p ${user_files_path}/02024_storage_filelog_mv/
rm -rf ${user_files_path}/logs/* rm -rf ${user_files_path}/02024_storage_filelog_mv/*
for i in {1..20} for i in {1..20}
do do
echo $i, $i >> ${user_files_path}/logs/a.txt echo $i, $i >> ${user_files_path}/02024_storage_filelog_mv/a.txt
done done
${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;" ${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;"
${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/logs/', 'CSV');" ${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/02024_storage_filelog_mv/', 'CSV');"
${CLICKHOUSE_CLIENT} --query "drop table if exists mv;" ${CLICKHOUSE_CLIENT} --query "drop table if exists mv;"
${CLICKHOUSE_CLIENT} --query "create Materialized View mv engine=MergeTree order by k as select * from file_log;" ${CLICKHOUSE_CLIENT} --query "create Materialized View mv engine=MergeTree order by k as select * from file_log;"
@ -39,17 +39,17 @@ done
${CLICKHOUSE_CLIENT} --query "select * from mv order by k;" ${CLICKHOUSE_CLIENT} --query "select * from mv order by k;"
cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/b.txt cp ${user_files_path}/02024_storage_filelog_mv/a.txt ${user_files_path}/02024_storage_filelog_mv/b.txt
# touch does not change file content, no event # touch does not change file content, no event
touch ${user_files_path}/logs/a.txt touch ${user_files_path}/02024_storage_filelog_mv/a.txt
cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/c.txt cp ${user_files_path}/02024_storage_filelog_mv/a.txt ${user_files_path}/02024_storage_filelog_mv/c.txt
cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/d.txt cp ${user_files_path}/02024_storage_filelog_mv/a.txt ${user_files_path}/02024_storage_filelog_mv/d.txt
for i in {100..120} for i in {100..120}
do do
echo $i, $i >> ${user_files_path}/logs/d.txt echo $i, $i >> ${user_files_path}/02024_storage_filelog_mv/d.txt
done done
while true; do while true; do
@ -62,4 +62,4 @@ ${CLICKHOUSE_CLIENT} --query "select * from mv order by k;"
${CLICKHOUSE_CLIENT} --query "drop table mv;" ${CLICKHOUSE_CLIENT} --query "drop table mv;"
${CLICKHOUSE_CLIENT} --query "drop table file_log;" ${CLICKHOUSE_CLIENT} --query "drop table file_log;"
rm -rf ${user_files_path}/logs rm -rf ${user_files_path}/02024_storage_filelog_mv

26
tests/queries/0_stateless/02025_storage_filelog_virtual_col.sh
View File

@ -12,37 +12,37 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# "insert into function file('exist.txt', 'CSV', 'val1 char') values ('aaaa'); select _path from file('exist.txt', 'CSV', 'val1 char')" # "insert into function file('exist.txt', 'CSV', 'val1 char') values ('aaaa'); select _path from file('exist.txt', 'CSV', 'val1 char')"
user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}')
mkdir -p ${user_files_path}/logs/ mkdir -p ${user_files_path}/02025_storage_filelog_virtual_col/
rm -rf ${user_files_path}/logs/* rm -rf ${user_files_path}/02025_storage_filelog_virtual_col/*
for i in {1..20} for i in {1..20}
do do
echo $i, $i >> ${user_files_path}/logs/a.txt echo $i, $i >> ${user_files_path}/02025_storage_filelog_virtual_col/a.txt
done done
${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;" ${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;"
${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/logs/', 'CSV');" ${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/02025_storage_filelog_virtual_col/', 'CSV');"
${CLICKHOUSE_CLIENT} --query "select *, _filename, _offset from file_log order by _filename, _offset;" ${CLICKHOUSE_CLIENT} --query "select *, _filename, _offset from file_log order by _filename, _offset;"
cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/b.txt cp ${user_files_path}/02025_storage_filelog_virtual_col/a.txt ${user_files_path}/02025_storage_filelog_virtual_col/b.txt
${CLICKHOUSE_CLIENT} --query "select *, _filename, _offset from file_log order by _filename, _offset;" ${CLICKHOUSE_CLIENT} --query "select *, _filename, _offset from file_log order by _filename, _offset;"
for i in {100..120} for i in {100..120}
do do
echo $i, $i >> ${user_files_path}/logs/a.txt echo $i, $i >> ${user_files_path}/02025_storage_filelog_virtual_col/a.txt
done done
# touch does not change file content, no event # touch does not change file content, no event
touch ${user_files_path}/logs/a.txt touch ${user_files_path}/02025_storage_filelog_virtual_col/a.txt
cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/c.txt cp ${user_files_path}/02025_storage_filelog_virtual_col/a.txt ${user_files_path}/02025_storage_filelog_virtual_col/c.txt
cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/d.txt cp ${user_files_path}/02025_storage_filelog_virtual_col/a.txt ${user_files_path}/02025_storage_filelog_virtual_col/d.txt
cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/e.txt cp ${user_files_path}/02025_storage_filelog_virtual_col/a.txt ${user_files_path}/02025_storage_filelog_virtual_col/e.txt
rm ${user_files_path}/logs/d.txt rm ${user_files_path}/02025_storage_filelog_virtual_col/d.txt
${CLICKHOUSE_CLIENT} --query "select *, _filename, _offset from file_log order by _filename, _offset;" ${CLICKHOUSE_CLIENT} --query "select *, _filename, _offset from file_log order by _filename, _offset;"
@ -52,11 +52,11 @@ ${CLICKHOUSE_CLIENT} --query "attach table file_log;"
# should no records return # should no records return
${CLICKHOUSE_CLIENT} --query "select *, _filename, _offset from file_log order by _filename, _offset;" ${CLICKHOUSE_CLIENT} --query "select *, _filename, _offset from file_log order by _filename, _offset;"
truncate ${user_files_path}/logs/a.txt --size 0 truncate ${user_files_path}/02025_storage_filelog_virtual_col/a.txt --size 0
# exception happend # exception happend
${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" 2>&1 | grep -q "Code: 33" && echo 'OK' || echo 'FAIL' ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" 2>&1 | grep -q "Code: 33" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} --query "drop table file_log;" ${CLICKHOUSE_CLIENT} --query "drop table file_log;"
rm -rf ${user_files_path}/logs rm -rf ${user_files_path}/02025_storage_filelog_virtual_col

16
tests/queries/0_stateless/02026_storage_filelog_largefile.sh
View File

@ -12,36 +12,36 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# "insert into function file('exist.txt', 'CSV', 'val1 char') values ('aaaa'); select _path from file('exist.txt', 'CSV', 'val1 char')" # "insert into function file('exist.txt', 'CSV', 'val1 char') values ('aaaa'); select _path from file('exist.txt', 'CSV', 'val1 char')"
user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}')
mkdir -p ${user_files_path}/logs/ mkdir -p ${user_files_path}/02026_storage_filelog_largefile/
rm -rf ${user_files_path}/logs/* rm -rf ${user_files_path}/02026_storage_filelog_largefile/*
chmod 777 ${user_files_path}/logs/ chmod 777 ${user_files_path}/02026_storage_filelog_largefile/
for i in {1..200} for i in {1..200}
do do
${CLICKHOUSE_CLIENT} --query "insert into function file('${user_files_path}/logs/test$i.csv', 'CSV', 'k UInt32, v UInt32') select number, number from numbers(10000);" ${CLICKHOUSE_CLIENT} --query "insert into function file('${user_files_path}/02026_storage_filelog_largefile/test$i.csv', 'CSV', 'k UInt32, v UInt32') select number, number from numbers(10000);"
done done
${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;" ${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;"
${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt32, v UInt32) engine=FileLog('${user_files_path}/logs/', 'CSV');" ${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt32, v UInt32) engine=FileLog('${user_files_path}/02026_storage_filelog_largefile/', 'CSV');"
${CLICKHOUSE_CLIENT} --query "select count() from file_log " ${CLICKHOUSE_CLIENT} --query "select count() from file_log "
for i in {201..400} for i in {201..400}
do do
${CLICKHOUSE_CLIENT} --query "insert into function file('${user_files_path}/logs/test$i.csv', 'CSV', 'k UInt32, v UInt32') select number, number from numbers(10000);" ${CLICKHOUSE_CLIENT} --query "insert into function file('${user_files_path}/02026_storage_filelog_largefile/test$i.csv', 'CSV', 'k UInt32, v UInt32') select number, number from numbers(10000);"
done done
${CLICKHOUSE_CLIENT} --query "select count() from file_log " ${CLICKHOUSE_CLIENT} --query "select count() from file_log "
for i in {401..600} for i in {401..600}
do do
${CLICKHOUSE_CLIENT} --query "insert into function file('${user_files_path}/logs/test$i.csv', 'CSV', 'k UInt32, v UInt32') select number, number from numbers(10000);" ${CLICKHOUSE_CLIENT} --query "insert into function file('${user_files_path}/02026_storage_filelog_largefile/test$i.csv', 'CSV', 'k UInt32, v UInt32') select number, number from numbers(10000);"
done done
${CLICKHOUSE_CLIENT} --query "select count() from file_log " ${CLICKHOUSE_CLIENT} --query "select count() from file_log "
${CLICKHOUSE_CLIENT} --query "drop table file_log;" ${CLICKHOUSE_CLIENT} --query "drop table file_log;"
rm -rf ${user_files_path}/logs rm -rf ${user_files_path}/02026_storage_filelog_largefile