From 13a305d6489ea29ba6c7c7b5c3629fdf37ec6a5d Mon Sep 17 00:00:00 2001 From: feng lv Date: Fri, 29 Oct 2021 02:39:27 +0000 Subject: [PATCH] filelog engine tests improve fix --- cmake/find/filelog.cmake | 7 ---- src/Storages/FileLog/DirectoryWatcherBase.cpp | 3 +- .../02022_storage_filelog_one_file.sh | 10 ++--- .../0_stateless/02023_storage_filelog.sh | 38 +++++++++---------- .../0_stateless/02024_storage_filelog_mv.sh | 20 +++++----- .../02025_storage_filelog_virtual_col.sh | 26 ++++++------- .../02026_storage_filelog_largefile.sh | 16 ++++---- 7 files changed, 57 insertions(+), 63 deletions(-) diff --git a/cmake/find/filelog.cmake b/cmake/find/filelog.cmake index 4d2f94f3f20..f545ee9d0ed 100644 --- a/cmake/find/filelog.cmake +++ b/cmake/find/filelog.cmake @@ -1,10 +1,3 @@ -option (ENABLE_FILELOG "Enable FILELOG" ON) - -if (NOT ENABLE_FILELOG) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use StorageFileLog with ENABLE_FILELOG=OFF") - return() -endif() - # StorageFileLog only support Linux platform if (OS_LINUX) set (USE_FILELOG 1) diff --git a/src/Storages/FileLog/DirectoryWatcherBase.cpp b/src/Storages/FileLog/DirectoryWatcherBase.cpp index 37a82ad4ee3..f2737219fd8 100644 --- a/src/Storages/FileLog/DirectoryWatcherBase.cpp +++ b/src/Storages/FileLog/DirectoryWatcherBase.cpp @@ -35,7 +35,7 @@ DirectoryWatcherBase::DirectoryWatcherBase( fd = inotify_init(); if (fd == -1) - throw Exception("Cannot initialize inotify", ErrorCodes::IO_SETUP_ERROR); + throwFromErrno("Cannot initialize inotify", ErrorCodes::IO_SETUP_ERROR); watch_task = getContext()->getSchedulePool().createTask("directory_watch", [this] { watchFunc(); }); start(); @@ -59,6 +59,7 @@ void DirectoryWatcherBase::watchFunc() if (wd == -1) { owner.onError(Exception(ErrorCodes::IO_SETUP_ERROR, "Watch directory {} failed", path)); + throwFromErrnoWithPath("Watch directory {} failed", path, ErrorCodes::IO_SETUP_ERROR); } std::string buffer; diff --git a/tests/queries/0_stateless/02022_storage_filelog_one_file.sh b/tests/queries/0_stateless/02022_storage_filelog_one_file.sh index 600e537a352..bed97c7c0be 100755 --- a/tests/queries/0_stateless/02022_storage_filelog_one_file.sh +++ b/tests/queries/0_stateless/02022_storage_filelog_one_file.sh @@ -14,23 +14,23 @@ user_files_path=$(clickhouse-client --query "select _path,_file from file('nonex for i in {1..20} do - echo $i, $i >> ${user_files_path}/a.txt + echo $i, $i >> ${user_files_path}/02022_storage_filelog_one_file.txt done ${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;" -${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/a.txt', 'CSV');" +${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/02022_storage_filelog_one_file.txt', 'CSV');" ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" for i in {100..120} do - echo $i, $i >> ${user_files_path}/a.txt + echo $i, $i >> ${user_files_path}/02022_storage_filelog_one_file.txt done ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" # touch does not change file content, no event -touch ${user_files_path}/a.txt +touch ${user_files_path}/02022_storage_filelog_one_file.txt ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" ${CLICKHOUSE_CLIENT} --query "detach table file_log;" @@ -39,4 +39,4 @@ ${CLICKHOUSE_CLIENT} --query "attach table file_log;" # should no records return ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" -rm -rf ${user_files_path}/a.txt +rm -rf ${user_files_path}/02022_storage_filelog_one_file.txt diff --git a/tests/queries/0_stateless/02023_storage_filelog.sh b/tests/queries/0_stateless/02023_storage_filelog.sh index 8279e09ff65..fd242aad359 100755 --- a/tests/queries/0_stateless/02023_storage_filelog.sh +++ b/tests/queries/0_stateless/02023_storage_filelog.sh @@ -12,52 +12,52 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # "insert into function file('exist.txt', 'CSV', 'val1 char') values ('aaaa'); select _path from file('exist.txt', 'CSV', 'val1 char')" user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') -mkdir -p ${user_files_path}/logs/ +mkdir -p ${user_files_path}/02023_storage_filelog/ -rm -rf ${user_files_path}/logs/* +rm -rf ${user_files_path}/02023_storage_filelog/* for i in {1..20} do - echo $i, $i >> ${user_files_path}/logs/a.txt + echo $i, $i >> ${user_files_path}/02023_storage_filelog/a.txt done ${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;" -${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/logs/', 'CSV');" +${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/02023_storage_filelog/', 'CSV');" ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" -cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/b.txt +cp ${user_files_path}/02023_storage_filelog/a.txt ${user_files_path}/02023_storage_filelog/b.txt ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" for i in {100..120} do - echo $i, $i >> ${user_files_path}/logs/a.txt + echo $i, $i >> ${user_files_path}/02023_storage_filelog/a.txt done # touch does not change file content, no event -touch ${user_files_path}/logs/a.txt +touch ${user_files_path}/02023_storage_filelog/a.txt -cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/c.txt -cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/d.txt -cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/e.txt -mv ${user_files_path}/logs/b.txt ${user_files_path}/logs/j.txt +cp ${user_files_path}/02023_storage_filelog/a.txt ${user_files_path}/02023_storage_filelog/c.txt +cp ${user_files_path}/02023_storage_filelog/a.txt ${user_files_path}/02023_storage_filelog/d.txt +cp ${user_files_path}/02023_storage_filelog/a.txt ${user_files_path}/02023_storage_filelog/e.txt +mv ${user_files_path}/02023_storage_filelog/b.txt ${user_files_path}/02023_storage_filelog/j.txt -rm ${user_files_path}/logs/d.txt +rm ${user_files_path}/02023_storage_filelog/d.txt ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" ${CLICKHOUSE_CLIENT} --query "detach table file_log;" -cp ${user_files_path}/logs/e.txt ${user_files_path}/logs/f.txt -mv ${user_files_path}/logs/e.txt ${user_files_path}/logs/g.txt -mv ${user_files_path}/logs/c.txt ${user_files_path}/logs/h.txt +cp ${user_files_path}/02023_storage_filelog/e.txt ${user_files_path}/02023_storage_filelog/f.txt +mv ${user_files_path}/02023_storage_filelog/e.txt ${user_files_path}/02023_storage_filelog/g.txt +mv ${user_files_path}/02023_storage_filelog/c.txt ${user_files_path}/02023_storage_filelog/h.txt for i in {150..200} do - echo $i, $i >> ${user_files_path}/logs/h.txt + echo $i, $i >> ${user_files_path}/02023_storage_filelog/h.txt done for i in {200..250} do - echo $i, $i >> ${user_files_path}/logs/i.txt + echo $i, $i >> ${user_files_path}/02023_storage_filelog/i.txt done ${CLICKHOUSE_CLIENT} --query "attach table file_log;" @@ -69,11 +69,11 @@ ${CLICKHOUSE_CLIENT} --query "attach table file_log;" # should no records return ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" -truncate ${user_files_path}/logs/a.txt --size 0 +truncate ${user_files_path}/02023_storage_filelog/a.txt --size 0 # exception happend ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" 2>&1 | grep -q "Code: 33" && echo 'OK' || echo 'FAIL' ${CLICKHOUSE_CLIENT} --query "drop table file_log;" -rm -rf ${user_files_path}/logs +rm -rf ${user_files_path}/02023_storage_filelog diff --git a/tests/queries/0_stateless/02024_storage_filelog_mv.sh b/tests/queries/0_stateless/02024_storage_filelog_mv.sh index 9ac0c95eae0..1e2f990c135 100755 --- a/tests/queries/0_stateless/02024_storage_filelog_mv.sh +++ b/tests/queries/0_stateless/02024_storage_filelog_mv.sh @@ -12,16 +12,16 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # "insert into function file('exist.txt', 'CSV', 'val1 char') values ('aaaa'); select _path from file('exist.txt', 'CSV', 'val1 char')" user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') -mkdir -p ${user_files_path}/logs/ -rm -rf ${user_files_path}/logs/* +mkdir -p ${user_files_path}/02024_storage_filelog_mv/ +rm -rf ${user_files_path}/02024_storage_filelog_mv/* for i in {1..20} do - echo $i, $i >> ${user_files_path}/logs/a.txt + echo $i, $i >> ${user_files_path}/02024_storage_filelog_mv/a.txt done ${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;" -${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/logs/', 'CSV');" +${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/02024_storage_filelog_mv/', 'CSV');" ${CLICKHOUSE_CLIENT} --query "drop table if exists mv;" ${CLICKHOUSE_CLIENT} --query "create Materialized View mv engine=MergeTree order by k as select * from file_log;" @@ -39,17 +39,17 @@ done ${CLICKHOUSE_CLIENT} --query "select * from mv order by k;" -cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/b.txt +cp ${user_files_path}/02024_storage_filelog_mv/a.txt ${user_files_path}/02024_storage_filelog_mv/b.txt # touch does not change file content, no event -touch ${user_files_path}/logs/a.txt +touch ${user_files_path}/02024_storage_filelog_mv/a.txt -cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/c.txt -cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/d.txt +cp ${user_files_path}/02024_storage_filelog_mv/a.txt ${user_files_path}/02024_storage_filelog_mv/c.txt +cp ${user_files_path}/02024_storage_filelog_mv/a.txt ${user_files_path}/02024_storage_filelog_mv/d.txt for i in {100..120} do - echo $i, $i >> ${user_files_path}/logs/d.txt + echo $i, $i >> ${user_files_path}/02024_storage_filelog_mv/d.txt done while true; do @@ -62,4 +62,4 @@ ${CLICKHOUSE_CLIENT} --query "select * from mv order by k;" ${CLICKHOUSE_CLIENT} --query "drop table mv;" ${CLICKHOUSE_CLIENT} --query "drop table file_log;" -rm -rf ${user_files_path}/logs +rm -rf ${user_files_path}/02024_storage_filelog_mv diff --git a/tests/queries/0_stateless/02025_storage_filelog_virtual_col.sh b/tests/queries/0_stateless/02025_storage_filelog_virtual_col.sh index a92f93991ef..4545a18e650 100755 --- a/tests/queries/0_stateless/02025_storage_filelog_virtual_col.sh +++ b/tests/queries/0_stateless/02025_storage_filelog_virtual_col.sh @@ -12,37 +12,37 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # "insert into function file('exist.txt', 'CSV', 'val1 char') values ('aaaa'); select _path from file('exist.txt', 'CSV', 'val1 char')" user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') -mkdir -p ${user_files_path}/logs/ +mkdir -p ${user_files_path}/02025_storage_filelog_virtual_col/ -rm -rf ${user_files_path}/logs/* +rm -rf ${user_files_path}/02025_storage_filelog_virtual_col/* for i in {1..20} do - echo $i, $i >> ${user_files_path}/logs/a.txt + echo $i, $i >> ${user_files_path}/02025_storage_filelog_virtual_col/a.txt done ${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;" -${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/logs/', 'CSV');" +${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt8, v UInt8) engine=FileLog('${user_files_path}/02025_storage_filelog_virtual_col/', 'CSV');" ${CLICKHOUSE_CLIENT} --query "select *, _filename, _offset from file_log order by _filename, _offset;" -cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/b.txt +cp ${user_files_path}/02025_storage_filelog_virtual_col/a.txt ${user_files_path}/02025_storage_filelog_virtual_col/b.txt ${CLICKHOUSE_CLIENT} --query "select *, _filename, _offset from file_log order by _filename, _offset;" for i in {100..120} do - echo $i, $i >> ${user_files_path}/logs/a.txt + echo $i, $i >> ${user_files_path}/02025_storage_filelog_virtual_col/a.txt done # touch does not change file content, no event -touch ${user_files_path}/logs/a.txt +touch ${user_files_path}/02025_storage_filelog_virtual_col/a.txt -cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/c.txt -cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/d.txt -cp ${user_files_path}/logs/a.txt ${user_files_path}/logs/e.txt +cp ${user_files_path}/02025_storage_filelog_virtual_col/a.txt ${user_files_path}/02025_storage_filelog_virtual_col/c.txt +cp ${user_files_path}/02025_storage_filelog_virtual_col/a.txt ${user_files_path}/02025_storage_filelog_virtual_col/d.txt +cp ${user_files_path}/02025_storage_filelog_virtual_col/a.txt ${user_files_path}/02025_storage_filelog_virtual_col/e.txt -rm ${user_files_path}/logs/d.txt +rm ${user_files_path}/02025_storage_filelog_virtual_col/d.txt ${CLICKHOUSE_CLIENT} --query "select *, _filename, _offset from file_log order by _filename, _offset;" @@ -52,11 +52,11 @@ ${CLICKHOUSE_CLIENT} --query "attach table file_log;" # should no records return ${CLICKHOUSE_CLIENT} --query "select *, _filename, _offset from file_log order by _filename, _offset;" -truncate ${user_files_path}/logs/a.txt --size 0 +truncate ${user_files_path}/02025_storage_filelog_virtual_col/a.txt --size 0 # exception happend ${CLICKHOUSE_CLIENT} --query "select * from file_log order by k;" 2>&1 | grep -q "Code: 33" && echo 'OK' || echo 'FAIL' ${CLICKHOUSE_CLIENT} --query "drop table file_log;" -rm -rf ${user_files_path}/logs +rm -rf ${user_files_path}/02025_storage_filelog_virtual_col diff --git a/tests/queries/0_stateless/02026_storage_filelog_largefile.sh b/tests/queries/0_stateless/02026_storage_filelog_largefile.sh index a6c5d19287e..c480fcc05d3 100755 --- a/tests/queries/0_stateless/02026_storage_filelog_largefile.sh +++ b/tests/queries/0_stateless/02026_storage_filelog_largefile.sh @@ -12,36 +12,36 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # "insert into function file('exist.txt', 'CSV', 'val1 char') values ('aaaa'); select _path from file('exist.txt', 'CSV', 'val1 char')" user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') -mkdir -p ${user_files_path}/logs/ +mkdir -p ${user_files_path}/02026_storage_filelog_largefile/ -rm -rf ${user_files_path}/logs/* +rm -rf ${user_files_path}/02026_storage_filelog_largefile/* -chmod 777 ${user_files_path}/logs/ +chmod 777 ${user_files_path}/02026_storage_filelog_largefile/ for i in {1..200} do - ${CLICKHOUSE_CLIENT} --query "insert into function file('${user_files_path}/logs/test$i.csv', 'CSV', 'k UInt32, v UInt32') select number, number from numbers(10000);" + ${CLICKHOUSE_CLIENT} --query "insert into function file('${user_files_path}/02026_storage_filelog_largefile/test$i.csv', 'CSV', 'k UInt32, v UInt32') select number, number from numbers(10000);" done ${CLICKHOUSE_CLIENT} --query "drop table if exists file_log;" -${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt32, v UInt32) engine=FileLog('${user_files_path}/logs/', 'CSV');" +${CLICKHOUSE_CLIENT} --query "create table file_log(k UInt32, v UInt32) engine=FileLog('${user_files_path}/02026_storage_filelog_largefile/', 'CSV');" ${CLICKHOUSE_CLIENT} --query "select count() from file_log " for i in {201..400} do - ${CLICKHOUSE_CLIENT} --query "insert into function file('${user_files_path}/logs/test$i.csv', 'CSV', 'k UInt32, v UInt32') select number, number from numbers(10000);" + ${CLICKHOUSE_CLIENT} --query "insert into function file('${user_files_path}/02026_storage_filelog_largefile/test$i.csv', 'CSV', 'k UInt32, v UInt32') select number, number from numbers(10000);" done ${CLICKHOUSE_CLIENT} --query "select count() from file_log " for i in {401..600} do - ${CLICKHOUSE_CLIENT} --query "insert into function file('${user_files_path}/logs/test$i.csv', 'CSV', 'k UInt32, v UInt32') select number, number from numbers(10000);" + ${CLICKHOUSE_CLIENT} --query "insert into function file('${user_files_path}/02026_storage_filelog_largefile/test$i.csv', 'CSV', 'k UInt32, v UInt32') select number, number from numbers(10000);" done ${CLICKHOUSE_CLIENT} --query "select count() from file_log " ${CLICKHOUSE_CLIENT} --query "drop table file_log;" -rm -rf ${user_files_path}/logs +rm -rf ${user_files_path}/02026_storage_filelog_largefile