From 90294e6dd8fb075b2ade445af301aeef7c1f664e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 17 Aug 2021 18:22:41 +0200 Subject: [PATCH 1/6] 01889_sqlite_read_write: Made parallelizable and cleanup properly --- .../0_stateless/01889_sqlite_read_write.sh | 82 ++++++++++--------- tests/queries/skip_list.json | 1 - 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/tests/queries/0_stateless/01889_sqlite_read_write.sh b/tests/queries/0_stateless/01889_sqlite_read_write.sh index 73b106e9eb4..3e7e15d2830 100755 --- a/tests/queries/0_stateless/01889_sqlite_read_write.sh +++ b/tests/queries/0_stateless/01889_sqlite_read_write.sh @@ -7,60 +7,68 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # See 01658_read_file_to_string_column.sh user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') -mkdir -p ${user_files_path}/ -chmod 777 ${user_files_path} -DB_PATH=${user_files_path}/db1 +mkdir -p "${user_files_path}/" +chmod 777 "${user_files_path}" +export CURR_DATABASE="test_01889_sqllite_${CLICKHOUSE_DATABASE}" -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table1' -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table2' -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table3' -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table4' -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table5' +DB_PATH=${user_files_path}/${CURR_DATABASE}_db1 +DB_PATH2=$CUR_DIR/${CURR_DATABASE}_db2 -sqlite3 ${DB_PATH} 'CREATE TABLE table1 (col1 text, col2 smallint);' -sqlite3 ${DB_PATH} 'CREATE TABLE table2 (col1 int, col2 text);' +function cleanup() +{ + ${CLICKHOUSE_CLIENT} --query="DROP DATABASE IF EXISTS ${CURR_DATABASE}" + rm -r "${DB_PATH}" "${DB_PATH2}" +} +trap cleanup EXIT -chmod ugo+w ${DB_PATH} +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table1' +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table2' +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table3' +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table4' +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table5' -sqlite3 ${DB_PATH} "INSERT INTO table1 VALUES ('line1', 1), ('line2', 2), ('line3', 3)" -sqlite3 ${DB_PATH} "INSERT INTO table2 VALUES (1, 'text1'), (2, 'text2'), (3, 'text3')" +sqlite3 "${DB_PATH}" 'CREATE TABLE table1 (col1 text, col2 smallint);' +sqlite3 "${DB_PATH}" 'CREATE TABLE table2 (col1 int, col2 text);' -sqlite3 ${DB_PATH} 'CREATE TABLE table3 (col1 text, col2 int);' -sqlite3 ${DB_PATH} 'INSERT INTO table3 VALUES (NULL, 1)' -sqlite3 ${DB_PATH} "INSERT INTO table3 VALUES ('not a null', 2)" -sqlite3 ${DB_PATH} 'INSERT INTO table3 VALUES (NULL, 3)' -sqlite3 ${DB_PATH} "INSERT INTO table3 VALUES ('', 4)" +chmod ugo+w "${DB_PATH}" -sqlite3 ${DB_PATH} 'CREATE TABLE table4 (a int, b integer, c tinyint, d smallint, e mediumint, bigint, int2, int8)' -sqlite3 ${DB_PATH} 'CREATE TABLE table5 (a character(20), b varchar(10), c real, d double, e double precision, f float)' +sqlite3 "${DB_PATH}" "INSERT INTO table1 VALUES ('line1', 1), ('line2', 2), ('line3', 3)" +sqlite3 "${DB_PATH}" "INSERT INTO table2 VALUES (1, 'text1'), (2, 'text2'), (3, 'text3')" +sqlite3 "${DB_PATH}" 'CREATE TABLE table3 (col1 text, col2 int);' +sqlite3 "${DB_PATH}" 'INSERT INTO table3 VALUES (NULL, 1)' +sqlite3 "${DB_PATH}" "INSERT INTO table3 VALUES ('not a null', 2)" +sqlite3 "${DB_PATH}" 'INSERT INTO table3 VALUES (NULL, 3)' +sqlite3 "${DB_PATH}" "INSERT INTO table3 VALUES ('', 4)" + +sqlite3 "${DB_PATH}" 'CREATE TABLE table4 (a int, b integer, c tinyint, d smallint, e mediumint, bigint, int2, int8)' +sqlite3 "${DB_PATH}" 'CREATE TABLE table5 (a character(20), b varchar(10), c real, d double, e double precision, f float)' -${CLICKHOUSE_CLIENT} --query='DROP DATABASE IF EXISTS sqlite_database' ${CLICKHOUSE_CLIENT} --query="select 'create database engine'"; -${CLICKHOUSE_CLIENT} --query="CREATE DATABASE sqlite_database ENGINE = SQLite('${DB_PATH}')" +${CLICKHOUSE_CLIENT} --query="CREATE DATABASE ${CURR_DATABASE} ENGINE = SQLite('${DB_PATH}')" ${CLICKHOUSE_CLIENT} --query="select 'show database tables:'"; -${CLICKHOUSE_CLIENT} --query='SHOW TABLES FROM sqlite_database;' +${CLICKHOUSE_CLIENT} --query='SHOW TABLES FROM '"${CURR_DATABASE}"';' ${CLICKHOUSE_CLIENT} --query="select 'show creare table:'"; -${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table1;' | sed -r 's/(.*SQLite)(.*)/\1/' -${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table2;' | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query="SHOW CREATE TABLE ${CURR_DATABASE}.table1;" | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query="SHOW CREATE TABLE ${CURR_DATABASE}.table2;" | sed -r 's/(.*SQLite)(.*)/\1/' ${CLICKHOUSE_CLIENT} --query="select 'describe table:'"; -${CLICKHOUSE_CLIENT} --query='DESCRIBE TABLE sqlite_database.table1;' -${CLICKHOUSE_CLIENT} --query='DESCRIBE TABLE sqlite_database.table2;' +${CLICKHOUSE_CLIENT} --query="DESCRIBE TABLE ${CURR_DATABASE}.table1;" +${CLICKHOUSE_CLIENT} --query="DESCRIBE TABLE ${CURR_DATABASE}.table2;" ${CLICKHOUSE_CLIENT} --query="select 'select *:'"; -${CLICKHOUSE_CLIENT} --query='SELECT * FROM sqlite_database.table1 ORDER BY col2' -${CLICKHOUSE_CLIENT} --query='SELECT * FROM sqlite_database.table2 ORDER BY col1;' +${CLICKHOUSE_CLIENT} --query="SELECT * FROM ${CURR_DATABASE}.table1 ORDER BY col2" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM ${CURR_DATABASE}.table2 ORDER BY col1" ${CLICKHOUSE_CLIENT} --query="select 'test types'"; -${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table4;' | sed -r 's/(.*SQLite)(.*)/\1/' -${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table5;' | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query="SHOW CREATE TABLE ${CURR_DATABASE}.table4;" | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query="SHOW CREATE TABLE ${CURR_DATABASE}.table5;" | sed -r 's/(.*SQLite)(.*)/\1/' -${CLICKHOUSE_CLIENT} --query='DROP DATABASE IF EXISTS sqlite_database' +${CLICKHOUSE_CLIENT} --query="DROP DATABASE IF EXISTS ${CURR_DATABASE}" ${CLICKHOUSE_CLIENT} --query="select 'create table engine with table3'"; @@ -79,11 +87,9 @@ ${CLICKHOUSE_CLIENT} --query="INSERT INTO TABLE FUNCTION sqlite('${DB_PATH}', 't ${CLICKHOUSE_CLIENT} --query="SELECT * FROM sqlite('${DB_PATH}', 'table1') ORDER BY col2" -sqlite3 $CUR_DIR/db2 'DROP TABLE IF EXISTS table1' -sqlite3 $CUR_DIR/db2 'CREATE TABLE table1 (col1 text, col2 smallint);' -sqlite3 $CUR_DIR/db2 "INSERT INTO table1 VALUES ('line1', 1), ('line2', 2), ('line3', 3)" +sqlite3 "${DB_PATH2}" 'DROP TABLE IF EXISTS table1' +sqlite3 "${DB_PATH2}" 'CREATE TABLE table1 (col1 text, col2 smallint);' +sqlite3 "${DB_PATH2}" "INSERT INTO table1 VALUES ('line1', 1), ('line2', 2), ('line3', 3)" ${CLICKHOUSE_CLIENT} --query="select 'test path in clickhouse-local'"; -${CLICKHOUSE_LOCAL} --query="SELECT * FROM sqlite('$CUR_DIR/db2', 'table1') ORDER BY col2" - -rm -r ${DB_PATH} +${CLICKHOUSE_LOCAL} --query="SELECT * FROM sqlite('${DB_PATH2}', 'table1') ORDER BY col2" diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 83ad14c44dc..c24de285856 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -487,7 +487,6 @@ "01824_prefer_global_in_and_join", "01870_modulo_partition_key", "01870_buffer_flush", // creates database - "01889_sqlite_read_write", "01889_postgresql_protocol_null_fields", "01889_check_row_policy_defined_using_user_function", "01921_concurrent_ttl_and_normal_merges_zookeeper_long", // heavy test, better to run sequentially From 0e5cfdbb9dbe0f6d823a99be508b4cd6c824444c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 17 Aug 2021 18:27:51 +0200 Subject: [PATCH 2/6] 01054_cache_dictionary_overflow_cell: Drop database at the end --- .../queries/0_stateless/01054_cache_dictionary_overflow_cell.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql b/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql index d8d1d61be63..1b317e2165e 100644 --- a/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql +++ b/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql @@ -54,3 +54,4 @@ FROM ); DROP TABLE if exists test_01054.ints; +DROP DATABASE test_01054_overflow; From 19f087cecefb2cd5e4c8e9569ea224a7659e9fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 17 Aug 2021 18:31:40 +0200 Subject: [PATCH 3/6] 01114_mysql_database_engine_segfault: Cleanup beforehand --- .../queries/0_stateless/01114_mysql_database_engine_segfault.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql b/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql index af88c5af53a..5893365e11c 100644 --- a/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql +++ b/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql @@ -1 +1,2 @@ +DROP DATABASE IF EXISTS conv_main; CREATE DATABASE conv_main ENGINE = MySQL('127.0.0.1:3456', conv_main, 'metrika', 'password'); -- { serverError 501 } From b80ddd4bd1f64cbb634583fda44d32a1303211d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 17 Aug 2021 18:41:31 +0200 Subject: [PATCH 4/6] 01516_drop_table_stress: Parallelizable and cleanup --- .../0_stateless/01516_drop_table_stress.sh | 21 +++++++++++-------- tests/queries/skip_list.json | 1 - 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/01516_drop_table_stress.sh b/tests/queries/0_stateless/01516_drop_table_stress.sh index d72104c8c7f..20e76ce49c7 100755 --- a/tests/queries/0_stateless/01516_drop_table_stress.sh +++ b/tests/queries/0_stateless/01516_drop_table_stress.sh @@ -4,26 +4,29 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +export CURR_DATABASE="test_01516_${CLICKHOUSE_DATABASE}" + function drop_database() { # redirect stderr since it is racy with DROP TABLE - # and tries to remove db_01516.data too. - ${CLICKHOUSE_CLIENT} -q "DROP DATABASE IF EXISTS db_01516" 2>/dev/null + # and tries to remove ${CURR_DATABASE}.data too. + ${CLICKHOUSE_CLIENT} -q "DROP DATABASE IF EXISTS ${CURR_DATABASE}" 2>/dev/null } +trap drop_database EXIT function drop_table() { - ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db_01516.data3;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" - ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db_01516.data1;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" - ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db_01516.data2;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${CURR_DATABASE}.data3;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${CURR_DATABASE}.data1;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${CURR_DATABASE}.data2;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" } function create() { - ${CLICKHOUSE_CLIENT} -q "CREATE DATABASE IF NOT EXISTS db_01516;" - ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS db_01516.data1 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" - ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS db_01516.data2 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" - ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS db_01516.data3 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "CREATE DATABASE IF NOT EXISTS ${CURR_DATABASE};" + ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS ${CURR_DATABASE}.data1 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS ${CURR_DATABASE}.data2 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS ${CURR_DATABASE}.data3 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" } for _ in {1..100}; do diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index c24de285856..c2b5782e766 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -416,7 +416,6 @@ "01507_clickhouse_server_start_with_embedded_config", "01509_dictionary_preallocate", "01516_create_table_primary_key", - "01516_drop_table_stress", "01517_drop_mv_with_inner_table", "01526_complex_key_dict_direct_layout", "01527_clickhouse_local_optimize", From f7ed6c4fb8fa7b27d67a8967298cfd9bd88cc66b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 17 Aug 2021 18:42:31 +0200 Subject: [PATCH 5/6] 01516_drop_table_stress: Tag as long --- ...le_stress.reference => 01516_drop_table_stress_long.reference} | 0 ...01516_drop_table_stress.sh => 01516_drop_table_stress_long.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{01516_drop_table_stress.reference => 01516_drop_table_stress_long.reference} (100%) rename tests/queries/0_stateless/{01516_drop_table_stress.sh => 01516_drop_table_stress_long.sh} (100%) diff --git a/tests/queries/0_stateless/01516_drop_table_stress.reference b/tests/queries/0_stateless/01516_drop_table_stress_long.reference similarity index 100% rename from tests/queries/0_stateless/01516_drop_table_stress.reference rename to tests/queries/0_stateless/01516_drop_table_stress_long.reference diff --git a/tests/queries/0_stateless/01516_drop_table_stress.sh b/tests/queries/0_stateless/01516_drop_table_stress_long.sh similarity index 100% rename from tests/queries/0_stateless/01516_drop_table_stress.sh rename to tests/queries/0_stateless/01516_drop_table_stress_long.sh From ac5ac0a106a6c2217210bb358e3d8e97ed3a898d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 17 Aug 2021 18:48:32 +0200 Subject: [PATCH 6/6] 01280_ssd_complex_key_dictionary: Delete database at end --- tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh index ff62b70c184..e1e0018a1dd 100755 --- a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh +++ b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh @@ -122,3 +122,5 @@ $CLICKHOUSE_CLIENT -n --query="DROP DICTIONARY 01280_db.ssd_dict; SELECT arrayJoin([('1', toInt32(3)), ('2', toInt32(-1)), ('', toInt32(0)), ('', toInt32(0)), ('2', toInt32(-1)), ('1', toInt32(3))]) AS keys, dictGetInt32('01280_db.ssd_dict', 'b', keys); DROP DICTIONARY IF EXISTS database_for_dict.ssd_dict; DROP TABLE IF EXISTS database_for_dict.keys_table;" + +$CLICKHOUSE_CLIENT -n --query="DROP DATABASE IF EXISTS 01280_db;"