mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-12-01 03:52:15 +00:00
non replicated inserts with deduplication user token
This commit is contained in:
Sema Checherinda
parent
47c7b7fccf
commit
8b7563040c
@ -519,29 +519,29 @@ BlockIO InterpreterInsertQuery::execute()
|
||||
|
||||
if (settings.max_insert_threads > 1)
|
||||
{
|
||||
auto table_id = table->getStorageID();
|
||||
auto views = DatabaseCatalog::instance().getDependentViews(table_id);
|
||||
pre_streams_size = std::max<size_t>(settings.max_insert_threads, pipeline.getNumStreams());
|
||||
|
||||
/// It breaks some views-related tests and we have dedicated `parallel_view_processing` for views, so let's just skip them.
|
||||
/// Also it doesn't make sense to reshuffle data if storage doesn't support parallel inserts.
|
||||
const bool resize_to_max_insert_threads = !table->isView() && views.empty() && table->supportsParallelInsert();
|
||||
pre_streams_size = resize_to_max_insert_threads ? settings.max_insert_threads
|
||||
: std::min<size_t>(settings.max_insert_threads, pipeline.getNumStreams());
|
||||
|
||||
/// Deduplication when passing insert_deduplication_token breaks if using more than one thread
|
||||
if (!settings.insert_deduplication_token.toString().empty())
|
||||
{
|
||||
/// TODO!
|
||||
LOG_DEBUG(
|
||||
getLogger("InsertQuery"),
|
||||
"Insert-select query using insert_deduplication_token, setting streams to 1 to avoid deduplication issues");
|
||||
pre_streams_size = 1;
|
||||
}
|
||||
// /// Deduplication when passing insert_deduplication_token breaks if using more than one thread
|
||||
// if (!settings.insert_deduplication_token.toString().empty())
|
||||
// {
|
||||
// /// TODO!
|
||||
// LOG_DEBUG(
|
||||
// getLogger("InsertQuery"),
|
||||
// "Insert-select query using insert_deduplication_token, setting streams from {} to 1 to avoid deduplication issues, pipeline.getNumStreams() {}",
|
||||
// pre_streams_size, pipeline.getNumStreams());
|
||||
// pre_streams_size = 1;
|
||||
// }
|
||||
|
||||
if (table->supportsParallelInsert())
|
||||
sink_streams_size = pre_streams_size;
|
||||
}
|
||||
|
||||
LOG_DEBUG(
|
||||
getLogger("InsertQuery"),
|
||||
"pre_streams_size {}, pipeline.getNumStreams() {}",
|
||||
pre_streams_size, pipeline.getNumStreams());
|
||||
|
||||
pipeline.resize(pre_streams_size);
|
||||
|
||||
/// Allow to insert Nullable into non-Nullable columns, NULL values will be added as defaults values.
|
||||
|
||||
@ -487,7 +487,7 @@ Chain buildPushingToViewsChain(
|
||||
|
||||
for (const auto & view_id : views)
|
||||
{
|
||||
LOG_ERROR(&Poco::Logger::get("PushingToViews"), "dependent view: {}.{}", view_id.database_name, view_id.table_name);
|
||||
LOG_DEBUG(&Poco::Logger::get("PushingToViews"), "dependent view: {}.{}", view_id.database_name, view_id.table_name);
|
||||
|
||||
try
|
||||
{
|
||||
|
||||
@ -47,8 +47,7 @@ public:
|
||||
: data(data_)
|
||||
, log(getLogger(data.getLogName() + " (Writer)"))
|
||||
{
|
||||
LOG_WARNING(log, "MergeTreeDataWriter() called from:\n{}", StackTrace().toString());
|
||||
|
||||
LOG_DEBUG(log, "MergeTreeDataWriter() called from:\n{}", StackTrace().toString());
|
||||
}
|
||||
|
||||
/** Split the block to blocks, each of them must be written as separate part.
|
||||
|
||||
@ -51,7 +51,7 @@ MergedBlockOutputStream::MergedBlockOutputStream(
|
||||
|
||||
writer = data_part->getWriter(columns_list, metadata_snapshot, skip_indices, statistics, default_codec, writer_settings, computed_index_granularity);
|
||||
|
||||
LOG_WARNING(getLogger("MergedBlockOutputStream()"), "called c-tor");
|
||||
LOG_DEBUG(getLogger("MergedBlockOutputStream()"), "called c-tor");
|
||||
}
|
||||
|
||||
/// If data is pre-sorted.
|
||||
@ -331,7 +331,7 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis
|
||||
|
||||
void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Permutation * permutation)
|
||||
{
|
||||
LOG_WARNING(getLogger("MergedBlockOutputStream()"), "writeImpl block rows {} size {} getPartDirectory {}",
|
||||
LOG_DEBUG(getLogger("MergedBlockOutputStream()"), "writeImpl block rows {} size {} getPartDirectory {}",
|
||||
block.rows(), block.bytes(), data_part_storage->getPartDirectory());
|
||||
|
||||
block.checkNumberOfRows();
|
||||
|
||||
561
tests/queries/0_stateless/03008_deduplication.python
Normal file
561
tests/queries/0_stateless/03008_deduplication.python
Normal file
@ -0,0 +1,561 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import string
|
||||
|
||||
|
||||
CURDIR = os.path.dirname(os.path.realpath(__file__))
|
||||
sys.path.insert(0, os.path.join(CURDIR, "helpers"))
|
||||
|
||||
|
||||
def __format(template, **params):
|
||||
field_names = [v[1] for v in string.Formatter().parse(template) if v[1] is not None]
|
||||
kv_args = {}
|
||||
for field in field_names:
|
||||
if field in params:
|
||||
kv_args[field] = params[field]
|
||||
else:
|
||||
kv_args[field] = ""
|
||||
|
||||
return template.format(**kv_args)
|
||||
|
||||
|
||||
def instance_create_statement(table_name, table_columns, table_keys, table_engine, with_deduplication, no_merges=True):
|
||||
template = """
|
||||
CREATE TABLE {table_name}
|
||||
{table_columns}
|
||||
ENGINE = {table_engine}
|
||||
ORDER BY {table_keys}
|
||||
{table_settings};
|
||||
{table_no_merges}
|
||||
"""
|
||||
|
||||
params = dict()
|
||||
params["table_name"] = table_name
|
||||
params["table_columns"] = table_columns
|
||||
params["table_keys"] = table_keys
|
||||
params["table_no_merges"] = f"SYSTEM STOP MERGES {table_name};" if no_merges else ""
|
||||
params["table_engine"] = "MergeTree()" if table_engine == "MergeTree" else f"ReplicatedMergeTree('/clickhouse/tables/{{database}}/{table_name}', '1')"
|
||||
|
||||
deduplication_window_setting_name = "non_replicated_deduplication_window" if table_engine == "MergeTree" else "replicated_deduplication_window"
|
||||
deduplication_window_setting_value = 1000 if with_deduplication else 0
|
||||
|
||||
settings = list()
|
||||
settings += [f"{deduplication_window_setting_name}={deduplication_window_setting_value}"]
|
||||
params["table_settings"] = "SETTINGS " + ",".join(settings)
|
||||
|
||||
return __format(template, **params)
|
||||
|
||||
|
||||
def instance_insert_statement(table_name, count, insert_unique_blocks, use_insert_token):
|
||||
template = """
|
||||
INSERT INTO {table_name}
|
||||
SELECT {insert_columns}
|
||||
FROM numbers({count}) {insert_settings};
|
||||
"""
|
||||
return __format(
|
||||
template,
|
||||
table_name=table_name,
|
||||
count=count,
|
||||
insert_columns="'src_4', 4" if not insert_unique_blocks else "'src_' || toString(number), number",
|
||||
insert_settings="" if not use_insert_token else "SETTINGS insert_deduplication_token='UDT'",
|
||||
)
|
||||
|
||||
|
||||
def get_drop_tables_statements(tables):
|
||||
return "".join([f"DROP TABLE IF EXISTS {table_name};\n" for table_name in tables[::-1]])
|
||||
|
||||
|
||||
def get_logs_statement(args):
|
||||
if args.get_logs:
|
||||
return "SET send_logs_level='test';"
|
||||
return ""
|
||||
|
||||
def str2bool(v):
|
||||
if isinstance(v, bool):
|
||||
return v
|
||||
if v.lower() in ('yes', 'true', 't', 'y', '1'):
|
||||
return True
|
||||
elif v.lower() in ('no', 'false', 'f', 'n', '0'):
|
||||
return False
|
||||
else:
|
||||
raise argparse.ArgumentTypeError('Boolean value expected.')
|
||||
|
||||
class ArgsFactory:
|
||||
def __init__(self, parser):
|
||||
self.__parser = parser
|
||||
|
||||
def add_opt_engine(self):
|
||||
self.__parser.add_argument(
|
||||
"--table-engine", choices=["ReplicatedMergeTree", "MergeTree"], default="MergeTree")
|
||||
|
||||
def add_opt_user_token(self):
|
||||
self.__parser.add_argument("--use-insert-token", type=str2bool, nargs='?', const=True, default=False)
|
||||
|
||||
def add_opt_single_thread(self):
|
||||
self.__parser.add_argument("--single-thread", type=str2bool, nargs='?', const=True, default=True)
|
||||
|
||||
def add_opt_dedup_src(self):
|
||||
self.__parser.add_argument("--deduplicate-src-table", type=str2bool, nargs='?', const=True, default=True)
|
||||
|
||||
def add_opt_dedup_dst(self):
|
||||
self.__parser.add_argument("--deduplicate-dst-table", type=str2bool, nargs='?', const=True, default=True)
|
||||
|
||||
def add_opt_get_logs(self):
|
||||
self.__parser.add_argument("--get-logs", type=str2bool, nargs='?', const=True, default=False)
|
||||
|
||||
def add_opt_uniq_blocks(self):
|
||||
self.__parser.add_argument("--insert-unique-blocks", type=str2bool, nargs='?', const=True, default=True)
|
||||
|
||||
def add_all(self):
|
||||
self.add_opt_engine()
|
||||
self.add_opt_user_token()
|
||||
self.add_opt_single_thread()
|
||||
self.add_opt_dedup_src()
|
||||
self.add_opt_dedup_dst()
|
||||
self.add_opt_get_logs()
|
||||
self.add_opt_uniq_blocks()
|
||||
|
||||
|
||||
def test_insert_several_blocks(parser):
|
||||
ArgsFactory(parser).add_all()
|
||||
|
||||
def calle(args):
|
||||
create_table_a_b_statement = instance_create_statement(
|
||||
table_name="table_a_b",
|
||||
table_columns="(a String, b UInt64)",
|
||||
table_keys="(a, b)",
|
||||
table_engine=args.table_engine,
|
||||
with_deduplication=args.deduplicate_src_table,
|
||||
)
|
||||
|
||||
create_table_when_b_even_statement = instance_create_statement(
|
||||
table_name="table_when_b_even",
|
||||
table_columns="(a String, b UInt64)",
|
||||
table_keys="(a, b)",
|
||||
table_engine=args.table_engine,
|
||||
with_deduplication=args.deduplicate_dst_table,
|
||||
)
|
||||
|
||||
create_mv_statement = """
|
||||
CREATE MATERIALIZED VIEW mv_b_even
|
||||
TO table_when_b_even
|
||||
AS
|
||||
SELECT a, b
|
||||
FROM table_a_b
|
||||
WHERE b % 2 = 0;
|
||||
"""
|
||||
|
||||
drop_tables_statements = get_drop_tables_statements( ["table_a_b", "table_when_b_even", "mv_b_even"] )
|
||||
|
||||
insert_statement = instance_insert_statement(
|
||||
"table_a_b", 10, args.insert_unique_blocks, args.use_insert_token
|
||||
)
|
||||
|
||||
print_details_statements = f"""
|
||||
SELECT 'table_a_b';
|
||||
SELECT 'count', count() FROM table_a_b;
|
||||
{"" if not args.get_logs else "SELECT _part, count() FROM table_a_b GROUP BY _part ORDER BY _part;"}
|
||||
|
||||
SELECT 'table_when_b_even';
|
||||
SELECT 'count', count() FROM table_when_b_even;
|
||||
{"" if not args.get_logs else "SELECT _part, count() FROM table_when_b_even GROUP BY _part ORDER BY _part;"}
|
||||
"""
|
||||
|
||||
|
||||
|
||||
if args.insert_unique_blocks:
|
||||
assert_first_insert_statements = f"""
|
||||
SELECT throwIf( count() != 10 )
|
||||
FROM table_a_b;
|
||||
SELECT throwIf( count() != 5 )
|
||||
FROM table_when_b_even;
|
||||
"""
|
||||
assert_second_insert_statements = f"""
|
||||
SELECT throwIf( count() != {10 if args.deduplicate_src_table else 20} )
|
||||
FROM table_a_b;
|
||||
SELECT throwIf( count() != {5 if args.deduplicate_dst_table else 10} )
|
||||
FROM table_when_b_even;
|
||||
"""
|
||||
else:
|
||||
if args.use_insert_token:
|
||||
assert_first_insert_statements = """
|
||||
SELECT throwIf( count() != 10 )
|
||||
FROM table_a_b;
|
||||
SELECT throwIf( count() != 10 )
|
||||
FROM table_when_b_even;
|
||||
"""
|
||||
assert_second_insert_statements = f"""
|
||||
SELECT throwIf( count() != {10 if args.deduplicate_src_table else 20} )
|
||||
FROM table_a_b;
|
||||
SELECT throwIf( count() != {10 if args.deduplicate_dst_table else 20} )
|
||||
FROM table_when_b_even;
|
||||
"""
|
||||
else:
|
||||
assert_first_insert_statements = f"""
|
||||
SELECT throwIf( count() != {1 if args.deduplicate_src_table else 10} )
|
||||
FROM table_a_b;
|
||||
SELECT throwIf( count() != {1 if args.deduplicate_dst_table else 10} )
|
||||
FROM table_when_b_even;
|
||||
"""
|
||||
assert_second_insert_statements = f"""
|
||||
SELECT throwIf( count() != {1 if args.deduplicate_src_table else 20} )
|
||||
FROM table_a_b;
|
||||
SELECT throwIf( count() != {1 if args.deduplicate_dst_table else 20} )
|
||||
FROM table_when_b_even;
|
||||
"""
|
||||
|
||||
script = f"""
|
||||
{get_logs_statement(args)}
|
||||
|
||||
SET max_insert_threads={1 if args.single_thread else 10};
|
||||
SET update_insert_deduplication_token_in_dependent_materialized_views=1;
|
||||
SET deduplicate_blocks_in_dependent_materialized_views=1;
|
||||
|
||||
SET max_block_size=1;
|
||||
SET min_insert_block_size_rows=0;
|
||||
SET min_insert_block_size_bytes=0;
|
||||
|
||||
{drop_tables_statements}
|
||||
|
||||
{create_table_a_b_statement}
|
||||
|
||||
{create_table_when_b_even_statement}
|
||||
|
||||
{create_mv_statement}
|
||||
|
||||
-- first insert
|
||||
{insert_statement}
|
||||
|
||||
{print_details_statements}
|
||||
|
||||
{assert_first_insert_statements}
|
||||
|
||||
-- second insert, it is retry
|
||||
{insert_statement}
|
||||
|
||||
{print_details_statements}
|
||||
|
||||
{assert_second_insert_statements}
|
||||
|
||||
{drop_tables_statements}
|
||||
"""
|
||||
|
||||
print(script)
|
||||
|
||||
parser.set_defaults(func=calle)
|
||||
|
||||
|
||||
def test_mv_generates_several_blocks(parser):
|
||||
ArgsFactory(parser).add_all()
|
||||
|
||||
def calle(args):
|
||||
tables = ["table_for_join_with", "table_a_b", "table_when_b_even_and_joined", "mv_b_even"]
|
||||
drop_tables_statements = get_drop_tables_statements(tables)
|
||||
|
||||
details_print_for_table_for_join_with = ""
|
||||
if args.get_logs:
|
||||
details_print_for_table_for_join_with = """
|
||||
SELECT 'table_for_join_with';
|
||||
SELECT a_join, b, _part FROM table_for_join_with ORDER BY _part, a_join, b;
|
||||
"""
|
||||
|
||||
create_table_a_b_statement = instance_create_statement(
|
||||
table_name="table_a_b",
|
||||
table_columns="(a_src String, b UInt64)",
|
||||
table_keys="(a_src, b)",
|
||||
table_engine=args.table_engine,
|
||||
with_deduplication=args.deduplicate_src_table,
|
||||
)
|
||||
|
||||
create_table_when_b_even_and_joined_statement = instance_create_statement(
|
||||
table_name="table_when_b_even_and_joined",
|
||||
table_columns="(a_src String, a_join String, b UInt64)",
|
||||
table_keys="(a_src, a_join, b)",
|
||||
table_engine=args.table_engine,
|
||||
with_deduplication=args.deduplicate_dst_table,
|
||||
)
|
||||
|
||||
insert_statement = instance_insert_statement(
|
||||
"table_a_b", 5, args.insert_unique_blocks, args.use_insert_token
|
||||
)
|
||||
|
||||
details_print_statements = f"""
|
||||
SELECT 'table_a_b';
|
||||
SELECT 'count', count() FROM table_a_b;
|
||||
|
||||
SELECT 'table_when_b_even_and_joined';
|
||||
SELECT 'count', count() FROM table_when_b_even_and_joined;
|
||||
{"" if not args.get_logs else "SELECT _part, a_src, a_join, b FROM table_when_b_even_and_joined ORDER BY _part;"}
|
||||
"""
|
||||
|
||||
if args.insert_unique_blocks:
|
||||
assert_first_insert_statements = f"""
|
||||
SELECT throwIf( count() != 5 )
|
||||
FROM table_a_b;
|
||||
|
||||
SELECT throwIf( count() != 47 )
|
||||
FROM table_when_b_even_and_joined;
|
||||
"""
|
||||
assert_second_insert_statements = f"""
|
||||
SELECT throwIf( count() != {5 if args.deduplicate_src_table else 10} )
|
||||
FROM table_a_b;
|
||||
|
||||
SELECT throwIf( count() != {47 if args.deduplicate_dst_table else 94} )
|
||||
FROM table_when_b_even_and_joined;
|
||||
"""
|
||||
else:
|
||||
if args.use_insert_token:
|
||||
assert_first_insert_statements = f"""
|
||||
SELECT throwIf( count() != {5 if args.deduplicate_src_table else 5} )
|
||||
FROM table_a_b;
|
||||
|
||||
SELECT throwIf( count() != {45 if args.deduplicate_dst_table else 45} )
|
||||
FROM table_when_b_even_and_joined;
|
||||
"""
|
||||
assert_second_insert_statements = f"""
|
||||
SELECT throwIf( count() != {5 if args.deduplicate_src_table else 10} )
|
||||
FROM table_a_b;
|
||||
|
||||
SELECT throwIf( count() != {45 if args.deduplicate_dst_table else 90} )
|
||||
FROM table_when_b_even_and_joined;
|
||||
"""
|
||||
else:
|
||||
assert_first_insert_statements = f"""
|
||||
SELECT throwIf( count() != {1 if args.deduplicate_src_table else 5} )
|
||||
FROM table_a_b;
|
||||
|
||||
SELECT throwIf( count() != {9 if args.deduplicate_dst_table else 45} )
|
||||
FROM table_when_b_even_and_joined;
|
||||
"""
|
||||
assert_second_insert_statements = f"""
|
||||
SELECT throwIf( count() != {1 if args.deduplicate_src_table else 10} )
|
||||
FROM table_a_b;
|
||||
|
||||
SELECT throwIf( count() != {9 if args.deduplicate_dst_table else 90} )
|
||||
FROM table_when_b_even_and_joined;
|
||||
"""
|
||||
|
||||
script = f"""
|
||||
{get_logs_statement(args)}
|
||||
|
||||
SET max_insert_threads={1 if args.single_thread else 10};
|
||||
SET update_insert_deduplication_token_in_dependent_materialized_views=1;
|
||||
SET deduplicate_blocks_in_dependent_materialized_views=1;
|
||||
|
||||
SET max_block_size=1;
|
||||
SET min_insert_block_size_rows=0;
|
||||
SET min_insert_block_size_bytes=0;
|
||||
|
||||
{drop_tables_statements}
|
||||
|
||||
CREATE TABLE table_for_join_with
|
||||
(a_join String, b UInt64)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (a_join, b);
|
||||
INSERT INTO table_for_join_with
|
||||
SELECT 'joined_' || toString(number), number
|
||||
FROM numbers(9);
|
||||
{details_print_for_table_for_join_with}
|
||||
|
||||
{create_table_a_b_statement}
|
||||
SYSTEM STOP MERGES table_a_b;
|
||||
|
||||
{create_table_when_b_even_and_joined_statement}
|
||||
SYSTEM STOP MERGES table_when_b_even_and_joined;
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_b_even
|
||||
TO table_when_b_even_and_joined
|
||||
AS
|
||||
SELECT a_src, a_join, table_for_join_with.b as b
|
||||
FROM table_a_b
|
||||
FULL OUTER JOIN table_for_join_with
|
||||
ON table_a_b.b = table_for_join_with.b AND table_a_b.b % 2 = 0
|
||||
ORDER BY a_src, a_join, b;
|
||||
|
||||
-- first insert
|
||||
{insert_statement}
|
||||
|
||||
{details_print_statements}
|
||||
|
||||
-- first assertion
|
||||
{assert_first_insert_statements}
|
||||
|
||||
-- second insert
|
||||
{insert_statement}
|
||||
|
||||
{details_print_statements}
|
||||
|
||||
-- second assertion
|
||||
{assert_second_insert_statements}
|
||||
|
||||
{drop_tables_statements}
|
||||
"""
|
||||
|
||||
print(script)
|
||||
|
||||
parser.set_defaults(func=calle)
|
||||
|
||||
|
||||
def test_several_mv_into_one_table(parser):
|
||||
ArgsFactory(parser).add_all()
|
||||
|
||||
def calle(args):
|
||||
tables = ["table_src", "table_dst", "mv_b_even", "mv_b_even_even"]
|
||||
drop_tables_statements = get_drop_tables_statements(tables)
|
||||
|
||||
create_table_src_statement = instance_create_statement(
|
||||
table_name="table_src",
|
||||
table_columns="(a String, b UInt64)",
|
||||
table_keys="(a, b)",
|
||||
table_engine=args.table_engine,
|
||||
with_deduplication=args.deduplicate_src_table,
|
||||
)
|
||||
|
||||
create_table_dst_statement = instance_create_statement(
|
||||
table_name="table_dst",
|
||||
table_columns="(a String, b UInt64)",
|
||||
table_keys="(a, b)",
|
||||
table_engine=args.table_engine,
|
||||
with_deduplication=args.deduplicate_dst_table,
|
||||
)
|
||||
|
||||
insert_statement = instance_insert_statement(
|
||||
"table_src", 8, args.insert_unique_blocks, args.use_insert_token
|
||||
)
|
||||
|
||||
details_print_statements = f"""
|
||||
SELECT 'table_src count', count() FROM table_src;
|
||||
|
||||
SELECT 'table_dst count', count() FROM table_dst;
|
||||
{"" if not args.get_logs else "SELECT _part, count() FROM table_dst GROUP BY _part ORDER BY _part;"}
|
||||
"""
|
||||
|
||||
if args.insert_unique_blocks:
|
||||
assert_first_insert_statements = f"""
|
||||
SELECT throwIf( count() != 8 )
|
||||
FROM table_src;
|
||||
|
||||
SELECT throwIf( count() != 6 )
|
||||
FROM table_dst;
|
||||
"""
|
||||
assert_second_insert_statements = f"""
|
||||
SELECT throwIf( count() != {8 if args.deduplicate_src_table else 16} )
|
||||
FROM table_src;
|
||||
|
||||
SELECT throwIf( count() != {6 if args.deduplicate_dst_table else 12} )
|
||||
FROM table_dst;
|
||||
"""
|
||||
else:
|
||||
if args.use_insert_token:
|
||||
assert_first_insert_statements = f"""
|
||||
SELECT throwIf( count() != {8 if args.deduplicate_src_table else 8} )
|
||||
FROM table_src;
|
||||
|
||||
SELECT throwIf( count() != {16 if args.deduplicate_dst_table else 16} )
|
||||
FROM table_dst;
|
||||
"""
|
||||
assert_second_insert_statements = f"""
|
||||
SELECT throwIf( count() != {8 if args.deduplicate_src_table else 16} )
|
||||
FROM table_src;
|
||||
|
||||
SELECT throwIf( count() != {16 if args.deduplicate_dst_table else 32} )
|
||||
FROM table_dst;
|
||||
"""
|
||||
else:
|
||||
assert_first_insert_statements = f"""
|
||||
SELECT throwIf( count() != {1 if args.deduplicate_src_table else 8} )
|
||||
FROM table_src;
|
||||
|
||||
SELECT throwIf( count() != {1 if args.deduplicate_dst_table else 16} )
|
||||
FROM table_dst;
|
||||
"""
|
||||
assert_second_insert_statements = f"""
|
||||
SELECT throwIf( count() != {1 if args.deduplicate_src_table else 16} )
|
||||
FROM table_src;
|
||||
|
||||
SELECT throwIf( count() != {2 if args.deduplicate_dst_table else 32} )
|
||||
FROM table_dst;
|
||||
"""
|
||||
|
||||
script = f"""
|
||||
{get_logs_statement(args)}
|
||||
|
||||
SET max_insert_threads={1 if args.single_thread else 10};
|
||||
SET update_insert_deduplication_token_in_dependent_materialized_views=1;
|
||||
SET deduplicate_blocks_in_dependent_materialized_views=1;
|
||||
|
||||
SET max_block_size=1;
|
||||
SET min_insert_block_size_rows=0;
|
||||
SET min_insert_block_size_bytes=0;
|
||||
|
||||
{drop_tables_statements}
|
||||
|
||||
{create_table_src_statement}
|
||||
|
||||
{create_table_dst_statement}
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_b_even
|
||||
TO table_dst
|
||||
AS
|
||||
SELECT a, b
|
||||
FROM table_src
|
||||
WHERE b % 2 = 0;
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_b_even_even
|
||||
TO table_dst
|
||||
AS
|
||||
SELECT a, b
|
||||
FROM table_src
|
||||
WHERE b % 4 = 0;
|
||||
|
||||
-- first insert
|
||||
{insert_statement}
|
||||
|
||||
{details_print_statements}
|
||||
|
||||
{assert_first_insert_statements}
|
||||
|
||||
-- second insert, retry
|
||||
{insert_statement}
|
||||
|
||||
{details_print_statements}
|
||||
|
||||
{assert_second_insert_statements}
|
||||
|
||||
{drop_tables_statements}
|
||||
"""
|
||||
|
||||
print(script)
|
||||
|
||||
parser.set_defaults(func=calle)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="test")
|
||||
test_insert_several_blocks(
|
||||
subparsers.add_parser("insert_several_blocks_into_table")
|
||||
)
|
||||
test_mv_generates_several_blocks(
|
||||
subparsers.add_parser("mv_generates_several_blocks")
|
||||
)
|
||||
test_several_mv_into_one_table(
|
||||
subparsers.add_parser("several_mv_into_one_table")
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.test is None:
|
||||
parser.print_help()
|
||||
return args
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
if args.test is not None:
|
||||
args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,870 @@
|
||||
|
||||
Test case 0: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 1: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 2: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 3: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 20
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 4: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 5: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 6: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 7: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 20
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 8: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 1
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 9: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 1
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 10: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 5
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 11: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 10
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 12: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 13: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 14: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 15: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 20
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 16: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 17: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 18: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 19: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 20
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 20: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 21: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 22: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 23: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 20
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 24: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 25: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 26: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 27: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 20
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 28: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 29: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 30: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 31: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 20
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 32: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 33: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 34: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 35: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 20
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 36: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 37: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 38: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 39: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 20
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 40: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 1
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 41: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 1
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 42: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 5
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 43: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 10
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 44: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 45: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 46: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 47: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 20
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 48: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 49: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 50: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 51: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 20
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 52: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 53: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 54: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 55: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 20
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 56: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 57: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 58: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 59: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even
|
||||
count 20
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 60: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 61: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 1
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 62: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 5
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 63: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even
|
||||
count 10
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 20
|
||||
table_when_b_even
|
||||
count 20
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
All cases executed
|
||||
92
tests/queries/0_stateless/03008_deduplication_insert_several_blocks.sh
Executable file
92
tests/queries/0_stateless/03008_deduplication_insert_several_blocks.sh
Executable file
@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
# Test case 8: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
# Test case 9: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
# Test case 10: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
# Test case 11: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
# fails, it is a error. Several blocks in scr table with the same user token are processed in parallel and deduplicated
|
||||
|
||||
# Test case 12: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True"
|
||||
# Test case 13: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False"
|
||||
# fails, it is a error. The same situation as first one, but on dst table.
|
||||
|
||||
RUN_ONLY=""
|
||||
#RUN_ONLY=""
|
||||
|
||||
KNOWN_ERRORS=(8 9 10 11 12 13)
|
||||
|
||||
function is_known_error()
|
||||
{
|
||||
n=$1
|
||||
for e in "${KNOWN_ERRORS[@]}"; do
|
||||
if [ "$n" -eq "$e" ] || [ "$n" -eq "$((e+32))" ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
i=0
|
||||
for engine in "MergeTree" "ReplicatedMergeTree"; do
|
||||
for use_insert_token in "True" "False"; do
|
||||
for single_thread in "True" "False"; do
|
||||
for deduplicate_src_table in "True" "False"; do
|
||||
for deduplicate_dst_table in "True" "False"; do
|
||||
for insert_unique_blocks in "True" "False"; do
|
||||
|
||||
THIS_RUN="Test case $i:"
|
||||
THIS_RUN+=" engine=$engine"
|
||||
THIS_RUN+=" use_insert_token=$use_insert_token"
|
||||
THIS_RUN+=" single_thread=$single_thread"
|
||||
THIS_RUN+=" deduplicate_src_table=$deduplicate_src_table"
|
||||
THIS_RUN+=" deduplicate_dst_table=$deduplicate_dst_table"
|
||||
THIS_RUN+=" insert_unique_blocks=$insert_unique_blocks"
|
||||
|
||||
is_error=$(is_known_error "$i" && echo Y || echo N)
|
||||
i=$((i+1))
|
||||
|
||||
echo
|
||||
if [ -n "$RUN_ONLY" ] && [ "$RUN_ONLY" != "$THIS_RUN" ]; then
|
||||
echo "skip $THIS_RUN"
|
||||
continue
|
||||
fi
|
||||
echo "$THIS_RUN"
|
||||
|
||||
if [ "$is_error" = Y ]; then
|
||||
$CLICKHOUSE_CLIENT -nmq "
|
||||
$(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \
|
||||
--table-engine $engine \
|
||||
--use-insert-token $use_insert_token \
|
||||
--single-thread $single_thread \
|
||||
--deduplicate-src-table $deduplicate_src_table \
|
||||
--deduplicate-dst-table $deduplicate_dst_table \
|
||||
--insert-unique-blocks $insert_unique_blocks \
|
||||
--get-logs false \
|
||||
)
|
||||
" 2>/dev/null && echo FIXED || echo EXPECTED_TO_FAIL
|
||||
else
|
||||
$CLICKHOUSE_CLIENT -nmq "
|
||||
$(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \
|
||||
--table-engine $engine \
|
||||
--use-insert-token $use_insert_token \
|
||||
--single-thread $single_thread \
|
||||
--deduplicate-src-table $deduplicate_src_table \
|
||||
--deduplicate-dst-table $deduplicate_dst_table \
|
||||
--insert-unique-blocks $insert_unique_blocks \
|
||||
--get-logs false \
|
||||
)
|
||||
" && echo OK || echo FAIL
|
||||
fi
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
echo
|
||||
echo "All cases executed"
|
||||
@ -0,0 +1,814 @@
|
||||
|
||||
Test case 0: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 1: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 2: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 94
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 3: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 90
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 4: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 5: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 6: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 94
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 7: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 90
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 8: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 10
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 9: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 10: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 11: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 12: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 10
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 13: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 14: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 94
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 15: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 90
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 16: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 14
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 17: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 18: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 94
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 19: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 90
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 20: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 14
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 21: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 22: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 94
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 23: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 90
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 24: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 14
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 25: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 26: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 94
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 27: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 90
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 28: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 14
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 29: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 30: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 94
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 31: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 90
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 32: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 33: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 34: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 94
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 35: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 90
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 36: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 37: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 38: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 94
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 39: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 90
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 40: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 10
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 41: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 42: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 43: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 44: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 10
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 45: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 46: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 94
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 47: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 90
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 48: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 14
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 49: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 50: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 94
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 51: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 90
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 52: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 14
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 53: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 54: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 94
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 55: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 90
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 56: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 14
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 57: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 58: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 94
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 59: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 1
|
||||
table_when_b_even_and_joined
|
||||
count 90
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 60: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 14
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 61: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 9
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 62: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 47
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 94
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 63: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_a_b
|
||||
count 5
|
||||
table_when_b_even_and_joined
|
||||
count 45
|
||||
0
|
||||
0
|
||||
table_a_b
|
||||
count 10
|
||||
table_when_b_even_and_joined
|
||||
count 90
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
All cases executed
|
||||
98
tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks.sh
Executable file
98
tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks.sh
Executable file
@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
# Test case 8: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
# Test case 9: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
# Test case 10: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
# Test case 11: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
# failed due to race in multi thread insertion, blocks are deduplicated in different threads
|
||||
|
||||
# Test case 12: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
# Test case 13: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
# the same as first but for dst table
|
||||
|
||||
# Test case 16: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
# Test case 20: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
# Test case 24: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
# Test case 28: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
# dst table deduplicates all incoming blocks from one insert because not uniq hash
|
||||
|
||||
RUN_ONLY=""
|
||||
#RUN_ONLY="Test case 20: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True"
|
||||
|
||||
KNOWN_ERRORS=(8 9 10 11 12 13 16 20 24 28)
|
||||
|
||||
function is_known_error()
|
||||
{
|
||||
n=$1
|
||||
for e in "${KNOWN_ERRORS[@]}"; do
|
||||
if [ "$n" -eq "$e" ] || [ "$n" -eq "$((e+32))" ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
i=0
|
||||
for engine in "MergeTree" "ReplicatedMergeTree"; do
|
||||
for use_insert_token in "True" "False"; do
|
||||
for single_thread in "True" "False"; do
|
||||
for deduplicate_src_table in "True" "False"; do
|
||||
for deduplicate_dst_table in "True" "False"; do
|
||||
for insert_unique_blocks in "True" "False"; do
|
||||
|
||||
THIS_RUN="Test case $i:"
|
||||
THIS_RUN+=" engine=$engine"
|
||||
THIS_RUN+=" use_insert_token=$use_insert_token"
|
||||
THIS_RUN+=" single_thread=$single_thread"
|
||||
THIS_RUN+=" deduplicate_src_table=$deduplicate_src_table"
|
||||
THIS_RUN+=" deduplicate_dst_table=$deduplicate_dst_table"
|
||||
THIS_RUN+=" insert_unique_blocks=$insert_unique_blocks"
|
||||
|
||||
is_error=$(is_known_error "$i" && echo Y || echo N)
|
||||
i=$((i+1))
|
||||
|
||||
echo
|
||||
if [ -n "$RUN_ONLY" ] && [ "$RUN_ONLY" != "$THIS_RUN" ]; then
|
||||
echo "skip $THIS_RUN"
|
||||
continue
|
||||
fi
|
||||
echo "$THIS_RUN"
|
||||
|
||||
if [ "$is_error" = Y ]; then
|
||||
$CLICKHOUSE_CLIENT -nmq "
|
||||
$(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \
|
||||
--table-engine $engine \
|
||||
--use-insert-token $use_insert_token \
|
||||
--single-thread $single_thread \
|
||||
--deduplicate-src-table $deduplicate_src_table \
|
||||
--deduplicate-dst-table $deduplicate_dst_table \
|
||||
--insert-unique-blocks $insert_unique_blocks \
|
||||
--get-logs false \
|
||||
)
|
||||
" 2>/dev/null && echo FIXED || echo EXPECTED_TO_FAIL
|
||||
else
|
||||
$CLICKHOUSE_CLIENT -nmq "
|
||||
$(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \
|
||||
--table-engine $engine \
|
||||
--use-insert-token $use_insert_token \
|
||||
--single-thread $single_thread \
|
||||
--deduplicate-src-table $deduplicate_src_table \
|
||||
--deduplicate-dst-table $deduplicate_dst_table \
|
||||
--insert-unique-blocks $insert_unique_blocks \
|
||||
--get-logs false \
|
||||
)
|
||||
" && echo OK || echo FAIL
|
||||
fi
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
echo
|
||||
echo "All cases executed"
|
||||
@ -0,0 +1,590 @@
|
||||
|
||||
Test case 0: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 1: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 2: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 8
|
||||
table_dst count 12
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 3: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 8
|
||||
table_dst count 32
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 4: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 5: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 6: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 12
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 7: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 32
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 8: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 1
|
||||
table_dst count 2
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 9: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 1
|
||||
table_dst count 2
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 10: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 1
|
||||
table_dst count 6
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 11: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 1
|
||||
table_dst count 16
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 12: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 2
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 13: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 2
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 14: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 12
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 15: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 32
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 16: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 4
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 17: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 1
|
||||
table_dst count 1
|
||||
0
|
||||
0
|
||||
table_src count 1
|
||||
table_dst count 1
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 18: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 8
|
||||
table_dst count 12
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 19: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 1
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 1
|
||||
table_dst count 32
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 20: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 4
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 21: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 1
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 1
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 22: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 12
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 23: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 32
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 24: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 4
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 25: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 1
|
||||
table_dst count 1
|
||||
0
|
||||
0
|
||||
table_src count 1
|
||||
table_dst count 1
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 26: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 8
|
||||
table_dst count 12
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 27: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 1
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 1
|
||||
table_dst count 32
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 28: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 4
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 29: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 1
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 1
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 30: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 12
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 31: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 32
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 32: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 33: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 34: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 8
|
||||
table_dst count 12
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 35: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 8
|
||||
table_dst count 32
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 36: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 37: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 38: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 12
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 39: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 32
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 40: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 1
|
||||
table_dst count 2
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 41: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 1
|
||||
table_dst count 2
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 42: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 1
|
||||
table_dst count 6
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 43: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 1
|
||||
table_dst count 16
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 44: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 2
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 45: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 2
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 46: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 12
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 47: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 32
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 48: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 4
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 49: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 1
|
||||
table_dst count 1
|
||||
0
|
||||
0
|
||||
table_src count 1
|
||||
table_dst count 1
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 50: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 8
|
||||
table_dst count 12
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 51: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 1
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 1
|
||||
table_dst count 32
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 52: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 4
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 53: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 1
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 1
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 54: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 12
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 55: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 32
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 56: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 4
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 57: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 1
|
||||
table_dst count 1
|
||||
0
|
||||
0
|
||||
table_src count 1
|
||||
table_dst count 1
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 58: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 8
|
||||
table_dst count 12
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 59: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 1
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 1
|
||||
table_dst count 32
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 60: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 4
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 61: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 1
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 1
|
||||
0
|
||||
EXPECTED_TO_FAIL
|
||||
|
||||
Test case 62: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
|
||||
table_src count 8
|
||||
table_dst count 6
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 12
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
Test case 63: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
|
||||
table_src count 8
|
||||
table_dst count 16
|
||||
0
|
||||
0
|
||||
table_src count 16
|
||||
table_dst count 32
|
||||
0
|
||||
0
|
||||
OK
|
||||
|
||||
All cases executed
|
||||
106
tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table.sh
Executable file
106
tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table.sh
Executable file
@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
# Test case 8: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
# Test case 9: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
# Test case 10: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
|
||||
# Test case 11: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
|
||||
# race condition on insert into src table
|
||||
|
||||
# Test case 12: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
# Test case 13: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
# race condition on insert into dst table
|
||||
|
||||
# Test case 16: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
# Test case 20: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
# Test case 24: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
|
||||
# Test case 28: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
|
||||
# dst deduplicates blocks from one inserts from different materialized view
|
||||
|
||||
# Test case 17: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
# Test case 21: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
# Test case 25: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
|
||||
# Test case 29: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
|
||||
# dst deduplicates blocks from different inserts by hash
|
||||
|
||||
KNOWN_ERRORS=(8 9 10 11 12 13 16 20 24 28 17 21 25 29)
|
||||
|
||||
function is_known_error()
|
||||
{
|
||||
n=$1
|
||||
for e in "${KNOWN_ERRORS[@]}"; do
|
||||
if [ "$n" -eq "$e" ] || [ "$n" -eq "$((e+32))" ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
RUN_ONLY=""
|
||||
#RUN_ONLY="Test case 0: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True"
|
||||
|
||||
i=0
|
||||
for engine in "MergeTree" "ReplicatedMergeTree"; do
|
||||
for use_insert_token in "True" "False"; do
|
||||
for single_thread in "True" "False"; do
|
||||
for deduplicate_src_table in "True" "False"; do
|
||||
for deduplicate_dst_table in "True" "False"; do
|
||||
for insert_unique_blocks in "True" "False"; do
|
||||
|
||||
THIS_RUN="Test case $i:"
|
||||
THIS_RUN+=" engine=$engine"
|
||||
THIS_RUN+=" use_insert_token=$use_insert_token"
|
||||
THIS_RUN+=" single_thread=$single_thread"
|
||||
THIS_RUN+=" deduplicate_src_table=$deduplicate_src_table"
|
||||
THIS_RUN+=" deduplicate_dst_table=$deduplicate_dst_table"
|
||||
THIS_RUN+=" insert_unique_blocks=$insert_unique_blocks"
|
||||
|
||||
is_error=$(is_known_error "$i" && echo Y || echo N)
|
||||
i=$((i+1))
|
||||
|
||||
echo
|
||||
if [ -n "$RUN_ONLY" ] && [ "$RUN_ONLY" != "$THIS_RUN" ]; then
|
||||
echo "skip $THIS_RUN"
|
||||
continue
|
||||
fi
|
||||
echo "$THIS_RUN"
|
||||
|
||||
if [ "$is_error" = Y ]; then
|
||||
$CLICKHOUSE_CLIENT -nmq "
|
||||
$(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \
|
||||
--table-engine $engine \
|
||||
--use-insert-token $use_insert_token \
|
||||
--single-thread $single_thread \
|
||||
--deduplicate-src-table $deduplicate_src_table \
|
||||
--deduplicate-dst-table $deduplicate_dst_table \
|
||||
--insert-unique-blocks $insert_unique_blocks \
|
||||
--get-logs false \
|
||||
)
|
||||
" 2>/dev/null && echo FIXED || echo EXPECTED_TO_FAIL
|
||||
else
|
||||
$CLICKHOUSE_CLIENT -nmq "
|
||||
$(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \
|
||||
--table-engine $engine \
|
||||
--use-insert-token $use_insert_token \
|
||||
--single-thread $single_thread \
|
||||
--deduplicate-src-table $deduplicate_src_table \
|
||||
--deduplicate-dst-table $deduplicate_dst_table \
|
||||
--insert-unique-blocks $insert_unique_blocks \
|
||||
--get-logs false \
|
||||
)
|
||||
" && echo OK || echo FAIL
|
||||
fi
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
echo
|
||||
echo "All cases executed"
|
||||
|
||||
|
||||
@ -1,93 +0,0 @@
|
||||
DROP TABLE IF EXISTS table_a_b;
|
||||
DROP TABLE IF EXISTS table_when_b_even;
|
||||
DROP TABLE IF EXISTS mv_b_even;
|
||||
|
||||
|
||||
SET max_insert_threads=1;
|
||||
SET update_insert_deduplication_token_in_dependent_materialized_views=1;
|
||||
SET deduplicate_blocks_in_dependent_materialized_views=1;
|
||||
|
||||
SET max_block_size=3;
|
||||
SET min_insert_block_size_rows=0;
|
||||
SET min_insert_block_size_bytes=0;
|
||||
|
||||
|
||||
CREATE TABLE table_a_b
|
||||
(
|
||||
a String,
|
||||
b UInt64,
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (a, b)
|
||||
SETTINGS non_replicated_deduplication_window=10000;
|
||||
SYSTEM STOP MERGES table_a_b;
|
||||
|
||||
CREATE TABLE table_when_b_even_wo_dedup
|
||||
(
|
||||
a String,
|
||||
b UInt64,
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (a, b)
|
||||
SETTINGS non_replicated_deduplication_window=0;
|
||||
SYSTEM STOP MERGES table_when_b_even;
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_b_even_wo_dedup
|
||||
TO table_when_b_even_wo_dedup
|
||||
AS
|
||||
SELECT a, b
|
||||
FROM table_a_b
|
||||
WHERE b % 2 = 0;
|
||||
|
||||
CREATE TABLE table_when_b_even_dedup
|
||||
(
|
||||
a String,
|
||||
b UInt64,
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (a, b)
|
||||
SETTINGS non_replicated_deduplication_window=10000;
|
||||
SYSTEM STOP MERGES table_when_b_even;
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_b_even_dedup
|
||||
TO table_when_b_even_dedup
|
||||
AS
|
||||
SELECT a, b
|
||||
FROM table_a_b
|
||||
WHERE b % 2 = 0;
|
||||
|
||||
|
||||
SELECT 'first insert'
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
INSERT INTO table_a_b
|
||||
SELECT toString(number DIV 2), number
|
||||
FROM numbers(5)
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
|
||||
SELECT 'second insert'
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
INSERT INTO table_a_b
|
||||
SELECT toString(number DIV 2), number
|
||||
FROM numbers(5)
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
|
||||
SELECT 'table_a_b';
|
||||
SELECT 'count', count() FROM table_a_b;
|
||||
SELECT _part, count() FROM table_a_b GROUP BY _part;
|
||||
|
||||
SELECT 'table_when_b_even_wo_dedup';
|
||||
SELECT 'count', count() FROM table_when_b_even_wo_dedup;
|
||||
SELECT _part, count() FROM table_when_b_even_wo_dedup GROUP BY _part;
|
||||
|
||||
SELECT 'table_when_b_even_dedup';
|
||||
SELECT 'count', count() FROM table_when_b_even_dedup;
|
||||
SELECT _part, count() FROM table_when_b_even_dedup GROUP BY _part;
|
||||
|
||||
|
||||
DROP TABLE mv_b_even;
|
||||
DROP TABLE table_when_b_even;
|
||||
DROP TABLE table_a_b;
|
||||
@ -1,113 +0,0 @@
|
||||
DROP TABLE IF EXISTS test;
|
||||
|
||||
CREATE TABLE table_for_join_with
|
||||
(
|
||||
a_join String,
|
||||
b UInt64
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (a_join, b);
|
||||
|
||||
INSERT INTO table_for_join_with
|
||||
SELECT 'joined_' || toString(number), number
|
||||
FROM numbers(10);
|
||||
SELECT 'table_for_join_with';
|
||||
SELECT a_join, b, _part FROM table_for_join_with ORDER BY _part, a_join, b;
|
||||
|
||||
|
||||
CREATE TABLE table_a_b
|
||||
(
|
||||
a_src String,
|
||||
b UInt64
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (a_src, b)
|
||||
SETTINGS non_replicated_deduplication_window=10000;
|
||||
SYSTEM STOP MERGES table_a_b;
|
||||
|
||||
CREATE TABLE table_when_b_even_dedup
|
||||
(
|
||||
a_src String CODEC(NONE),
|
||||
a_join String CODEC(NONE),
|
||||
b UInt64 CODEC(NONE)
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (a_src, a_join, b)
|
||||
SETTINGS non_replicated_deduplication_window=10000;
|
||||
SYSTEM STOP MERGES table_when_b_even_dedup;
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_b_even_dedup
|
||||
TO table_when_b_even_dedup
|
||||
AS
|
||||
SELECT a_src, a_join, b
|
||||
FROM table_a_b
|
||||
FULL OUTER JOIN table_for_join_with
|
||||
ON table_a_b.b = table_for_join_with.b AND table_a_b.b % 2 = 0
|
||||
ORDER BY a_src, a_join, b;
|
||||
|
||||
CREATE TABLE table_when_b_even_wo_dedup
|
||||
(
|
||||
a_src String CODEC(NONE),
|
||||
a_join String CODEC(NONE),
|
||||
b UInt64 CODEC(NONE)
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (a_src, a_join, b)
|
||||
SETTINGS non_replicated_deduplication_window=0;
|
||||
SYSTEM STOP MERGES table_when_b_even_wo_dedup;
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_b_even_wo_dedup
|
||||
TO table_when_b_even_wo_dedup
|
||||
AS
|
||||
SELECT a_src, a_join, b
|
||||
FROM table_a_b
|
||||
FULL OUTER JOIN table_for_join_with
|
||||
ON table_a_b.b = table_for_join_with.b AND table_a_b.b % 2 = 0
|
||||
ORDER BY a_src, a_join, b;
|
||||
|
||||
|
||||
SET max_insert_threads=1;
|
||||
SET update_insert_deduplication_token_in_dependent_materialized_views=1;
|
||||
SET deduplicate_blocks_in_dependent_materialized_views=1;
|
||||
|
||||
SET max_block_size=1;
|
||||
SET min_insert_block_size_rows=0;
|
||||
SET min_insert_block_size_bytes=0;
|
||||
|
||||
|
||||
SELECT 'first insert'
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
INSERT INTO table_a_b
|
||||
SELECT 'source_' || toString(number), number
|
||||
FROM numbers(5)
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
|
||||
SELECT 'second insert'
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
INSERT INTO table_a_b
|
||||
SELECT 'source_' || toString(number), number
|
||||
FROM numbers(5)
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
|
||||
SELECT 'table_a_b';
|
||||
SELECT 'count', count() FROM table_a_b;
|
||||
SELECT _part, count() FROM table_a_b GROUP BY _part;
|
||||
|
||||
SELECT 'table_when_b_even_dedup, here the result if join is deduplicated inside one request, it is not correct';
|
||||
SELECT 'count', count() FROM table_when_b_even_dedup;
|
||||
SELECT _part, count() FROM table_when_b_even_dedup GROUP BY _part;
|
||||
|
||||
SELECT 'table_when_b_even_wo_dedup';
|
||||
SELECT 'count', count() FROM table_when_b_even_wo_dedup;
|
||||
SELECT _part, count() FROM table_when_b_even_wo_dedup GROUP BY _part ORDER BY _part;
|
||||
|
||||
|
||||
DROP TABLE mv_b_even_dedup;
|
||||
DROP TABLE table_when_b_even_dedup;
|
||||
DROP TABLE mv_b_even_wo_dedup;
|
||||
DROP TABLE table_when_b_even_wo_dedup;
|
||||
DROP TABLE table_a_b;
|
||||
@ -1,119 +0,0 @@
|
||||
DROP TABLE IF EXISTS test;
|
||||
|
||||
CREATE TABLE table_source
|
||||
(
|
||||
a String,
|
||||
b UInt64
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (a, b)
|
||||
SETTINGS non_replicated_deduplication_window=10000;
|
||||
SYSTEM STOP MERGES table_source;
|
||||
|
||||
CREATE TABLE table_dst_dedup
|
||||
(
|
||||
a String,
|
||||
b UInt64
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (a, b)
|
||||
SETTINGS non_replicated_deduplication_window=10000;
|
||||
SYSTEM STOP MERGES table_dst_dedup;
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_b_even_dedup
|
||||
TO table_dst_dedup
|
||||
AS
|
||||
SELECT a, b
|
||||
FROM table_source
|
||||
WHERE b % 2 = 0;
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_b_even_even_dedup
|
||||
TO table_dst_dedup
|
||||
AS
|
||||
SELECT a, b
|
||||
FROM table_source
|
||||
WHERE b % 4 = 0;
|
||||
|
||||
CREATE TABLE table_dst_wo_dedup
|
||||
(
|
||||
a String,
|
||||
b UInt64
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (a, b)
|
||||
SETTINGS non_replicated_deduplication_window=0;
|
||||
SYSTEM STOP MERGES table_dst_wo_dedup;
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_b_even_wo_dedup
|
||||
TO table_dst_wo_dedup
|
||||
AS
|
||||
SELECT a, b
|
||||
FROM table_source
|
||||
WHERE b % 2 = 0;
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_b_even_wo_even_dedup
|
||||
TO table_dst_wo_dedup
|
||||
AS
|
||||
SELECT a, b
|
||||
FROM table_source
|
||||
WHERE b % 4 = 0;
|
||||
|
||||
|
||||
SET max_insert_threads=1;
|
||||
SET update_insert_deduplication_token_in_dependent_materialized_views=1;
|
||||
SET deduplicate_blocks_in_dependent_materialized_views=1;
|
||||
|
||||
SET max_block_size=1;
|
||||
SET min_insert_block_size_rows=0;
|
||||
SET min_insert_block_size_bytes=0;
|
||||
|
||||
|
||||
SELECT 'first insert'
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
INSERT INTO table_source
|
||||
SELECT 'source_' || toString(number), number
|
||||
FROM numbers(8)
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
SELECT 'table_source';
|
||||
SELECT 'count', count() FROM table_source;
|
||||
SELECT _part, count() FROM table_source GROUP BY _part ORDER BY _part;
|
||||
|
||||
SELECT 'table_dst_dedup';
|
||||
SELECT 'count', count() FROM table_dst_dedup;
|
||||
SELECT _part, count() FROM table_dst_dedup GROUP BY _part ORDER BY _part;
|
||||
|
||||
SELECT 'table_dst_wo_dedup';
|
||||
SELECT 'count', count() FROM table_dst_wo_dedup;
|
||||
SELECT _part, count() FROM table_dst_wo_dedup GROUP BY _part ORDER BY _part;
|
||||
|
||||
|
||||
SELECT 'second insert'
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
INSERT INTO table_source
|
||||
SELECT 'source_' || toString(number), number
|
||||
FROM numbers(8)
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
SELECT 'table_source';
|
||||
SELECT 'count', count() FROM table_source;
|
||||
SELECT _part, count() FROM table_source GROUP BY _part ORDER BY _part;
|
||||
|
||||
SELECT 'table_dst_dedup, block from different mv is deduplicated, it is wrong';
|
||||
SELECT 'count', count() FROM table_dst_dedup;
|
||||
SELECT _part, count() FROM table_dst_dedup GROUP BY _part ORDER BY _part;
|
||||
|
||||
SELECT 'table_dst_wo_dedup';
|
||||
SELECT 'count', count() FROM table_dst_wo_dedup;
|
||||
SELECT _part, count() FROM table_dst_wo_dedup GROUP BY _part ORDER BY _part;
|
||||
|
||||
|
||||
DROP TABLE mv_b_even_dedup;
|
||||
DROP TABLE mv_b_even_even_dedup;
|
||||
DROP TABLE mv_b_even_wo_dedup;
|
||||
DROP TABLE mv_b_even_even_wo_dedup;
|
||||
DROP TABLE table_dst_dedup;
|
||||
DROP TABLE table_dst_wo_dedup;
|
||||
DROP TABLE table_source;
|
||||
@ -1,76 +0,0 @@
|
||||
DROP TABLE IF EXISTS test;
|
||||
|
||||
CREATE TABLE table_a_b
|
||||
(
|
||||
a String,
|
||||
b UInt64
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (a, b)
|
||||
SETTINGS non_replicated_deduplication_window=10000;
|
||||
SYSTEM STOP MERGES table_a_b;
|
||||
|
||||
CREATE TABLE table_when_b_even
|
||||
(
|
||||
a String CODEC(NONE),
|
||||
b UInt64 CODEC(NONE)
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (a, b)
|
||||
SETTINGS non_replicated_deduplication_window=10000;
|
||||
SYSTEM STOP MERGES table_when_b_even;
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_b_even
|
||||
TO table_when_b_even
|
||||
AS
|
||||
SELECT a, b
|
||||
FROM table_a_b
|
||||
WHERE b % 2 = 0;
|
||||
|
||||
|
||||
SET max_insert_threads=1;
|
||||
SET update_insert_deduplication_token_in_dependent_materialized_views=1;
|
||||
SET deduplicate_blocks_in_dependent_materialized_views=1;
|
||||
|
||||
SET max_block_size=1;
|
||||
SET min_insert_block_size_rows=0;
|
||||
SET min_insert_block_size_bytes=0;
|
||||
|
||||
|
||||
SELECT 'first insert'
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
INSERT INTO table_a_b
|
||||
SELECT 'source_' || toString(1), 1
|
||||
FROM numbers(5)
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
SELECT 'table_a_b, it deduplicates rows within one insert, it is wrong';
|
||||
SELECT 'count', count() FROM table_a_b;
|
||||
SELECT _part, count() FROM table_a_b GROUP BY _part ORDER BY _part;
|
||||
|
||||
SELECT 'table_when_b_even';
|
||||
SELECT 'count', count() FROM table_when_b_even;
|
||||
SELECT _part, count() FROM table_when_b_even GROUP BY _part ORDER BY _part;
|
||||
|
||||
|
||||
SELECT 'second insert'
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
INSERT INTO table_a_b
|
||||
SELECT 'source_' || toString(1), 1
|
||||
FROM numbers(5)
|
||||
SETTINGS send_logs_level='trace';
|
||||
|
||||
SELECT 'table_a_b';
|
||||
SELECT 'count', count() FROM table_a_b;
|
||||
SELECT _part, count() FROM table_a_b GROUP BY _part;
|
||||
|
||||
SELECT 'table_when_b_even';
|
||||
SELECT 'count', count() FROM table_when_b_even;
|
||||
SELECT _part, count() FROM table_when_b_even GROUP BY _part;
|
||||
|
||||
|
||||
DROP TABLE mv_b_even;
|
||||
DROP TABLE table_when_b_even;
|
||||
DROP TABLE table_a_b;
|
||||
Loading…
Reference in New Issue
Block a user