non replicated inserts with deduplication user token

src/Interpreters/InterpreterInsertQuery.cpp

@@ -519,29 +519,29 @@ BlockIO InterpreterInsertQuery::execute()
 
             if (settings.max_insert_threads > 1)
             {
-                auto table_id = table->getStorageID();
+                pre_streams_size = std::max<size_t>(settings.max_insert_threads, pipeline.getNumStreams());
-                auto views = DatabaseCatalog::instance().getDependentViews(table_id);
 
-                /// It breaks some views-related tests and we have dedicated `parallel_view_processing` for views, so let's just skip them.
-                /// Also it doesn't make sense to reshuffle data if storage doesn't support parallel inserts.
-                const bool resize_to_max_insert_threads = !table->isView() && views.empty() && table->supportsParallelInsert();
-                pre_streams_size = resize_to_max_insert_threads ? settings.max_insert_threads
-                                                                : std::min<size_t>(settings.max_insert_threads, pipeline.getNumStreams());
 
-                /// Deduplication when passing insert_deduplication_token breaks if using more than one thread
+//                /// Deduplication when passing insert_deduplication_token breaks if using more than one thread
-                if (!settings.insert_deduplication_token.toString().empty())
+//                if (!settings.insert_deduplication_token.toString().empty())
-                {
+//                {
-                    /// TODO!
+//                    /// TODO!
-                    LOG_DEBUG(
+//                    LOG_DEBUG(
-                        getLogger("InsertQuery"),
+//                        getLogger("InsertQuery"),
-                        "Insert-select query using insert_deduplication_token, setting streams to 1 to avoid deduplication issues");
+//                        "Insert-select query using insert_deduplication_token, setting streams from {} to 1 to avoid deduplication issues, pipeline.getNumStreams() {}",
-                    pre_streams_size = 1;
+//                        pre_streams_size, pipeline.getNumStreams());
-                }
+//                    pre_streams_size = 1;
+//                }
 
                 if (table->supportsParallelInsert())
                     sink_streams_size = pre_streams_size;
             }
 
+            LOG_DEBUG(
+                getLogger("InsertQuery"),
+                "pre_streams_size {}, pipeline.getNumStreams() {}",
+                pre_streams_size, pipeline.getNumStreams());
+
             pipeline.resize(pre_streams_size);
 
             /// Allow to insert Nullable into non-Nullable columns, NULL values will be added as defaults values.

src/Processors/Transforms/buildPushingToViewsChain.cpp

@@ -487,7 +487,7 @@ Chain buildPushingToViewsChain(
 
     for (const auto & view_id : views)
     {
-        LOG_ERROR(&Poco::Logger::get("PushingToViews"), "dependent view: {}.{}", view_id.database_name, view_id.table_name);
+        LOG_DEBUG(&Poco::Logger::get("PushingToViews"), "dependent view: {}.{}", view_id.database_name, view_id.table_name);
 
         try
         {

src/Storages/MergeTree/MergeTreeDataWriter.h

@@ -47,8 +47,7 @@ public:
         : data(data_)
         , log(getLogger(data.getLogName() + " (Writer)"))
     {
-        LOG_WARNING(log, "MergeTreeDataWriter() called from:\n{}", StackTrace().toString());
+        LOG_DEBUG(log, "MergeTreeDataWriter() called from:\n{}", StackTrace().toString());
-
     }
 
     /** Split the block to blocks, each of them must be written as separate part.

src/Storages/MergeTree/MergedBlockOutputStream.cpp

@@ -51,7 +51,7 @@ MergedBlockOutputStream::MergedBlockOutputStream(
 
     writer = data_part->getWriter(columns_list, metadata_snapshot, skip_indices, statistics, default_codec, writer_settings, computed_index_granularity);
 
-    LOG_WARNING(getLogger("MergedBlockOutputStream()"), "called c-tor");
+    LOG_DEBUG(getLogger("MergedBlockOutputStream()"), "called c-tor");
 }
 
 /// If data is pre-sorted.
@@ -331,7 +331,7 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis
 
 void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Permutation * permutation)
 {
-    LOG_WARNING(getLogger("MergedBlockOutputStream()"), "writeImpl block rows {} size {} getPartDirectory {}",
+    LOG_DEBUG(getLogger("MergedBlockOutputStream()"), "writeImpl block rows {} size {} getPartDirectory {}",
                 block.rows(), block.bytes(), data_part_storage->getPartDirectory());
 
     block.checkNumberOfRows();

tests/queries/0_stateless/03008_deduplication.python

@@ -0,0 +1,561 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import argparse
+import string
+
+
+CURDIR = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, os.path.join(CURDIR, "helpers"))
+
+
+def __format(template, **params):
+    field_names = [v[1] for v in string.Formatter().parse(template) if v[1] is not None]
+    kv_args = {}
+    for field in field_names:
+        if field in params:
+            kv_args[field] = params[field]
+        else:
+            kv_args[field] = ""
+
+    return template.format(**kv_args)
+
+
+def instance_create_statement(table_name, table_columns, table_keys, table_engine, with_deduplication, no_merges=True):
+    template = """
+        CREATE TABLE {table_name}
+        {table_columns}
+        ENGINE = {table_engine}
+        ORDER BY {table_keys}
+        {table_settings};
+        {table_no_merges}
+        """
+
+    params = dict()
+    params["table_name"] = table_name
+    params["table_columns"] = table_columns
+    params["table_keys"] = table_keys
+    params["table_no_merges"] = f"SYSTEM STOP MERGES {table_name};" if no_merges else ""
+    params["table_engine"] = "MergeTree()" if table_engine == "MergeTree" else f"ReplicatedMergeTree('/clickhouse/tables/{{database}}/{table_name}', '1')"
+
+    deduplication_window_setting_name = "non_replicated_deduplication_window" if table_engine == "MergeTree" else "replicated_deduplication_window"
+    deduplication_window_setting_value = 1000 if with_deduplication else 0
+
+    settings = list()
+    settings += [f"{deduplication_window_setting_name}={deduplication_window_setting_value}"]
+    params["table_settings"] = "SETTINGS " + ",".join(settings)
+
+    return __format(template, **params)
+
+
+def instance_insert_statement(table_name, count, insert_unique_blocks, use_insert_token):
+    template = """
+        INSERT INTO {table_name}
+            SELECT {insert_columns}
+            FROM numbers({count}) {insert_settings};
+        """
+    return __format(
+        template,
+        table_name=table_name,
+        count=count,
+        insert_columns="'src_4', 4" if not insert_unique_blocks else "'src_' || toString(number), number",
+        insert_settings="" if not use_insert_token else "SETTINGS insert_deduplication_token='UDT'",
+    )
+
+
+def get_drop_tables_statements(tables):
+    return "".join([f"DROP TABLE IF EXISTS {table_name};\n" for table_name in tables[::-1]])
+
+
+def get_logs_statement(args):
+    if args.get_logs:
+        return "SET send_logs_level='test';"
+    return ""
+
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ('yes', 'true', 't', 'y', '1'):
+        return True
+    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+        return False
+    else:
+        raise argparse.ArgumentTypeError('Boolean value expected.')
+
+class ArgsFactory:
+    def __init__(self, parser):
+        self.__parser = parser
+
+    def add_opt_engine(self):
+        self.__parser.add_argument(
+            "--table-engine", choices=["ReplicatedMergeTree", "MergeTree"], default="MergeTree")
+
+    def add_opt_user_token(self):
+        self.__parser.add_argument("--use-insert-token", type=str2bool, nargs='?', const=True, default=False)
+
+    def add_opt_single_thread(self):
+        self.__parser.add_argument("--single-thread", type=str2bool, nargs='?', const=True, default=True)
+
+    def add_opt_dedup_src(self):
+        self.__parser.add_argument("--deduplicate-src-table", type=str2bool, nargs='?', const=True, default=True)
+
+    def add_opt_dedup_dst(self):
+        self.__parser.add_argument("--deduplicate-dst-table", type=str2bool, nargs='?', const=True, default=True)
+
+    def add_opt_get_logs(self):
+        self.__parser.add_argument("--get-logs", type=str2bool, nargs='?', const=True, default=False)
+
+    def add_opt_uniq_blocks(self):
+        self.__parser.add_argument("--insert-unique-blocks", type=str2bool, nargs='?', const=True, default=True)
+
+    def add_all(self):
+        self.add_opt_engine()
+        self.add_opt_user_token()
+        self.add_opt_single_thread()
+        self.add_opt_dedup_src()
+        self.add_opt_dedup_dst()
+        self.add_opt_get_logs()
+        self.add_opt_uniq_blocks()
+
+
+def test_insert_several_blocks(parser):
+    ArgsFactory(parser).add_all()
+
+    def calle(args):
+        create_table_a_b_statement = instance_create_statement(
+            table_name="table_a_b",
+            table_columns="(a String, b UInt64)",
+            table_keys="(a, b)",
+            table_engine=args.table_engine,
+            with_deduplication=args.deduplicate_src_table,
+        )
+
+        create_table_when_b_even_statement = instance_create_statement(
+            table_name="table_when_b_even",
+            table_columns="(a String, b UInt64)",
+            table_keys="(a, b)",
+            table_engine=args.table_engine,
+            with_deduplication=args.deduplicate_dst_table,
+        )
+
+        create_mv_statement = """
+            CREATE MATERIALIZED VIEW mv_b_even
+            TO table_when_b_even
+            AS
+                SELECT a, b
+                FROM table_a_b
+                WHERE b % 2 = 0;
+            """
+
+        drop_tables_statements = get_drop_tables_statements( ["table_a_b", "table_when_b_even", "mv_b_even"] )
+
+        insert_statement = instance_insert_statement(
+            "table_a_b", 10, args.insert_unique_blocks, args.use_insert_token
+        )
+
+        print_details_statements = f"""
+            SELECT 'table_a_b';
+            SELECT 'count', count() FROM table_a_b;
+            {"" if not args.get_logs else "SELECT _part, count() FROM table_a_b GROUP BY _part ORDER BY _part;"}
+            
+            SELECT 'table_when_b_even';
+            SELECT 'count', count() FROM table_when_b_even;
+            {"" if not args.get_logs else "SELECT _part, count() FROM table_when_b_even GROUP BY _part ORDER BY _part;"}
+            """
+
+
+
+        if args.insert_unique_blocks:
+            assert_first_insert_statements = f"""
+                SELECT throwIf( count() != 10 )
+                    FROM table_a_b;
+                SELECT throwIf( count() != 5 )
+                    FROM table_when_b_even;
+                """
+            assert_second_insert_statements = f"""
+               SELECT throwIf( count() != {10 if args.deduplicate_src_table else 20} )
+                   FROM table_a_b;
+               SELECT throwIf( count() != {5 if args.deduplicate_dst_table else 10} )
+                   FROM table_when_b_even;
+               """
+        else:
+            if args.use_insert_token:
+                assert_first_insert_statements = """
+                                SELECT throwIf( count() != 10 )
+                                    FROM table_a_b;
+                                SELECT throwIf( count() != 10 )
+                                    FROM table_when_b_even;
+                                """
+                assert_second_insert_statements = f"""
+                               SELECT throwIf( count() != {10 if args.deduplicate_src_table else 20} )
+                                   FROM table_a_b;
+                               SELECT throwIf( count() != {10 if args.deduplicate_dst_table else 20} )
+                                   FROM table_when_b_even;
+                               """
+            else:
+                assert_first_insert_statements = f"""
+                    SELECT throwIf( count() != {1 if args.deduplicate_src_table else 10} )
+                        FROM table_a_b;
+                    SELECT throwIf( count() != {1 if args.deduplicate_dst_table else 10} )
+                        FROM table_when_b_even;
+                    """
+                assert_second_insert_statements = f"""
+                    SELECT throwIf( count() != {1 if args.deduplicate_src_table else 20} )
+                        FROM table_a_b;
+                    SELECT throwIf( count() != {1 if args.deduplicate_dst_table else 20} )
+                        FROM table_when_b_even;
+                    """
+
+        script = f"""
+            {get_logs_statement(args)}
+            
+            SET max_insert_threads={1 if args.single_thread else 10};
+            SET update_insert_deduplication_token_in_dependent_materialized_views=1;
+            SET deduplicate_blocks_in_dependent_materialized_views=1;
+            
+            SET max_block_size=1;
+            SET min_insert_block_size_rows=0;
+            SET min_insert_block_size_bytes=0;
+                    
+            {drop_tables_statements}
+            
+            {create_table_a_b_statement}
+                        
+            {create_table_when_b_even_statement}
+                        
+            {create_mv_statement}
+            
+            -- first insert
+            {insert_statement}
+            
+            {print_details_statements}
+            
+            {assert_first_insert_statements}
+            
+            -- second insert, it is retry
+            {insert_statement}
+            
+            {print_details_statements}
+            
+            {assert_second_insert_statements}
+            
+            {drop_tables_statements}
+            """
+
+        print(script)
+
+    parser.set_defaults(func=calle)
+
+
+def test_mv_generates_several_blocks(parser):
+    ArgsFactory(parser).add_all()
+
+    def calle(args):
+        tables = ["table_for_join_with", "table_a_b", "table_when_b_even_and_joined", "mv_b_even"]
+        drop_tables_statements = get_drop_tables_statements(tables)
+
+        details_print_for_table_for_join_with = ""
+        if args.get_logs:
+            details_print_for_table_for_join_with = """
+                SELECT 'table_for_join_with';
+                SELECT a_join, b, _part FROM table_for_join_with ORDER BY _part, a_join, b;
+                """
+
+        create_table_a_b_statement = instance_create_statement(
+            table_name="table_a_b",
+            table_columns="(a_src String, b UInt64)",
+            table_keys="(a_src, b)",
+            table_engine=args.table_engine,
+            with_deduplication=args.deduplicate_src_table,
+        )
+
+        create_table_when_b_even_and_joined_statement = instance_create_statement(
+            table_name="table_when_b_even_and_joined",
+            table_columns="(a_src String, a_join String, b UInt64)",
+            table_keys="(a_src, a_join, b)",
+            table_engine=args.table_engine,
+            with_deduplication=args.deduplicate_dst_table,
+        )
+
+        insert_statement = instance_insert_statement(
+            "table_a_b", 5, args.insert_unique_blocks, args.use_insert_token
+        )
+
+        details_print_statements = f"""
+            SELECT 'table_a_b';
+            SELECT 'count', count() FROM table_a_b;
+            
+            SELECT 'table_when_b_even_and_joined';
+            SELECT 'count', count() FROM table_when_b_even_and_joined;
+            {"" if not args.get_logs else "SELECT _part, a_src, a_join, b FROM table_when_b_even_and_joined ORDER BY _part;"}
+        """
+
+        if args.insert_unique_blocks:
+            assert_first_insert_statements = f"""
+                SELECT throwIf( count() != 5 )
+                    FROM table_a_b;
+    
+                SELECT throwIf( count() != 47 )
+                    FROM table_when_b_even_and_joined;
+                """
+            assert_second_insert_statements = f"""
+                SELECT throwIf( count() != {5 if args.deduplicate_src_table else 10} )
+                    FROM table_a_b;
+
+                SELECT throwIf( count() != {47 if args.deduplicate_dst_table else 94} )
+                    FROM table_when_b_even_and_joined;
+                """
+        else:
+            if args.use_insert_token:
+                assert_first_insert_statements = f"""
+                    SELECT throwIf( count() != {5 if args.deduplicate_src_table else 5} )
+                        FROM table_a_b;
+    
+                    SELECT throwIf( count() != {45 if args.deduplicate_dst_table else 45} )
+                        FROM table_when_b_even_and_joined;
+                    """
+                assert_second_insert_statements = f"""
+                    SELECT throwIf( count() != {5 if args.deduplicate_src_table else 10} )
+                        FROM table_a_b;
+
+                    SELECT throwIf( count() != {45 if args.deduplicate_dst_table else 90} )
+                        FROM table_when_b_even_and_joined;
+                    """
+            else:
+                assert_first_insert_statements = f"""
+                    SELECT throwIf( count() != {1 if args.deduplicate_src_table else 5} )
+                       FROM table_a_b;
+                    
+                    SELECT throwIf( count() != {9 if args.deduplicate_dst_table else 45} )
+                       FROM table_when_b_even_and_joined;
+                    """
+                assert_second_insert_statements = f"""
+                    SELECT throwIf( count() != {1 if args.deduplicate_src_table else 10} )
+                        FROM table_a_b;
+    
+                    SELECT throwIf( count() != {9 if args.deduplicate_dst_table else 90} )
+                        FROM table_when_b_even_and_joined;
+                    """
+
+        script = f"""
+            {get_logs_statement(args)}
+
+            SET max_insert_threads={1 if args.single_thread else 10};
+            SET update_insert_deduplication_token_in_dependent_materialized_views=1;
+            SET deduplicate_blocks_in_dependent_materialized_views=1;
+            
+            SET max_block_size=1;
+            SET min_insert_block_size_rows=0;
+            SET min_insert_block_size_bytes=0;
+            
+            {drop_tables_statements}
+            
+            CREATE TABLE table_for_join_with
+                (a_join String, b UInt64)
+                ENGINE = MergeTree()
+                ORDER BY (a_join, b);
+            INSERT INTO table_for_join_with
+                SELECT 'joined_' || toString(number), number
+                FROM numbers(9);
+            {details_print_for_table_for_join_with}
+            
+            {create_table_a_b_statement}
+            SYSTEM STOP MERGES table_a_b;
+            
+            {create_table_when_b_even_and_joined_statement}
+            SYSTEM STOP MERGES table_when_b_even_and_joined;
+            
+            CREATE MATERIALIZED VIEW mv_b_even
+                TO table_when_b_even_and_joined
+                AS
+                    SELECT a_src, a_join, table_for_join_with.b as b
+                        FROM table_a_b
+                        FULL OUTER JOIN table_for_join_with
+                        ON table_a_b.b = table_for_join_with.b AND table_a_b.b % 2 = 0
+                        ORDER BY a_src, a_join, b;
+
+            -- first insert
+            {insert_statement}
+            
+            {details_print_statements}
+            
+            -- first assertion
+            {assert_first_insert_statements}
+            
+            -- second insert
+            {insert_statement}
+            
+            {details_print_statements}
+            
+            -- second assertion
+            {assert_second_insert_statements}
+            
+            {drop_tables_statements}
+        """
+
+        print(script)
+
+    parser.set_defaults(func=calle)
+
+
+def test_several_mv_into_one_table(parser):
+    ArgsFactory(parser).add_all()
+
+    def calle(args):
+        tables = ["table_src", "table_dst", "mv_b_even", "mv_b_even_even"]
+        drop_tables_statements = get_drop_tables_statements(tables)
+
+        create_table_src_statement = instance_create_statement(
+            table_name="table_src",
+            table_columns="(a String, b UInt64)",
+            table_keys="(a, b)",
+            table_engine=args.table_engine,
+            with_deduplication=args.deduplicate_src_table,
+        )
+
+        create_table_dst_statement = instance_create_statement(
+            table_name="table_dst",
+            table_columns="(a String, b UInt64)",
+            table_keys="(a, b)",
+            table_engine=args.table_engine,
+            with_deduplication=args.deduplicate_dst_table,
+        )
+
+        insert_statement = instance_insert_statement(
+            "table_src", 8, args.insert_unique_blocks, args.use_insert_token
+        )
+
+        details_print_statements = f"""
+            SELECT 'table_src count', count() FROM table_src;
+            
+            SELECT 'table_dst count', count() FROM table_dst;
+            {"" if not args.get_logs else "SELECT _part, count() FROM table_dst GROUP BY _part ORDER BY _part;"}
+        """
+
+        if args.insert_unique_blocks:
+            assert_first_insert_statements = f"""
+                SELECT throwIf( count() != 8 )
+                    FROM table_src;
+
+                SELECT throwIf( count() != 6 )
+                    FROM table_dst;
+                """
+            assert_second_insert_statements = f"""
+                SELECT throwIf( count() != {8 if args.deduplicate_src_table else 16} )
+                    FROM table_src;
+
+                SELECT throwIf( count() != {6 if args.deduplicate_dst_table else 12} )
+                    FROM table_dst;
+                """
+        else:
+            if args.use_insert_token:
+                assert_first_insert_statements = f"""
+                    SELECT throwIf( count() != {8 if args.deduplicate_src_table else 8} )
+                        FROM table_src;
+    
+                    SELECT throwIf( count() != {16 if args.deduplicate_dst_table else 16} )
+                        FROM table_dst;
+                    """
+                assert_second_insert_statements = f"""
+                    SELECT throwIf( count() != {8 if args.deduplicate_src_table else 16} )
+                        FROM table_src;
+
+                    SELECT throwIf( count() != {16 if args.deduplicate_dst_table else 32} )
+                        FROM table_dst;
+                    """
+            else:
+                assert_first_insert_statements = f"""
+                    SELECT throwIf( count() != {1 if args.deduplicate_src_table else 8} )
+                        FROM table_src;
+
+                    SELECT throwIf( count() != {1 if args.deduplicate_dst_table else 16} )
+                        FROM table_dst;
+                    """
+                assert_second_insert_statements = f"""
+                    SELECT throwIf( count() != {1 if args.deduplicate_src_table else 16} )
+                        FROM table_src;
+
+                    SELECT throwIf( count() != {2 if args.deduplicate_dst_table else 32} )
+                        FROM table_dst;
+                    """
+
+        script = f"""
+            {get_logs_statement(args)}
+
+            SET max_insert_threads={1 if args.single_thread else 10};
+            SET update_insert_deduplication_token_in_dependent_materialized_views=1;
+            SET deduplicate_blocks_in_dependent_materialized_views=1;
+            
+            SET max_block_size=1;
+            SET min_insert_block_size_rows=0;
+            SET min_insert_block_size_bytes=0;
+        
+            {drop_tables_statements}
+            
+            {create_table_src_statement}
+
+            {create_table_dst_statement}
+
+            CREATE MATERIALIZED VIEW mv_b_even
+                TO table_dst
+                AS
+                SELECT a, b
+                    FROM table_src
+                    WHERE b % 2 = 0;
+                    
+            CREATE MATERIALIZED VIEW mv_b_even_even
+                TO table_dst
+                AS
+                SELECT a, b
+                    FROM table_src
+                    WHERE b % 4 = 0;
+
+            -- first insert
+            {insert_statement}
+
+            {details_print_statements}
+            
+            {assert_first_insert_statements}
+
+            -- second insert, retry
+            {insert_statement}
+            
+            {details_print_statements}
+
+            {assert_second_insert_statements}
+            
+            {drop_tables_statements}
+            """
+
+        print(script)
+
+    parser.set_defaults(func=calle)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(dest="test")
+    test_insert_several_blocks(
+        subparsers.add_parser("insert_several_blocks_into_table")
+    )
+    test_mv_generates_several_blocks(
+        subparsers.add_parser("mv_generates_several_blocks")
+    )
+    test_several_mv_into_one_table(
+        subparsers.add_parser("several_mv_into_one_table")
+    )
+    args = parser.parse_args()
+    if args.test is None:
+        parser.print_help()
+    return args
+
+
+def main():
+    args = parse_args()
+    if args.test is not None:
+        args.func(args)
+
+
+if __name__ == "__main__":
+    main()

tests/queries/0_stateless/03008_deduplication_insert_several_blocks.reference

@@ -0,0 +1,870 @@
+
+Test case 0: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 1: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 2: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 3: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 4: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 5: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 6: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 7: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 8: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	1
+table_when_b_even
+count	1
+EXPECTED_TO_FAIL
+
+Test case 9: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	1
+EXPECTED_TO_FAIL
+
+Test case 10: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	1
+table_when_b_even
+count	5
+EXPECTED_TO_FAIL
+
+Test case 11: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	10
+EXPECTED_TO_FAIL
+
+Test case 12: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+EXPECTED_TO_FAIL
+
+Test case 13: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+EXPECTED_TO_FAIL
+
+Test case 14: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 15: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 16: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 17: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 18: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 19: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 20: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 21: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 22: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 23: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 24: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 25: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 26: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 27: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 28: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 29: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 30: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 31: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 32: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 33: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 34: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 35: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 36: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 37: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 38: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 39: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 40: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	1
+table_when_b_even
+count	1
+EXPECTED_TO_FAIL
+
+Test case 41: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	1
+EXPECTED_TO_FAIL
+
+Test case 42: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	1
+table_when_b_even
+count	5
+EXPECTED_TO_FAIL
+
+Test case 43: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	10
+EXPECTED_TO_FAIL
+
+Test case 44: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+EXPECTED_TO_FAIL
+
+Test case 45: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+EXPECTED_TO_FAIL
+
+Test case 46: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 47: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 48: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 49: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 50: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 51: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 52: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 53: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 54: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 55: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 56: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 57: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 58: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 59: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 60: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 61: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 62: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 63: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+All cases executed

tests/queries/0_stateless/03008_deduplication_insert_several_blocks.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+# Test case 8: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+# Test case 9: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+# Test case 10: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+# Test case 11: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+# fails, it is a error. Several blocks in scr table with the same user token are processed in parallel and deduplicated
+
+# Test case 12: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True"
+# Test case 13: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False"
+# fails, it is a error. The same situation as first one, but on dst table.
+
+RUN_ONLY=""
+#RUN_ONLY=""
+
+KNOWN_ERRORS=(8 9 10 11 12 13)
+
+function is_known_error()
+{
+    n=$1
+    for e in "${KNOWN_ERRORS[@]}"; do
+      if [ "$n" -eq "$e" ] || [ "$n" -eq "$((e+32))" ]; then
+        return 0
+      fi
+    done
+    return 1
+}
+
+i=0
+for engine in "MergeTree" "ReplicatedMergeTree"; do
+    for use_insert_token in "True" "False"; do
+        for single_thread in "True" "False"; do
+            for deduplicate_src_table in "True" "False"; do
+                for deduplicate_dst_table in "True" "False"; do
+                    for insert_unique_blocks in "True" "False"; do
+
+                        THIS_RUN="Test case $i:"
+                        THIS_RUN+=" engine=$engine"
+                        THIS_RUN+=" use_insert_token=$use_insert_token"
+                        THIS_RUN+=" single_thread=$single_thread"
+                        THIS_RUN+=" deduplicate_src_table=$deduplicate_src_table"
+                        THIS_RUN+=" deduplicate_dst_table=$deduplicate_dst_table"
+                        THIS_RUN+=" insert_unique_blocks=$insert_unique_blocks"
+
+                        is_error=$(is_known_error "$i" && echo Y || echo N)
+                        i=$((i+1))
+
+                        echo
+                        if [ -n "$RUN_ONLY" ] && [ "$RUN_ONLY" != "$THIS_RUN" ]; then
+                          echo "skip $THIS_RUN"
+                          continue
+                        fi
+                        echo "$THIS_RUN"
+
+                        if [ "$is_error" = Y ]; then
+                            $CLICKHOUSE_CLIENT -nmq "
+                              $(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \
+                                  --table-engine $engine \
+                                  --use-insert-token $use_insert_token \
+                                  --single-thread $single_thread \
+                                  --deduplicate-src-table $deduplicate_src_table \
+                                  --deduplicate-dst-table $deduplicate_dst_table \
+                                  --insert-unique-blocks $insert_unique_blocks \
+                                  --get-logs false \
+                              )
+                            " 2>/dev/null && echo FIXED || echo EXPECTED_TO_FAIL
+                        else
+                            $CLICKHOUSE_CLIENT -nmq "
+                              $(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \
+                                  --table-engine $engine \
+                                  --use-insert-token $use_insert_token \
+                                  --single-thread $single_thread \
+                                  --deduplicate-src-table $deduplicate_src_table \
+                                  --deduplicate-dst-table $deduplicate_dst_table \
+                                  --insert-unique-blocks $insert_unique_blocks \
+                                  --get-logs false \
+                              )
+                            " && echo OK || echo FAIL
+                        fi
+                    done
+                done
+            done
+        done
+    done
+done
+
+echo
+echo "All cases executed"

tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks.reference

@@ -0,0 +1,814 @@
+
+Test case 0: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+OK
+
+Test case 1: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+OK
+
+Test case 2: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	94
+0
+0
+OK
+
+Test case 3: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	90
+0
+0
+OK
+
+Test case 4: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	47
+0
+0
+OK
+
+Test case 5: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	45
+0
+0
+OK
+
+Test case 6: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	94
+0
+0
+OK
+
+Test case 7: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	90
+0
+0
+OK
+
+Test case 8: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	10
+EXPECTED_TO_FAIL
+
+Test case 9: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	9
+EXPECTED_TO_FAIL
+
+Test case 10: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	47
+EXPECTED_TO_FAIL
+
+Test case 11: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	45
+EXPECTED_TO_FAIL
+
+Test case 12: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+EXPECTED_TO_FAIL
+
+Test case 13: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+EXPECTED_TO_FAIL
+
+Test case 14: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	94
+0
+0
+OK
+
+Test case 15: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	90
+0
+0
+OK
+
+Test case 16: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	14
+0
+EXPECTED_TO_FAIL
+
+Test case 17: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 18: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	94
+0
+0
+OK
+
+Test case 19: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	90
+0
+0
+OK
+
+Test case 20: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	14
+0
+EXPECTED_TO_FAIL
+
+Test case 21: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 22: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	94
+0
+0
+OK
+
+Test case 23: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	90
+0
+0
+OK
+
+Test case 24: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	14
+0
+EXPECTED_TO_FAIL
+
+Test case 25: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 26: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	94
+0
+0
+OK
+
+Test case 27: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	90
+0
+0
+OK
+
+Test case 28: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	14
+0
+EXPECTED_TO_FAIL
+
+Test case 29: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 30: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	94
+0
+0
+OK
+
+Test case 31: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	90
+0
+0
+OK
+
+Test case 32: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+OK
+
+Test case 33: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+OK
+
+Test case 34: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	94
+0
+0
+OK
+
+Test case 35: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	90
+0
+0
+OK
+
+Test case 36: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	47
+0
+0
+OK
+
+Test case 37: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	45
+0
+0
+OK
+
+Test case 38: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	94
+0
+0
+OK
+
+Test case 39: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	90
+0
+0
+OK
+
+Test case 40: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	10
+EXPECTED_TO_FAIL
+
+Test case 41: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	9
+EXPECTED_TO_FAIL
+
+Test case 42: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	47
+EXPECTED_TO_FAIL
+
+Test case 43: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	45
+EXPECTED_TO_FAIL
+
+Test case 44: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+EXPECTED_TO_FAIL
+
+Test case 45: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+EXPECTED_TO_FAIL
+
+Test case 46: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	94
+0
+0
+OK
+
+Test case 47: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	90
+0
+0
+OK
+
+Test case 48: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	14
+0
+EXPECTED_TO_FAIL
+
+Test case 49: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 50: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	94
+0
+0
+OK
+
+Test case 51: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	90
+0
+0
+OK
+
+Test case 52: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	14
+0
+EXPECTED_TO_FAIL
+
+Test case 53: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 54: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	94
+0
+0
+OK
+
+Test case 55: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	90
+0
+0
+OK
+
+Test case 56: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	14
+0
+EXPECTED_TO_FAIL
+
+Test case 57: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 58: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	94
+0
+0
+OK
+
+Test case 59: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	90
+0
+0
+OK
+
+Test case 60: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	14
+0
+EXPECTED_TO_FAIL
+
+Test case 61: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 62: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	47
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	94
+0
+0
+OK
+
+Test case 63: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	45
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	90
+0
+0
+OK
+
+All cases executed

tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+# Test case 8: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+# Test case 9: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+# Test case 10: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+# Test case 11: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+# failed due to race in multi thread insertion, blocks are deduplicated in different threads
+
+# Test case 12: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+# Test case 13: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+# the same as first but for dst table
+
+# Test case 16: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+# Test case 20: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+# Test case 24: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+# Test case 28: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+# dst table deduplicates all incoming blocks from one insert because not uniq hash
+
+RUN_ONLY=""
+#RUN_ONLY="Test case 20: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True"
+
+KNOWN_ERRORS=(8 9 10 11 12 13 16 20 24 28)
+
+function is_known_error()
+{
+    n=$1
+    for e in "${KNOWN_ERRORS[@]}"; do
+      if [ "$n" -eq "$e" ] || [ "$n" -eq "$((e+32))" ]; then
+        return 0
+      fi
+    done
+    return 1
+}
+
+i=0
+for engine in "MergeTree" "ReplicatedMergeTree"; do
+    for use_insert_token in "True" "False"; do
+        for single_thread in "True" "False"; do
+            for deduplicate_src_table in "True" "False"; do
+                for deduplicate_dst_table in "True" "False"; do
+                    for insert_unique_blocks in "True" "False"; do
+
+                        THIS_RUN="Test case $i:"
+                        THIS_RUN+=" engine=$engine"
+                        THIS_RUN+=" use_insert_token=$use_insert_token"
+                        THIS_RUN+=" single_thread=$single_thread"
+                        THIS_RUN+=" deduplicate_src_table=$deduplicate_src_table"
+                        THIS_RUN+=" deduplicate_dst_table=$deduplicate_dst_table"
+                        THIS_RUN+=" insert_unique_blocks=$insert_unique_blocks"
+
+                        is_error=$(is_known_error "$i" && echo Y || echo N)
+                        i=$((i+1))
+
+                        echo
+                        if [ -n "$RUN_ONLY" ] && [ "$RUN_ONLY" != "$THIS_RUN" ]; then
+                          echo "skip $THIS_RUN"
+                          continue
+                        fi
+                        echo "$THIS_RUN"
+
+                        if [ "$is_error" = Y ]; then
+                            $CLICKHOUSE_CLIENT -nmq "
+                              $(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \
+                                  --table-engine $engine \
+                                  --use-insert-token $use_insert_token \
+                                  --single-thread $single_thread \
+                                  --deduplicate-src-table $deduplicate_src_table \
+                                  --deduplicate-dst-table $deduplicate_dst_table \
+                                  --insert-unique-blocks $insert_unique_blocks \
+                                  --get-logs false \
+                              )
+                            " 2>/dev/null && echo FIXED || echo EXPECTED_TO_FAIL
+                        else
+                            $CLICKHOUSE_CLIENT -nmq "
+                              $(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \
+                                  --table-engine $engine \
+                                  --use-insert-token $use_insert_token \
+                                  --single-thread $single_thread \
+                                  --deduplicate-src-table $deduplicate_src_table \
+                                  --deduplicate-dst-table $deduplicate_dst_table \
+                                  --insert-unique-blocks $insert_unique_blocks \
+                                  --get-logs false \
+                              )
+                            " && echo OK || echo FAIL
+                        fi
+                    done
+                done
+            done
+        done
+    done
+done
+
+echo
+echo "All cases executed"

tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table.reference

@@ -0,0 +1,590 @@
+
+Test case 0: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 1: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	16
+0
+0
+OK
+
+Test case 2: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 3: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	32
+0
+0
+OK
+
+Test case 4: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 5: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	16
+0
+0
+OK
+
+Test case 6: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 7: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 8: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	1
+table_dst count	2
+EXPECTED_TO_FAIL
+
+Test case 9: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	1
+table_dst count	2
+EXPECTED_TO_FAIL
+
+Test case 10: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	1
+table_dst count	6
+EXPECTED_TO_FAIL
+
+Test case 11: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	1
+table_dst count	16
+EXPECTED_TO_FAIL
+
+Test case 12: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	2
+0
+EXPECTED_TO_FAIL
+
+Test case 13: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	2
+0
+EXPECTED_TO_FAIL
+
+Test case 14: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 15: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 16: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	4
+0
+EXPECTED_TO_FAIL
+
+Test case 17: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	1
+table_dst count	1
+0
+0
+table_src count	1
+table_dst count	1
+0
+EXPECTED_TO_FAIL
+
+Test case 18: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 19: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	1
+table_dst count	16
+0
+0
+table_src count	1
+table_dst count	32
+0
+0
+OK
+
+Test case 20: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	4
+0
+EXPECTED_TO_FAIL
+
+Test case 21: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	1
+0
+0
+table_src count	16
+table_dst count	1
+0
+EXPECTED_TO_FAIL
+
+Test case 22: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 23: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 24: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	4
+0
+EXPECTED_TO_FAIL
+
+Test case 25: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	1
+table_dst count	1
+0
+0
+table_src count	1
+table_dst count	1
+0
+EXPECTED_TO_FAIL
+
+Test case 26: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 27: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	1
+table_dst count	16
+0
+0
+table_src count	1
+table_dst count	32
+0
+0
+OK
+
+Test case 28: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	4
+0
+EXPECTED_TO_FAIL
+
+Test case 29: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	1
+0
+0
+table_src count	16
+table_dst count	1
+0
+EXPECTED_TO_FAIL
+
+Test case 30: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 31: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 32: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 33: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	16
+0
+0
+OK
+
+Test case 34: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 35: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	32
+0
+0
+OK
+
+Test case 36: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 37: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	16
+0
+0
+OK
+
+Test case 38: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 39: engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 40: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	1
+table_dst count	2
+EXPECTED_TO_FAIL
+
+Test case 41: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	1
+table_dst count	2
+EXPECTED_TO_FAIL
+
+Test case 42: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	1
+table_dst count	6
+EXPECTED_TO_FAIL
+
+Test case 43: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	1
+table_dst count	16
+EXPECTED_TO_FAIL
+
+Test case 44: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	2
+0
+EXPECTED_TO_FAIL
+
+Test case 45: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	2
+0
+EXPECTED_TO_FAIL
+
+Test case 46: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 47: engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 48: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	4
+0
+EXPECTED_TO_FAIL
+
+Test case 49: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	1
+table_dst count	1
+0
+0
+table_src count	1
+table_dst count	1
+0
+EXPECTED_TO_FAIL
+
+Test case 50: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 51: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	1
+table_dst count	16
+0
+0
+table_src count	1
+table_dst count	32
+0
+0
+OK
+
+Test case 52: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	4
+0
+EXPECTED_TO_FAIL
+
+Test case 53: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	1
+0
+0
+table_src count	16
+table_dst count	1
+0
+EXPECTED_TO_FAIL
+
+Test case 54: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 55: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 56: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	4
+0
+EXPECTED_TO_FAIL
+
+Test case 57: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	1
+table_dst count	1
+0
+0
+table_src count	1
+table_dst count	1
+0
+EXPECTED_TO_FAIL
+
+Test case 58: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 59: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	1
+table_dst count	16
+0
+0
+table_src count	1
+table_dst count	32
+0
+0
+OK
+
+Test case 60: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	4
+0
+EXPECTED_TO_FAIL
+
+Test case 61: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	1
+0
+0
+table_src count	16
+table_dst count	1
+0
+EXPECTED_TO_FAIL
+
+Test case 62: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 63: engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+All cases executed

tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+# Test case 8: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+# Test case 9: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+# Test case 10: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+# Test case 11: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+# race condition on insert into src table
+
+# Test case 12: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+# Test case 13: engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+# race condition on insert into dst table
+
+# Test case 16: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+# Test case 20: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+# Test case 24: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+# Test case 28: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+# dst deduplicates blocks from one inserts from different materialized view
+
+# Test case 17: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+# Test case 21: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+# Test case 25: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+# Test case 29: engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+# dst  deduplicates blocks from different inserts by hash
+
+KNOWN_ERRORS=(8 9 10 11 12 13 16 20 24 28 17 21 25 29)
+
+function is_known_error()
+{
+    n=$1
+    for e in "${KNOWN_ERRORS[@]}"; do
+      if [ "$n" -eq "$e" ] || [ "$n" -eq "$((e+32))" ]; then
+        return 0
+      fi
+    done
+    return 1
+}
+
+RUN_ONLY=""
+#RUN_ONLY="Test case 0: engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True"
+
+i=0
+for engine in "MergeTree" "ReplicatedMergeTree"; do
+    for use_insert_token in "True" "False"; do
+        for single_thread in "True" "False"; do
+            for deduplicate_src_table in "True" "False"; do
+                for deduplicate_dst_table in "True" "False"; do
+                    for insert_unique_blocks in "True" "False"; do
+
+                        THIS_RUN="Test case $i:"
+                        THIS_RUN+=" engine=$engine"
+                        THIS_RUN+=" use_insert_token=$use_insert_token"
+                        THIS_RUN+=" single_thread=$single_thread"
+                        THIS_RUN+=" deduplicate_src_table=$deduplicate_src_table"
+                        THIS_RUN+=" deduplicate_dst_table=$deduplicate_dst_table"
+                        THIS_RUN+=" insert_unique_blocks=$insert_unique_blocks"
+
+                        is_error=$(is_known_error "$i" && echo Y || echo N)
+                        i=$((i+1))
+
+                        echo
+                        if [ -n "$RUN_ONLY" ] && [ "$RUN_ONLY" != "$THIS_RUN" ]; then
+                          echo "skip $THIS_RUN"
+                          continue
+                        fi
+                        echo "$THIS_RUN"
+
+                        if [ "$is_error" = Y ]; then
+                            $CLICKHOUSE_CLIENT -nmq "
+                              $(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \
+                                  --table-engine $engine \
+                                  --use-insert-token $use_insert_token \
+                                  --single-thread $single_thread \
+                                  --deduplicate-src-table $deduplicate_src_table \
+                                  --deduplicate-dst-table $deduplicate_dst_table \
+                                  --insert-unique-blocks $insert_unique_blocks \
+                                  --get-logs false \
+                              )
+                            " 2>/dev/null && echo FIXED || echo EXPECTED_TO_FAIL
+                        else
+                            $CLICKHOUSE_CLIENT -nmq "
+                              $(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \
+                                  --table-engine $engine \
+                                  --use-insert-token $use_insert_token \
+                                  --single-thread $single_thread \
+                                  --deduplicate-src-table $deduplicate_src_table \
+                                  --deduplicate-dst-table $deduplicate_dst_table \
+                                  --insert-unique-blocks $insert_unique_blocks \
+                                  --get-logs false \
+                              )
+                            " && echo OK || echo FAIL
+                        fi
+                    done
+                done
+            done
+        done
+    done
+done
+
+echo
+echo "All cases executed"
+
+

tests/queries/0_stateless/03008_non_replicated_deduplication_mv.sql

@@ -1,93 +0,0 @@
-DROP TABLE IF EXISTS table_a_b;
-DROP TABLE IF EXISTS table_when_b_even;
-DROP TABLE IF EXISTS mv_b_even;
-
-
-SET max_insert_threads=1;
-SET update_insert_deduplication_token_in_dependent_materialized_views=1;
-SET deduplicate_blocks_in_dependent_materialized_views=1;
-
-SET max_block_size=3;
-SET min_insert_block_size_rows=0;
-SET min_insert_block_size_bytes=0;
-
-
-CREATE TABLE table_a_b
-    (
-        a String,
-        b UInt64,
-    )
-    ENGINE = MergeTree()
-    ORDER BY (a, b)
-    SETTINGS non_replicated_deduplication_window=10000;
-SYSTEM STOP MERGES table_a_b;
-
-CREATE TABLE table_when_b_even_wo_dedup
-    (
-        a String,
-        b UInt64,
-    )
-    ENGINE = MergeTree()
-    ORDER BY (a, b)
-    SETTINGS non_replicated_deduplication_window=0;
-SYSTEM STOP MERGES table_when_b_even;
-
-CREATE MATERIALIZED VIEW mv_b_even_wo_dedup
-TO table_when_b_even_wo_dedup
-AS
-    SELECT a, b
-    FROM table_a_b
-    WHERE b % 2 = 0;
-
-CREATE TABLE table_when_b_even_dedup
-    (
-        a String,
-        b UInt64,
-    )
-    ENGINE = MergeTree()
-    ORDER BY (a, b)
-    SETTINGS non_replicated_deduplication_window=10000;
-SYSTEM STOP MERGES table_when_b_even;
-
-CREATE MATERIALIZED VIEW mv_b_even_dedup
-TO table_when_b_even_dedup
-AS
-    SELECT a, b
-    FROM table_a_b
-    WHERE b % 2 = 0;
-
-
-SELECT 'first insert'
-SETTINGS send_logs_level='trace';
-
-INSERT INTO table_a_b
-SELECT toString(number DIV  2), number
-FROM numbers(5)
-SETTINGS send_logs_level='trace';
-
-
-SELECT 'second insert'
-SETTINGS send_logs_level='trace';
-
-INSERT INTO table_a_b
-SELECT toString(number DIV  2), number
-FROM numbers(5)
-SETTINGS send_logs_level='trace';
-
-
-SELECT 'table_a_b';
-SELECT 'count', count() FROM table_a_b;
-SELECT _part, count() FROM table_a_b GROUP BY _part;
-
-SELECT 'table_when_b_even_wo_dedup';
-SELECT 'count', count() FROM table_when_b_even_wo_dedup;
-SELECT _part, count() FROM table_when_b_even_wo_dedup GROUP BY _part;
-
-SELECT 'table_when_b_even_dedup';
-SELECT 'count', count() FROM table_when_b_even_dedup;
-SELECT _part, count() FROM table_when_b_even_dedup GROUP BY _part;
-
-
-DROP TABLE mv_b_even;
-DROP TABLE table_when_b_even;
-DROP TABLE table_a_b;

tests/queries/0_stateless/03008_non_replicated_deduplication_mv_collision_in_dst.sql

@@ -1,113 +0,0 @@
-DROP TABLE IF EXISTS test;
-
-CREATE TABLE table_for_join_with
-    (
-        a_join String,
-        b UInt64
-    )
-    ENGINE = MergeTree()
-    ORDER BY (a_join, b);
-
-INSERT INTO table_for_join_with
-    SELECT 'joined_' || toString(number), number
-    FROM numbers(10);
-SELECT 'table_for_join_with';
-SELECT a_join, b, _part FROM table_for_join_with ORDER BY _part, a_join, b;
-
-
-CREATE TABLE table_a_b
-    (
-        a_src String,
-        b UInt64
-    )
-    ENGINE = MergeTree()
-    ORDER BY (a_src, b)
-    SETTINGS non_replicated_deduplication_window=10000;
-SYSTEM STOP MERGES table_a_b;
-
-CREATE TABLE table_when_b_even_dedup
-    (
-        a_src String CODEC(NONE),
-        a_join String CODEC(NONE),
-        b UInt64 CODEC(NONE)
-    )
-    ENGINE = MergeTree()
-    ORDER BY (a_src, a_join, b)
-    SETTINGS non_replicated_deduplication_window=10000;
-SYSTEM STOP MERGES table_when_b_even_dedup;
-
-CREATE MATERIALIZED VIEW mv_b_even_dedup
-    TO table_when_b_even_dedup
-    AS
-        SELECT a_src, a_join, b
-            FROM table_a_b
-            FULL OUTER JOIN table_for_join_with
-            ON table_a_b.b = table_for_join_with.b AND table_a_b.b % 2 = 0
-            ORDER BY a_src, a_join, b;
-
-CREATE TABLE table_when_b_even_wo_dedup
-    (
-        a_src String CODEC(NONE),
-        a_join String CODEC(NONE),
-        b UInt64 CODEC(NONE)
-    )
-    ENGINE = MergeTree()
-    ORDER BY (a_src, a_join, b)
-    SETTINGS non_replicated_deduplication_window=0;
-SYSTEM STOP MERGES table_when_b_even_wo_dedup;
-
-CREATE MATERIALIZED VIEW mv_b_even_wo_dedup
-    TO table_when_b_even_wo_dedup
-    AS
-        SELECT a_src, a_join, b
-            FROM table_a_b
-            FULL OUTER JOIN table_for_join_with
-            ON table_a_b.b = table_for_join_with.b AND table_a_b.b % 2 = 0
-            ORDER BY a_src, a_join, b;
-
-
-SET max_insert_threads=1;
-SET update_insert_deduplication_token_in_dependent_materialized_views=1;
-SET deduplicate_blocks_in_dependent_materialized_views=1;
-
-SET max_block_size=1;
-SET min_insert_block_size_rows=0;
-SET min_insert_block_size_bytes=0;
-
-
-SELECT 'first insert'
-SETTINGS send_logs_level='trace';
-
-INSERT INTO table_a_b
-SELECT 'source_' || toString(number), number
-FROM numbers(5)
-SETTINGS send_logs_level='trace';
-
-
-SELECT 'second insert'
-SETTINGS send_logs_level='trace';
-
-INSERT INTO table_a_b
-SELECT 'source_' || toString(number), number
-FROM numbers(5)
-SETTINGS send_logs_level='trace';
-
-
-SELECT 'table_a_b';
-SELECT 'count', count() FROM table_a_b;
-SELECT _part, count() FROM table_a_b GROUP BY _part;
-
-SELECT 'table_when_b_even_dedup, here the result if join is deduplicated inside one request, it is not correct';
-SELECT 'count', count() FROM table_when_b_even_dedup;
-SELECT _part, count() FROM table_when_b_even_dedup GROUP BY _part;
-
-SELECT 'table_when_b_even_wo_dedup';
-SELECT 'count', count() FROM table_when_b_even_wo_dedup;
-SELECT _part, count() FROM table_when_b_even_wo_dedup GROUP BY _part ORDER BY _part;
-
-
-DROP TABLE mv_b_even_dedup;
-DROP TABLE table_when_b_even_dedup;
-DROP TABLE mv_b_even_wo_dedup;
-DROP TABLE table_when_b_even_wo_dedup;
-DROP TABLE table_a_b;

tests/queries/0_stateless/03008_non_replicated_deduplication_mv_collision_in_dst_from_different_src.sql

@@ -1,119 +0,0 @@
-DROP TABLE IF EXISTS test;
-
-CREATE TABLE table_source
-    (
-        a String,
-        b UInt64
-    )
-    ENGINE = MergeTree()
-    ORDER BY (a, b)
-    SETTINGS non_replicated_deduplication_window=10000;
-SYSTEM STOP MERGES table_source;
-
-CREATE TABLE table_dst_dedup
-    (
-        a String,
-        b UInt64
-    )
-    ENGINE = MergeTree()
-    ORDER BY (a, b)
-    SETTINGS non_replicated_deduplication_window=10000;
-SYSTEM STOP MERGES table_dst_dedup;
-
-CREATE MATERIALIZED VIEW mv_b_even_dedup
-    TO table_dst_dedup
-    AS
-    SELECT a, b
-        FROM table_source
-        WHERE b % 2 = 0;
-
-CREATE MATERIALIZED VIEW mv_b_even_even_dedup
-    TO table_dst_dedup
-    AS
-    SELECT a, b
-        FROM table_source
-        WHERE b % 4 = 0;
-
-CREATE TABLE table_dst_wo_dedup
-    (
-        a String,
-        b UInt64
-    )
-    ENGINE = MergeTree()
-    ORDER BY (a, b)
-    SETTINGS non_replicated_deduplication_window=0;
-SYSTEM STOP MERGES table_dst_wo_dedup;
-
-CREATE MATERIALIZED VIEW mv_b_even_wo_dedup
-    TO table_dst_wo_dedup
-    AS
-    SELECT a, b
-        FROM table_source
-        WHERE b % 2 = 0;
-
-CREATE MATERIALIZED VIEW mv_b_even_wo_even_dedup
-    TO table_dst_wo_dedup
-    AS
-    SELECT a, b
-        FROM table_source
-        WHERE b % 4 = 0;
-
-
-SET max_insert_threads=1;
-SET update_insert_deduplication_token_in_dependent_materialized_views=1;
-SET deduplicate_blocks_in_dependent_materialized_views=1;
-
-SET max_block_size=1;
-SET min_insert_block_size_rows=0;
-SET min_insert_block_size_bytes=0;
-
-
-SELECT 'first insert'
-SETTINGS send_logs_level='trace';
-
-INSERT INTO table_source
-SELECT 'source_' || toString(number), number
-FROM numbers(8)
-SETTINGS send_logs_level='trace';
-
-SELECT 'table_source';
-SELECT 'count', count() FROM table_source;
-SELECT _part, count() FROM table_source GROUP BY _part ORDER BY _part;
-
-SELECT 'table_dst_dedup';
-SELECT 'count', count() FROM table_dst_dedup;
-SELECT _part, count() FROM table_dst_dedup GROUP BY _part ORDER BY _part;
-
-SELECT 'table_dst_wo_dedup';
-SELECT 'count', count() FROM table_dst_wo_dedup;
-SELECT _part, count() FROM table_dst_wo_dedup GROUP BY _part ORDER BY _part;
-
-
-SELECT 'second insert'
-SETTINGS send_logs_level='trace';
-
-INSERT INTO table_source
-SELECT 'source_' || toString(number), number
-FROM numbers(8)
-SETTINGS send_logs_level='trace';
-
-SELECT 'table_source';
-SELECT 'count', count() FROM table_source;
-SELECT _part, count() FROM table_source GROUP BY _part ORDER BY _part;
-
-SELECT 'table_dst_dedup, block from different mv is deduplicated, it is wrong';
-SELECT 'count', count() FROM table_dst_dedup;
-SELECT _part, count() FROM table_dst_dedup GROUP BY _part ORDER BY _part;
-
-SELECT 'table_dst_wo_dedup';
-SELECT 'count', count() FROM table_dst_wo_dedup;
-SELECT _part, count() FROM table_dst_wo_dedup GROUP BY _part ORDER BY _part;
-
-
-DROP TABLE mv_b_even_dedup;
-DROP TABLE mv_b_even_even_dedup;
-DROP TABLE mv_b_even_wo_dedup;
-DROP TABLE mv_b_even_even_wo_dedup;
-DROP TABLE table_dst_dedup;
-DROP TABLE table_dst_wo_dedup;
-DROP TABLE table_source;

tests/queries/0_stateless/03008_non_replicated_deduplication_mv_collision_in_src.sql

@@ -1,76 +0,0 @@
-DROP TABLE IF EXISTS test;
-
-CREATE TABLE table_a_b
-    (
-        a String,
-        b UInt64
-    )
-    ENGINE = MergeTree()
-    ORDER BY (a, b)
-    SETTINGS non_replicated_deduplication_window=10000;
-SYSTEM STOP MERGES table_a_b;
-
-CREATE TABLE table_when_b_even
-    (
-        a String CODEC(NONE),
-        b UInt64 CODEC(NONE)
-    )
-    ENGINE = MergeTree()
-    ORDER BY (a, b)
-    SETTINGS non_replicated_deduplication_window=10000;
-SYSTEM STOP MERGES table_when_b_even;
-
-CREATE MATERIALIZED VIEW mv_b_even
-    TO table_when_b_even
-    AS
-    SELECT a, b
-        FROM table_a_b
-        WHERE b % 2 = 0;
-
-
-SET max_insert_threads=1;
-SET update_insert_deduplication_token_in_dependent_materialized_views=1;
-SET deduplicate_blocks_in_dependent_materialized_views=1;
-
-SET max_block_size=1;
-SET min_insert_block_size_rows=0;
-SET min_insert_block_size_bytes=0;
-
-
-SELECT 'first insert'
-SETTINGS send_logs_level='trace';
-
-INSERT INTO table_a_b
-SELECT 'source_' || toString(1), 1
-FROM numbers(5)
-SETTINGS send_logs_level='trace';
-
-SELECT 'table_a_b, it deduplicates rows within one insert, it is wrong';
-SELECT 'count', count() FROM table_a_b;
-SELECT _part, count() FROM table_a_b GROUP BY _part ORDER BY _part;
-
-SELECT 'table_when_b_even';
-SELECT 'count', count() FROM table_when_b_even;
-SELECT _part, count() FROM table_when_b_even GROUP BY _part ORDER BY _part;
-
-
-SELECT 'second insert'
-SETTINGS send_logs_level='trace';
-
-INSERT INTO table_a_b
-SELECT 'source_' || toString(1), 1
-FROM numbers(5)
-SETTINGS send_logs_level='trace';
-
-SELECT 'table_a_b';
-SELECT 'count', count() FROM table_a_b;
-SELECT _part, count() FROM table_a_b GROUP BY _part;
-
-SELECT 'table_when_b_even';
-SELECT 'count', count() FROM table_when_b_even;
-SELECT _part, count() FROM table_when_b_even GROUP BY _part;
-
-
-DROP TABLE mv_b_even;
-DROP TABLE table_when_b_even;
-DROP TABLE table_a_b;