Revert "wip fixing test_dictionaries_all_layouts_separate_sources"

This reverts commit 2750507793.

tests/integration/helpers/external_sources.py

@@ -237,12 +237,7 @@ class SourceMongo(ExternalSource):
                 self.converters[field.name] = lambda x: x
 
         self.db = self.connection["test"]
-        user_info = self.db.command("usersInfo", self.user)
-        if user_info['users']:
-            self.db.command("updateUser", self.user, pwd=self.password)
-        else:
-            self.db.command("createUser", self.user, pwd=self.password, roles=["readWrite"])
-
+        self.db.add_user(self.user, self.password)
         self.prepared = True
 
     def load_data(self, data, table_name):

tests/integration/test_dictionaries_all_layouts_separate_sources/common.py

@@ -186,18 +186,16 @@ class BaseLayoutTester:
         self.test_name = test_name
         self.layouts = []
 
-    @staticmethod
-    def get_dict_directory(test_name):
-        return os.path.join(DICT_CONFIG_PATH, test_name)
+    def get_dict_directory(self):
+        return os.path.join(DICT_CONFIG_PATH, self.test_name)
 
     def cleanup(self):
-        shutil.rmtree(self.get_dict_directory(self.test_name), ignore_errors=True)
-        os.makedirs(self.get_dict_directory(self.test_name))
+        shutil.rmtree(self.get_dict_directory(), ignore_errors=True)
+        os.makedirs(self.get_dict_directory())
 
-    @staticmethod
-    def list_dictionaries(test_name):
+    def list_dictionaries(self):
         dictionaries = []
-        directory = BaseLayoutTester.get_dict_directory(test_name)
+        directory = self.get_dict_directory()
         for fname in os.listdir(directory):
             dictionaries.append(os.path.join(directory, fname))
         return dictionaries
@@ -217,9 +215,7 @@ class BaseLayoutTester:
     def get_dict(self, source, layout, fields, suffix_name=""):
         structure = DictionaryStructure(layout, fields)
         dict_name = source.name + "_" + layout.name + "_" + suffix_name
-        dict_path = os.path.join(
-            self.get_dict_directory(self.test_name), dict_name + ".xml"
-        )
+        dict_path = os.path.join(self.get_dict_directory(), dict_name + ".xml")
         dictionary = Dictionary(
             dict_name, structure, source, dict_path, "table_" + dict_name, fields
         )

tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py

@@ -17,7 +17,7 @@ node = None
 simple_tester = None
 complex_tester = None
 ranged_tester = None
-TEST_NAME = "local"
+test_name = "local"
 
 
 def setup_module(module):
@@ -27,14 +27,14 @@ def setup_module(module):
     global complex_tester
     global ranged_tester
 
-    simple_tester = SimpleLayoutTester(TEST_NAME)
+    simple_tester = SimpleLayoutTester(test_name)
     simple_tester.cleanup()
     simple_tester.create_dictionaries(SOURCE)
 
-    complex_tester = ComplexLayoutTester(TEST_NAME)
+    complex_tester = ComplexLayoutTester(test_name)
     complex_tester.create_dictionaries(SOURCE)
 
-    ranged_tester = RangedLayoutTester(TEST_NAME)
+    ranged_tester = RangedLayoutTester(test_name)
     ranged_tester.create_dictionaries(SOURCE)
     # Since that all .xml configs were created
 
@@ -43,7 +43,7 @@ def setup_module(module):
     main_configs = []
     main_configs.append(os.path.join("configs", "disable_ssl_verification.xml"))
 
-    dictionaries = simple_tester.list_dictionaries(TEST_NAME)
+    dictionaries = simple_tester.list_dictionaries()
 
     node = cluster.add_instance(
         "local_node", main_configs=main_configs, dictionaries=dictionaries

tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py

@@ -17,7 +17,7 @@ node = None
 simple_tester = None
 complex_tester = None
 ranged_tester = None
-TEST_NAME = "remote"
+test_name = "remote"
 
 
 def setup_module(module):
@@ -27,14 +27,14 @@ def setup_module(module):
     global complex_tester
     global ranged_tester
 
-    simple_tester = SimpleLayoutTester(TEST_NAME)
+    simple_tester = SimpleLayoutTester(test_name)
     simple_tester.cleanup()
     simple_tester.create_dictionaries(SOURCE)
 
-    complex_tester = ComplexLayoutTester(TEST_NAME)
+    complex_tester = ComplexLayoutTester(test_name)
     complex_tester.create_dictionaries(SOURCE)
 
-    ranged_tester = RangedLayoutTester(TEST_NAME)
+    ranged_tester = RangedLayoutTester(test_name)
     ranged_tester.create_dictionaries(SOURCE)
     # Since that all .xml configs were created
 
@@ -43,7 +43,7 @@ def setup_module(module):
     main_configs = []
     main_configs.append(os.path.join("configs", "disable_ssl_verification.xml"))
 
-    dictionaries = simple_tester.list_dictionaries(TEST_NAME)
+    dictionaries = simple_tester.list_dictionaries()
 
     cluster.add_instance("clickhouse_remote", main_configs=main_configs)
 

tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py

@@ -17,7 +17,7 @@ node = None
 simple_tester = None
 complex_tester = None
 ranged_tester = None
-TEST_NAME = "cache"
+test_name = "cache"
 
 
 def setup_module(module):
@@ -27,14 +27,14 @@ def setup_module(module):
     global complex_tester
     global ranged_tester
 
-    simple_tester = SimpleLayoutTester(TEST_NAME)
+    simple_tester = SimpleLayoutTester(test_name)
     simple_tester.cleanup()
     simple_tester.create_dictionaries(SOURCE)
 
-    complex_tester = ComplexLayoutTester(TEST_NAME)
+    complex_tester = ComplexLayoutTester(test_name)
     complex_tester.create_dictionaries(SOURCE)
 
-    ranged_tester = RangedLayoutTester(TEST_NAME)
+    ranged_tester = RangedLayoutTester(test_name)
     ranged_tester.create_dictionaries(SOURCE)
     # Since that all .xml configs were created
 
@@ -43,7 +43,7 @@ def setup_module(module):
     main_configs = []
     main_configs.append(os.path.join("configs", "disable_ssl_verification.xml"))
 
-    dictionaries = simple_tester.list_dictionaries(TEST_NAME)
+    dictionaries = simple_tester.list_dictionaries()
 
     node = cluster.add_instance(
         "cache_node", main_configs=main_configs, dictionaries=dictionaries

tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py

@@ -17,7 +17,7 @@ node = None
 simple_tester = None
 complex_tester = None
 ranged_tester = None
-TEST_NAME = "hashed"
+test_name = "hashed"
 
 
 def setup_module(module):
@@ -27,14 +27,14 @@ def setup_module(module):
     global complex_tester
     global ranged_tester
 
-    simple_tester = SimpleLayoutTester(TEST_NAME)
+    simple_tester = SimpleLayoutTester(test_name)
     simple_tester.cleanup()
     simple_tester.create_dictionaries(SOURCE)
 
-    complex_tester = ComplexLayoutTester(TEST_NAME)
+    complex_tester = ComplexLayoutTester(test_name)
     complex_tester.create_dictionaries(SOURCE)
 
-    ranged_tester = RangedLayoutTester(TEST_NAME)
+    ranged_tester = RangedLayoutTester(test_name)
     ranged_tester.create_dictionaries(SOURCE)
     # Since that all .xml configs were created
 
@@ -43,7 +43,7 @@ def setup_module(module):
     main_configs = []
     main_configs.append(os.path.join("configs", "disable_ssl_verification.xml"))
 
-    dictionaries = simple_tester.list_dictionaries(TEST_NAME)
+    dictionaries = simple_tester.list_dictionaries()
 
     node = cluster.add_instance(
         "hashed_node", main_configs=main_configs, dictionaries=dictionaries

tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py

@@ -15,7 +15,7 @@ node = None
 simple_tester = None
 complex_tester = None
 ranged_tester = None
-TEST_NAME = "file"
+test_name = "file"
 
 
 def setup_module(module):
@@ -25,14 +25,14 @@ def setup_module(module):
     global complex_tester
     global ranged_tester
 
-    simple_tester = SimpleLayoutTester(TEST_NAME)
+    simple_tester = SimpleLayoutTester(test_name)
     simple_tester.cleanup()
     simple_tester.create_dictionaries(SOURCE)
 
-    complex_tester = ComplexLayoutTester(TEST_NAME)
+    complex_tester = ComplexLayoutTester(test_name)
     complex_tester.create_dictionaries(SOURCE)
 
-    ranged_tester = RangedLayoutTester(TEST_NAME)
+    ranged_tester = RangedLayoutTester(test_name)
     ranged_tester.create_dictionaries(SOURCE)
     # Since that all .xml configs were created
 
@@ -41,7 +41,7 @@ def setup_module(module):
     main_configs = []
     main_configs.append(os.path.join("configs", "disable_ssl_verification.xml"))
 
-    dictionaries = simple_tester.list_dictionaries(TEST_NAME)
+    dictionaries = simple_tester.list_dictionaries()
 
     node = cluster.add_instance(
         "file_node", main_configs=main_configs, dictionaries=dictionaries

tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py

@@ -15,7 +15,7 @@ node = None
 simple_tester = None
 complex_tester = None
 ranged_tester = None
-TEST_NAME = "http"
+test_name = "http"
 
 
 def setup_module(module):
@@ -25,14 +25,14 @@ def setup_module(module):
     global complex_tester
     global ranged_tester
 
-    simple_tester = SimpleLayoutTester(TEST_NAME)
+    simple_tester = SimpleLayoutTester(test_name)
     simple_tester.cleanup()
     simple_tester.create_dictionaries(SOURCE)
 
-    complex_tester = ComplexLayoutTester(TEST_NAME)
+    complex_tester = ComplexLayoutTester(test_name)
     complex_tester.create_dictionaries(SOURCE)
 
-    ranged_tester = RangedLayoutTester(TEST_NAME)
+    ranged_tester = RangedLayoutTester(test_name)
     ranged_tester.create_dictionaries(SOURCE)
     # Since that all .xml configs were created
 
@@ -41,7 +41,7 @@ def setup_module(module):
     main_configs = []
     main_configs.append(os.path.join("configs", "disable_ssl_verification.xml"))
 
-    dictionaries = simple_tester.list_dictionaries(TEST_NAME)
+    dictionaries = simple_tester.list_dictionaries()
 
     cluster.add_instance("clickhouse_h", main_configs=main_configs)
 

tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py

@@ -17,7 +17,7 @@ node = None
 simple_tester = None
 complex_tester = None
 ranged_tester = None
-TEST_NAME = "https"
+test_name = "https"
 
 
 def setup_module(module):
@@ -27,14 +27,14 @@ def setup_module(module):
     global complex_tester
     global ranged_tester
 
-    simple_tester = SimpleLayoutTester(TEST_NAME)
+    simple_tester = SimpleLayoutTester(test_name)
     simple_tester.cleanup()
     simple_tester.create_dictionaries(SOURCE)
 
-    complex_tester = ComplexLayoutTester(TEST_NAME)
+    complex_tester = ComplexLayoutTester(test_name)
     complex_tester.create_dictionaries(SOURCE)
 
-    ranged_tester = RangedLayoutTester(TEST_NAME)
+    ranged_tester = RangedLayoutTester(test_name)
     ranged_tester.create_dictionaries(SOURCE)
     # Since that all .xml configs were created
 
@@ -43,7 +43,7 @@ def setup_module(module):
     main_configs = []
     main_configs.append(os.path.join("configs", "disable_ssl_verification.xml"))
 
-    dictionaries = simple_tester.list_dictionaries(TEST_NAME)
+    dictionaries = simple_tester.list_dictionaries()
 
     cluster.add_instance("clickhouse_hs", main_configs=main_configs)
 

tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py

@@ -8,13 +8,13 @@ from helpers.cluster import ClickHouseCluster
 from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
 from helpers.external_sources import SourceMongo
 
+SOURCE = None
+cluster = None
+node = None
 simple_tester = None
 complex_tester = None
 ranged_tester = None
-TEST_NAME = "mongo"
-
-cluster_legacy = ClickHouseCluster(f"{__file__}_legacy")
-cluster_new = ClickHouseCluster(f"{__file__}_new")
+test_name = "mongo"
 
 
 @pytest.fixture(scope="module")
@@ -28,12 +28,16 @@ def legacy(request):
 
 
 @pytest.fixture(scope="module")
-def source(secure_connection, legacy, started_cluster):
-    print(f"XXXXX: {started_cluster.mongo_secure_port} {started_cluster.mongo_port} {secure_connection} {legacy}")
+def cluster(secure_connection):
+    return ClickHouseCluster(__file__)
+
+
+@pytest.fixture(scope="module")
+def source(secure_connection, legacy, cluster):
     return SourceMongo(
-        f"MongoDB",
+        "MongoDB",
         "localhost",
-        started_cluster.mongo_secure_port if secure_connection else started_cluster.mongo_port,
+        cluster.mongo_secure_port if secure_connection else cluster.mongo_port,
         "mongo_secure" if secure_connection else "mongo1",
         27017,
         "root",
@@ -44,31 +48,29 @@ def source(secure_connection, legacy, started_cluster):
 
 
 @pytest.fixture(scope="module")
-def simple_tester(source, started_cluster):
-    tester = SimpleLayoutTester(TEST_NAME)
+def simple_tester(source):
+    tester = SimpleLayoutTester(test_name)
     tester.cleanup()
     tester.create_dictionaries(source)
-    tester.prepare(started_cluster)
     return tester
 
 
 @pytest.fixture(scope="module")
-def complex_tester(source, started_cluster):
-    tester = ComplexLayoutTester(TEST_NAME)
+def complex_tester(source):
+    tester = ComplexLayoutTester(test_name)
     tester.create_dictionaries(source)
-    tester.prepare(started_cluster)
     return tester
 
 
 @pytest.fixture(scope="module")
-def ranged_tester(source, started_cluster):
-    tester = RangedLayoutTester(TEST_NAME)
+def ranged_tester(source):
+    tester = RangedLayoutTester(test_name)
     tester.create_dictionaries(source)
-    tester.prepare(started_cluster)
     return tester
 
 
-def get_config(secure_connection, legacy):
+@pytest.fixture(scope="module")
+def main_config(secure_connection, legacy):
     if legacy:
         main_config = [os.path.join("configs", "mongo", "legacy.xml")]
     else:
@@ -82,51 +84,51 @@ def get_config(secure_connection, legacy):
     return main_config
 
 
-def node_name(secure_connection, legacy):
-    secure_connection_suffix = "secure" if secure_connection else "non_secure"
-    legacy_suffix = "legacy" if legacy else "new"
+def get_node_name(secure_connection, legacy):
+    secure_connection_suffix = "_secure" if secure_connection else "non_secure"
+    legacy_suffix = "_legacy" if legacy else "_new"
     return f"node_mongo_{secure_connection_suffix}_{legacy_suffix}"
 
 
-def setup_nodes(is_secure, is_legacy):
-    cluster = cluster_legacy if is_legacy else cluster_new
-    return cluster.add_instance(
-        node_name(is_secure, is_legacy),
-        main_configs=get_config(is_secure, is_legacy),
-        dictionaries=BaseLayoutTester.list_dictionaries(TEST_NAME),
+@pytest.fixture(scope="module")
+def started_cluster(
+    secure_connection,
+    legacy,
+    cluster,
+    main_config,
+    simple_tester,
+    ranged_tester,
+    complex_tester,
+):
+    SOURCE = SourceMongo(
+        "MongoDB",
+        "localhost",
+        27017,
+        "mongo_secure" if secure_connection else "mongo1",
+        27017,
+        "root",
+        "clickhouse",
+        secure=secure_connection,
+        legacy=legacy,
+    )
+    dictionaries = simple_tester.list_dictionaries()
+
+    node = cluster.add_instance(
+        get_node_name(secure_connection, legacy),
+        main_configs=main_config,
+        dictionaries=dictionaries,
         with_mongo=True,
     )
 
-all_nodes = [
-    setup_nodes(is_secure, is_legacy)
-    for is_secure in [False, True]
-    for is_legacy in [False, True]
-]
-
-
-# mongo_sources = [
-#     SourceMongo(
-#         "MongoDB",
-#         "localhost",
-#         27017,
-#         "mongo_secure" if is_secure else "mongo1",
-#         27017,
-#         "root",
-#         "clickhouse",
-#         secure=is_secure,
-#         legacy=is_legacy,
-#     )
-#     for is_secure in [False, True]
-#     for is_legacy in [False, True]
-# ]
-
-
-@pytest.fixture(scope="module")
-def started_cluster(legacy):
-    cluster = cluster_legacy if legacy else cluster_new
     try:
         cluster.start()
+
+        simple_tester.prepare(cluster)
+        complex_tester.prepare(cluster)
+        ranged_tester.prepare(cluster)
+
         yield cluster
+
     finally:
         cluster.shutdown()
 
@@ -136,7 +138,7 @@ def started_cluster(legacy):
 @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE))
 def test_simple(secure_connection, legacy, started_cluster, layout_name, simple_tester):
     simple_tester.execute(
-        layout_name, started_cluster.instances[node_name(secure_connection, legacy)]
+        layout_name, started_cluster.instances[get_node_name(secure_connection, legacy)]
     )
 
 
@@ -147,7 +149,7 @@ def test_complex(
     secure_connection, legacy, started_cluster, layout_name, complex_tester
 ):
     complex_tester.execute(
-        layout_name, started_cluster.instances[node_name(secure_connection, legacy)]
+        layout_name, started_cluster.instances[get_node_name(secure_connection, legacy)]
     )
 
 
@@ -156,7 +158,7 @@ def test_complex(
 @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED))
 def test_ranged(secure_connection, legacy, started_cluster, layout_name, ranged_tester):
     ranged_tester.execute(
-        layout_name, started_cluster.instances[node_name(secure_connection, legacy)]
+        layout_name, started_cluster.instances[get_node_name(secure_connection, legacy)]
     )
 
 
@@ -167,5 +169,5 @@ def test_simple_ssl(
     secure_connection, legacy, started_cluster, layout_name, simple_tester
 ):
     simple_tester.execute(
-        layout_name, started_cluster.instances[node_name(secure_connection, legacy)]
+        layout_name, started_cluster.instances[get_node_name(secure_connection, legacy)]
     )

tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py

@@ -8,7 +8,7 @@ from helpers.cluster import ClickHouseCluster
 from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
 from helpers.external_sources import SourceMongoURI
 
-TEST_NAME = "mongo_uri"
+test_name = "mongo_uri"
 
 
 @pytest.fixture(scope="module")
@@ -43,7 +43,7 @@ def source(secure_connection, legacy, cluster):
 
 @pytest.fixture(scope="module")
 def simple_tester(source):
-    tester = SimpleLayoutTester(TEST_NAME)
+    tester = SimpleLayoutTester(test_name)
     tester.cleanup()
     tester.create_dictionaries(source)
     return tester
@@ -66,7 +66,7 @@ def main_config(secure_connection, legacy):
 
 @pytest.fixture(scope="module")
 def started_cluster(secure_connection, legacy, cluster, main_config, simple_tester):
-    dictionaries = simple_tester.list_dictionaries(TEST_NAME)
+    dictionaries = simple_tester.list_dictionaries()
 
     node = cluster.add_instance(
         "uri_node",

tests/integration/test_dictionaries_all_layouts_separate_sources/test_mysql.py

@@ -14,7 +14,7 @@ node = None
 simple_tester = None
 complex_tester = None
 ranged_tester = None
-TEST_NAME = "mysql"
+test_name = "mysql"
 
 
 def setup_module(module):
@@ -36,21 +36,21 @@ def setup_module(module):
         "clickhouse",
     )
 
-    simple_tester = SimpleLayoutTester(TEST_NAME)
+    simple_tester = SimpleLayoutTester(test_name)
     simple_tester.cleanup()
     simple_tester.create_dictionaries(SOURCE)
 
-    complex_tester = ComplexLayoutTester(TEST_NAME)
+    complex_tester = ComplexLayoutTester(test_name)
     complex_tester.create_dictionaries(SOURCE)
 
-    ranged_tester = RangedLayoutTester(TEST_NAME)
+    ranged_tester = RangedLayoutTester(test_name)
     ranged_tester.create_dictionaries(SOURCE)
     # Since that all .xml configs were created
 
     main_configs = []
     main_configs.append(os.path.join("configs", "disable_ssl_verification.xml"))
 
-    dictionaries = simple_tester.list_dictionaries(TEST_NAME)
+    dictionaries = simple_tester.list_dictionaries()
 
     node = cluster.add_instance(
         "node", main_configs=main_configs, dictionaries=dictionaries, with_mysql8=True