Merge pull request #15767 from ClickHouse/fix_materialize_mysql_count_v2

Try fix select count() for MaterializeMySQL (another way)

src/Storages/StorageMaterializeMySQL.cpp

@@ -105,6 +105,17 @@ NamesAndTypesList StorageMaterializeMySQL::getVirtuals() const
     return nested_storage->getVirtuals();
 }
 
+IStorage::ColumnSizeByName StorageMaterializeMySQL::getColumnSizes() const
+{
+    auto sizes = nested_storage->getColumnSizes();
+    auto nested_header = nested_storage->getInMemoryMetadataPtr()->getSampleBlock();
+    String sign_column_name = nested_header.getByPosition(nested_header.columns() - 2).name;
+    String version_column_name = nested_header.getByPosition(nested_header.columns() - 1).name;
+    sizes.erase(sign_column_name);
+    sizes.erase(version_column_name);
+    return sizes;
+}
+
 }
 
 #endif

src/Storages/StorageMaterializeMySQL.h

@@ -26,6 +26,7 @@ public:
         const Context & context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override;
 
     NamesAndTypesList getVirtuals() const override;
+    ColumnSizeByName getColumnSizes() const override;
 
 private:
     StoragePtr nested_storage;

src/Storages/StorageMerge.cpp

@@ -538,6 +538,14 @@ void StorageMerge::convertingSourceStream(
     }
 }
 
+IStorage::ColumnSizeByName StorageMerge::getColumnSizes() const
+{
+
+    auto first_materialize_mysql = getFirstTable([](const StoragePtr & table) { return table && table->getName() == "MaterializeMySQL"; });
+    if (!first_materialize_mysql)
+        return {};
+    return first_materialize_mysql->getColumnSizes();
+}
 
 void registerStorageMerge(StorageFactory & factory)
 {

src/Storages/StorageMerge.h

@@ -66,6 +66,7 @@ private:
     DatabaseTablesIteratorPtr getDatabaseIterator(const Context & context) const;
 
     NamesAndTypesList getVirtuals() const override;
+    ColumnSizeByName getColumnSizes() const override;
 
 protected:
     StorageMerge(

tests/integration/test_materialize_mysql_database/materialize_with_ddl.py

@@ -431,3 +431,33 @@ def query_event_with_empty_transaction(clickhouse_node, mysql_node, service_name
     check_query(clickhouse_node, "SELECT * FROM test_database.t1 ORDER BY a FORMAT TSV", "1\tBEGIN\n2\tBEGIN\n")
     clickhouse_node.query("DROP DATABASE test_database")
     mysql_node.query("DROP DATABASE test_database")
+
+def select_without_columns(clickhouse_node, mysql_node, service_name):
+    mysql_node.query("CREATE DATABASE db")
+    mysql_node.query("CREATE TABLE db.t (a INT PRIMARY KEY, b INT)")
+    clickhouse_node.query(
+        "CREATE DATABASE db ENGINE = MaterializeMySQL('{}:3306', 'db', 'root', 'clickhouse')".format(service_name))
+    check_query(clickhouse_node, "SHOW TABLES FROM db FORMAT TSV", "t\n")
+    clickhouse_node.query("SYSTEM STOP MERGES db.t")
+    clickhouse_node.query("CREATE VIEW v AS SELECT * FROM db.t")
+    mysql_node.query("INSERT INTO db.t VALUES (1, 1), (2, 2)")
+    mysql_node.query("DELETE FROM db.t WHERE a=2;")
+    check_query(clickhouse_node, "SELECT count((_sign, _version)) FROM db.t FORMAT TSV", "3\n")
+
+    assert clickhouse_node.query("SELECT count(_sign) FROM db.t FORMAT TSV") == "2\n"
+    assert clickhouse_node.query("SELECT count(_version) FROM db.t FORMAT TSV") == "2\n"
+
+    assert clickhouse_node.query("SELECT count() FROM db.t FORMAT TSV") == "1\n"
+    assert clickhouse_node.query("SELECT count(*) FROM db.t FORMAT TSV") == "1\n"
+    assert clickhouse_node.query("SELECT count() FROM (SELECT * FROM db.t) FORMAT TSV") == "1\n"
+    assert clickhouse_node.query("SELECT count() FROM v FORMAT TSV") == "1\n"
+    assert clickhouse_node.query("SELECT count() FROM merge('db', 't') FORMAT TSV") == "1\n"
+    assert clickhouse_node.query("SELECT count() FROM remote('localhost', 'db', 't') FORMAT TSV") == "1\n"
+
+    assert clickhouse_node.query("SELECT _part FROM db.t FORMAT TSV") == "0_1_1_0\n"
+    assert clickhouse_node.query("SELECT _part FROM remote('localhost', 'db', 't') FORMAT TSV") == "0_1_1_0\n"
+
+
+    clickhouse_node.query("DROP VIEW v")
+    clickhouse_node.query("DROP DATABASE db")
+    mysql_node.query("DROP DATABASE db")

tests/integration/test_materialize_mysql_database/test.py

@@ -146,3 +146,10 @@ def test_materialize_database_ddl_with_empty_transaction_5_7(started_cluster, st
 
 def test_materialize_database_ddl_with_empty_transaction_8_0(started_cluster, started_mysql_8_0):
     materialize_with_ddl.query_event_with_empty_transaction(clickhouse_node, started_mysql_8_0, "mysql8_0")
+
+
+def test_select_without_columns_5_7(started_cluster, started_mysql_5_7):
+    materialize_with_ddl.select_without_columns(clickhouse_node, started_mysql_5_7, "mysql1")
+
+def test_select_without_columns_8_0(started_cluster, started_mysql_8_0):
+    materialize_with_ddl.select_without_columns(clickhouse_node, started_mysql_8_0, "mysql8_0")