From 37b66c8a9e70cab6b04ab87d4366ee0d42858a7f Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 23 May 2022 12:48:48 +0000 Subject: [PATCH 1/3] Check format name on storage creation --- src/Formats/FormatFactory.cpp | 7 +++++++ src/Formats/FormatFactory.h | 3 +++ src/Storages/HDFS/StorageHDFS.cpp | 1 + src/Storages/StorageFile.cpp | 2 ++ src/Storages/StorageS3.cpp | 1 + src/Storages/StorageURL.cpp | 1 + .../02311_create_table_with_unknown_format.reference | 0 .../0_stateless/02311_create_table_with_unknown_format.sql | 4 ++++ 8 files changed, 19 insertions(+) create mode 100644 tests/queries/0_stateless/02311_create_table_with_unknown_format.reference create mode 100644 tests/queries/0_stateless/02311_create_table_with_unknown_format.sql diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 4c1b23a75ab..33fd68e67f7 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -585,6 +585,13 @@ bool FormatFactory::checkIfFormatHasAnySchemaReader(const String & name) return checkIfFormatHasSchemaReader(name) || checkIfFormatHasExternalSchemaReader(name); } +void FormatFactory::checkFormatName(const String & name) const +{ + auto it = dict.find(name); + if (it == dict.end()) + throw Exception("Unknown format " + name, ErrorCodes::UNKNOWN_FORMAT); +} + FormatFactory & FormatFactory::instance() { static FormatFactory ret; diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index f7d3c23d3b4..0431d1ef8c9 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -210,6 +210,9 @@ public: bool isInputFormat(const String & name) const; bool isOutputFormat(const String & name) const; + /// Check that format with specified name exists and throw an exception otherwise. + void checkFormatName(const String & name) const; + private: FormatsDictionary dict; FileExtensionFormats file_extension_formats; diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index d114bb67016..a90db60e341 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -146,6 +146,7 @@ StorageHDFS::StorageHDFS( , distributed_processing(distributed_processing_) , partition_by(partition_by_) { + FormatFactory::instance().checkFormatName(format_name); context_->getRemoteHostFilter().checkURL(Poco::URI(uri_)); checkHDFSURL(uri_); diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 47e32337dfe..a3a1d061ecc 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -382,6 +382,8 @@ StorageFile::StorageFile(CommonArguments args) , compression_method(args.compression_method) , base_path(args.getContext()->getPath()) { + if (format_name != "Distributed") + FormatFactory::instance().checkFormatName(format_name); } void StorageFile::setStorageMetadata(CommonArguments args) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index d402dce5ede..d7e3d5e5374 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -609,6 +609,7 @@ StorageS3::StorageS3( , partition_by(partition_by_) , is_key_with_globs(uri_.key.find_first_of("*?{") != std::string::npos) { + FormatFactory::instance().checkFormatName(format_name); context_->getGlobalContext()->getRemoteHostFilter().checkURL(uri_.uri); StorageInMemoryMetadata storage_metadata; diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 0db4fa75aba..1961711785d 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -74,6 +74,7 @@ IStorageURLBase::IStorageURLBase( , http_method(http_method_) , partition_by(partition_by_) { + FormatFactory::instance().checkFormatName(format_name); StorageInMemoryMetadata storage_metadata; if (columns_.empty()) diff --git a/tests/queries/0_stateless/02311_create_table_with_unknown_format.reference b/tests/queries/0_stateless/02311_create_table_with_unknown_format.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql b/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql new file mode 100644 index 00000000000..5f43ecd1c65 --- /dev/null +++ b/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql @@ -0,0 +1,4 @@ +create table test_02311 (x UInt32) engine=File(UnknownFormat); -- {serverError UNKNOWN_FORMAT} +create table test_02311 (x UInt32) engine=URL('http://some/url', UnknownFormat); -- {serverError UNKNOWN_FORMAT} +create table test_02311 (x UInt32) engine=S3('http://host:2020/test/data', UnknownFormat); -- {serverError UNKNOWN_FORMAT} +create table test_02311 (x UInt32) engine=HDFS('http://hdfs:9000/data', UnknownFormat); -- {serverError UNKNOWN_FORMAT} From b73d49158d73245e8ee190dfefad196b017f9bcb Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 23 May 2022 17:01:45 +0200 Subject: [PATCH 2/3] Fix test --- .../0_stateless/02311_create_table_with_unknown_format.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql b/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql index 5f43ecd1c65..d046ffebca2 100644 --- a/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql +++ b/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql @@ -1,3 +1,5 @@ +-- Tags: no-fasttest + create table test_02311 (x UInt32) engine=File(UnknownFormat); -- {serverError UNKNOWN_FORMAT} create table test_02311 (x UInt32) engine=URL('http://some/url', UnknownFormat); -- {serverError UNKNOWN_FORMAT} create table test_02311 (x UInt32) engine=S3('http://host:2020/test/data', UnknownFormat); -- {serverError UNKNOWN_FORMAT} From 9518c41dda147023d37c47b220c2650999411bea Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 25 May 2022 09:01:12 +0000 Subject: [PATCH 3/3] Try to fix tests --- .../test_allowed_url_from_config/test.py | 24 +++++++++---------- ...02311_create_table_with_unknown_format.sql | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index 01a2a500ebf..da9d4404c82 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -33,7 +33,7 @@ def start_cluster(): def test_config_with_hosts(start_cluster): assert ( node1.query( - "CREATE TABLE table_test_1_1 (word String) Engine=URL('http://host:80', HDFS)" + "CREATE TABLE table_test_1_1 (word String) Engine=URL('http://host:80', CSV)" ) == "" ) @@ -44,7 +44,7 @@ def test_config_with_hosts(start_cluster): == "" ) assert "not allowed" in node1.query_and_get_error( - "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://host:123', S3)" + "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://host:123', CSV)" ) assert "not allowed" in node1.query_and_get_error( "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://yandex2.ru', CSV)" @@ -60,7 +60,7 @@ def test_config_with_only_primary_hosts(start_cluster): ) assert ( node2.query( - "CREATE TABLE table_test_2_2 (word String) Engine=URL('https://host:123', S3)" + "CREATE TABLE table_test_2_2 (word String) Engine=URL('https://host:123', CSV)" ) == "" ) @@ -72,25 +72,25 @@ def test_config_with_only_primary_hosts(start_cluster): ) assert ( node2.query( - "CREATE TABLE table_test_2_4 (word String) Engine=URL('https://yandex.ru:87', HDFS)" + "CREATE TABLE table_test_2_4 (word String) Engine=URL('https://yandex.ru:87', CSV)" ) == "" ) assert "not allowed" in node2.query_and_get_error( - "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host', HDFS)" + "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host', CSV)" ) assert "not allowed" in node2.query_and_get_error( "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host:234', CSV)" ) assert "not allowed" in node2.query_and_get_error( - "CREATE TABLE table_test_2_6 (word String) Engine=URL('https://yandex2.ru', S3)" + "CREATE TABLE table_test_2_6 (word String) Engine=URL('https://yandex2.ru', CSV)" ) def test_config_with_only_regexp_hosts(start_cluster): assert ( node3.query( - "CREATE TABLE table_test_3_1 (word String) Engine=URL('https://host:80', HDFS)" + "CREATE TABLE table_test_3_1 (word String) Engine=URL('https://host:80', CSV)" ) == "" ) @@ -104,7 +104,7 @@ def test_config_with_only_regexp_hosts(start_cluster): "CREATE TABLE table_test_3_3 (word String) Engine=URL('https://host', CSV)" ) assert "not allowed" in node3.query_and_get_error( - "CREATE TABLE table_test_3_4 (word String) Engine=URL('https://yandex2.ru', S3)" + "CREATE TABLE table_test_3_4 (word String) Engine=URL('https://yandex2.ru', CSV)" ) @@ -123,7 +123,7 @@ def test_config_without_allowed_hosts_section(start_cluster): ) assert ( node4.query( - "CREATE TABLE table_test_4_3 (word String) Engine=URL('https://host', HDFS)" + "CREATE TABLE table_test_4_3 (word String) Engine=URL('https://host', CSV)" ) == "" ) @@ -135,7 +135,7 @@ def test_config_without_allowed_hosts_section(start_cluster): ) assert ( node4.query( - "CREATE TABLE table_test_4_5 (word String) Engine=URL('ftp://something.com', S3)" + "CREATE TABLE table_test_4_5 (word String) Engine=URL('ftp://something.com', CSV)" ) == "" ) @@ -149,13 +149,13 @@ def test_config_without_allowed_hosts(start_cluster): "CREATE TABLE table_test_5_2 (word String) Engine=S3('https://host:80/bucket/key', CSV)" ) assert "not allowed" in node5.query_and_get_error( - "CREATE TABLE table_test_5_3 (word String) Engine=URL('https://host', HDFS)" + "CREATE TABLE table_test_5_3 (word String) Engine=URL('https://host', CSV)" ) assert "not allowed" in node5.query_and_get_error( "CREATE TABLE table_test_5_4 (word String) Engine=URL('https://yandex.ru', CSV)" ) assert "not allowed" in node5.query_and_get_error( - "CREATE TABLE table_test_5_5 (word String) Engine=URL('ftp://something.com', S3)" + "CREATE TABLE table_test_5_5 (word String) Engine=URL('ftp://something.com', CSV)" ) diff --git a/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql b/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql index d046ffebca2..54e388c3cf0 100644 --- a/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql +++ b/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, use-hdfs, no-backward-compatibility-check:22.5 create table test_02311 (x UInt32) engine=File(UnknownFormat); -- {serverError UNKNOWN_FORMAT} create table test_02311 (x UInt32) engine=URL('http://some/url', UnknownFormat); -- {serverError UNKNOWN_FORMAT}