From caceb7c862001d1845781bc18862a2698f23e3eb Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 19 Jun 2023 13:21:09 +0300 Subject: [PATCH] Revert "Added ability to implicitly use file/hdfs/s3 table functions in clickhouse-local" --- programs/local/LocalServer.cpp | 13 +- src/Databases/DatabaseFactory.cpp | 70 +--- src/Databases/DatabaseFilesystem.cpp | 247 -------------- src/Databases/DatabaseFilesystem.h | 67 ---- src/Databases/DatabaseHDFS.cpp | 234 ------------- src/Databases/DatabaseHDFS.h | 68 ---- src/Databases/DatabaseS3.cpp | 312 ------------------ src/Databases/DatabaseS3.h | 81 ----- src/Databases/DatabasesOverlay.cpp | 266 --------------- src/Databases/DatabasesOverlay.h | 66 ---- src/Databases/IDatabase.h | 4 +- src/Interpreters/DatabaseCatalog.cpp | 14 +- tests/config/config.d/named_collection.xml | 5 - ...cal_implicit_file_table_function.reference | 9 - ...ouse_local_implicit_file_table_function.sh | 45 --- .../02722_database_filesystem.reference | 15 - .../0_stateless/02722_database_filesystem.sh | 72 ---- .../0_stateless/02724_database_s3.reference | 21 -- .../queries/0_stateless/02724_database_s3.sh | 63 ---- .../0_stateless/02725_database_hdfs.reference | 12 - .../0_stateless/02725_database_hdfs.sh | 60 ---- 21 files changed, 7 insertions(+), 1737 deletions(-) delete mode 100644 src/Databases/DatabaseFilesystem.cpp delete mode 100644 src/Databases/DatabaseFilesystem.h delete mode 100644 src/Databases/DatabaseHDFS.cpp delete mode 100644 src/Databases/DatabaseHDFS.h delete mode 100644 src/Databases/DatabaseS3.cpp delete mode 100644 src/Databases/DatabaseS3.h delete mode 100644 src/Databases/DatabasesOverlay.cpp delete mode 100644 src/Databases/DatabasesOverlay.h delete mode 100644 tests/queries/0_stateless/02707_clickhouse_local_implicit_file_table_function.reference delete mode 100755 tests/queries/0_stateless/02707_clickhouse_local_implicit_file_table_function.sh delete mode 100644 tests/queries/0_stateless/02722_database_filesystem.reference delete mode 100755 tests/queries/0_stateless/02722_database_filesystem.sh delete mode 100644 tests/queries/0_stateless/02724_database_s3.reference delete mode 100755 tests/queries/0_stateless/02724_database_s3.sh delete mode 100644 tests/queries/0_stateless/02725_database_hdfs.reference delete mode 100755 tests/queries/0_stateless/02725_database_hdfs.sh diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 2afcd48dafb..caca7cfb50d 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -8,9 +8,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -52,8 +50,6 @@ #include #include -#include "config.h" - #if defined(FUZZING_MODE) #include #endif @@ -174,13 +170,6 @@ static DatabasePtr createMemoryDatabaseIfNotExists(ContextPtr context, const Str return system_database; } -static DatabasePtr createClickHouseLocalDatabaseOverlay(const String & name_, ContextPtr context_) -{ - auto databaseCombiner = std::make_shared(name_, context_); - databaseCombiner->registerNextDatabase(std::make_shared(name_, "", context_)); - databaseCombiner->registerNextDatabase(std::make_shared(name_, context_)); - return databaseCombiner; -} /// If path is specified and not empty, will try to setup server environment and load existing metadata void LocalServer::tryInitPath() @@ -680,7 +669,7 @@ void LocalServer::processConfig() * if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons. */ std::string default_database = config().getString("default_database", "_local"); - DatabaseCatalog::instance().attachDatabase(default_database, createClickHouseLocalDatabaseOverlay(default_database, global_context)); + DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared(default_database, global_context)); global_context->setCurrentDatabase(default_database); applyCmdOptions(global_context); diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 9d90c61bb41..e1c8afa52c0 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -48,14 +47,6 @@ #include #endif -#if USE_AWS_S3 -#include -#endif - -#if USE_HDFS -#include -#endif - namespace fs = std::filesystem; namespace DB @@ -140,13 +131,13 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String static const std::unordered_set database_engines{"Ordinary", "Atomic", "Memory", "Dictionary", "Lazy", "Replicated", "MySQL", "MaterializeMySQL", "MaterializedMySQL", - "PostgreSQL", "MaterializedPostgreSQL", "SQLite", "Filesystem", "S3", "HDFS"}; + "PostgreSQL", "MaterializedPostgreSQL", "SQLite"}; if (!database_engines.contains(engine_name)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Database engine name `{}` does not exist", engine_name); static const std::unordered_set engines_with_arguments{"MySQL", "MaterializeMySQL", "MaterializedMySQL", - "Lazy", "Replicated", "PostgreSQL", "MaterializedPostgreSQL", "SQLite", "Filesystem", "S3", "HDFS"}; + "Lazy", "Replicated", "PostgreSQL", "MaterializedPostgreSQL", "SQLite"}; static const std::unordered_set engines_with_table_overrides{"MaterializeMySQL", "MaterializedMySQL", "MaterializedPostgreSQL"}; bool engine_may_have_arguments = engines_with_arguments.contains(engine_name); @@ -441,63 +432,6 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String } #endif - else if (engine_name == "Filesystem") - { - const ASTFunction * engine = engine_define->engine; - - /// If init_path is empty, then the current path will be used - std::string init_path; - - if (engine->arguments && !engine->arguments->children.empty()) - { - if (engine->arguments->children.size() != 1) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Filesystem database requires at most 1 argument: filesystem_path"); - - const auto & arguments = engine->arguments->children; - init_path = safeGetLiteralValue(arguments[0], engine_name); - } - - return std::make_shared(database_name, init_path, context); - } - -#if USE_AWS_S3 - else if (engine_name == "S3") - { - const ASTFunction * engine = engine_define->engine; - - DatabaseS3::Configuration config; - - if (engine->arguments && !engine->arguments->children.empty()) - { - ASTs & engine_args = engine->arguments->children; - config = DatabaseS3::parseArguments(engine_args, context); - } - - return std::make_shared(database_name, config, context); - } -#endif - -#if USE_HDFS - else if (engine_name == "HDFS") - { - const ASTFunction * engine = engine_define->engine; - - /// If source_url is empty, then table name must contain full url - std::string source_url; - - if (engine->arguments && !engine->arguments->children.empty()) - { - if (engine->arguments->children.size() != 1) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "HDFS database requires at most 1 argument: source_url"); - - const auto & arguments = engine->arguments->children; - source_url = safeGetLiteralValue(arguments[0], engine_name); - } - - return std::make_shared(database_name, source_url, context); - } -#endif - throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "Unknown database engine: {}", engine_name); } diff --git a/src/Databases/DatabaseFilesystem.cpp b/src/Databases/DatabaseFilesystem.cpp deleted file mode 100644 index 001aa1f9ef6..00000000000 --- a/src/Databases/DatabaseFilesystem.cpp +++ /dev/null @@ -1,247 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace fs = std::filesystem; - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; - extern const int UNKNOWN_TABLE; - extern const int DATABASE_ACCESS_DENIED; - extern const int BAD_ARGUMENTS; - extern const int FILE_DOESNT_EXIST; -} - -DatabaseFilesystem::DatabaseFilesystem(const String & name_, const String & path_, ContextPtr context_) - : IDatabase(name_), WithContext(context_->getGlobalContext()), path(path_), log(&Poco::Logger::get("DatabaseFileSystem(" + name_ + ")")) -{ - bool is_local = context_->getApplicationType() == Context::ApplicationType::LOCAL; - fs::path user_files_path = is_local ? "" : fs::canonical(getContext()->getUserFilesPath()); - - if (fs::path(path).is_relative()) - { - path = user_files_path / path; - } - else if (!is_local && !pathStartsWith(fs::path(path), user_files_path)) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Path must be inside user-files path: {}", user_files_path.string()); - } - - path = fs::absolute(path).lexically_normal(); - if (!fs::exists(path)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path does not exist: {}", path); -} - -std::string DatabaseFilesystem::getTablePath(const std::string & table_name) const -{ - fs::path table_path = fs::path(path) / table_name; - return table_path.lexically_normal().string(); -} - -void DatabaseFilesystem::addTable(const std::string & table_name, StoragePtr table_storage) const -{ - std::lock_guard lock(mutex); - auto [_, inserted] = loaded_tables.emplace(table_name, table_storage); - if (!inserted) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Table with name `{}` already exists in database `{}` (engine {})", - table_name, getDatabaseName(), getEngineName()); -} - -bool DatabaseFilesystem::checkTableFilePath(const std::string & table_path, ContextPtr context_, bool throw_on_error) const -{ - /// If run in Local mode, no need for path checking. - bool check_path = context_->getApplicationType() != Context::ApplicationType::LOCAL; - const auto & user_files_path = context_->getUserFilesPath(); - - /// Check access for file before checking its existence. - if (check_path && !fileOrSymlinkPathStartsWith(table_path, user_files_path)) - { - if (throw_on_error) - throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "File is not inside {}", user_files_path); - else - return false; - } - - /// Check if the corresponding file exists. - if (!fs::exists(table_path)) - { - if (throw_on_error) - throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "File does not exist: {}", table_path); - else - return false; - } - - if (!fs::is_regular_file(table_path)) - { - if (throw_on_error) - throw Exception(ErrorCodes::FILE_DOESNT_EXIST, - "File is directory, but expected a file: {}", table_path); - else - return false; - } - - return true; -} - -StoragePtr DatabaseFilesystem::tryGetTableFromCache(const std::string & name) const -{ - StoragePtr table = nullptr; - { - std::lock_guard lock(mutex); - auto it = loaded_tables.find(name); - if (it != loaded_tables.end()) - table = it->second; - } - - /// Invalidate cache if file no longer exists. - if (table && !fs::exists(getTablePath(name))) - { - std::lock_guard lock(mutex); - loaded_tables.erase(name); - return nullptr; - } - - return table; -} - -bool DatabaseFilesystem::isTableExist(const String & name, ContextPtr context_) const -{ - if (tryGetTableFromCache(name)) - return true; - - return checkTableFilePath(getTablePath(name), context_, /* throw_on_error */false); -} - -StoragePtr DatabaseFilesystem::getTableImpl(const String & name, ContextPtr context_) const -{ - /// Check if table exists in loaded tables map. - if (auto table = tryGetTableFromCache(name)) - return table; - - auto table_path = getTablePath(name); - checkTableFilePath(table_path, context_, /* throw_on_error */true); - - /// If the file exists, create a new table using TableFunctionFile and return it. - auto args = makeASTFunction("file", std::make_shared(table_path)); - - auto table_function = TableFunctionFactory::instance().get(args, context_); - if (!table_function) - return nullptr; - - /// TableFunctionFile throws exceptions, if table cannot be created. - auto table_storage = table_function->execute(args, context_, name); - if (table_storage) - addTable(name, table_storage); - - return table_storage; -} - -StoragePtr DatabaseFilesystem::getTable(const String & name, ContextPtr context_) const -{ - /// getTableImpl can throw exceptions, do not catch them to show correct error to user. - if (auto storage = getTableImpl(name, context_)) - return storage; - - throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", - backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(name)); -} - -StoragePtr DatabaseFilesystem::tryGetTable(const String & name, ContextPtr context_) const -{ - try - { - return getTableImpl(name, context_); - } - catch (const Exception & e) - { - /// Ignore exceptions thrown by TableFunctionFile, which indicate that there is no table - /// see tests/02722_database_filesystem.sh for more details. - if (e.code() == ErrorCodes::BAD_ARGUMENTS - || e.code() == ErrorCodes::DATABASE_ACCESS_DENIED - || e.code() == ErrorCodes::FILE_DOESNT_EXIST) - { - return nullptr; - } - throw; - } -} - -bool DatabaseFilesystem::empty() const -{ - std::lock_guard lock(mutex); - return loaded_tables.empty(); -} - -ASTPtr DatabaseFilesystem::getCreateDatabaseQuery() const -{ - const auto & settings = getContext()->getSettingsRef(); - const String query = fmt::format("CREATE DATABASE {} ENGINE = Filesystem('{}')", backQuoteIfNeed(getDatabaseName()), path); - - ParserCreateQuery parser; - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth); - - if (const auto database_comment = getDatabaseComment(); !database_comment.empty()) - { - auto & ast_create_query = ast->as(); - ast_create_query.set(ast_create_query.comment, std::make_shared(database_comment)); - } - - return ast; -} - -void DatabaseFilesystem::shutdown() -{ - Tables tables_snapshot; - { - std::lock_guard lock(mutex); - tables_snapshot = loaded_tables; - } - - for (const auto & kv : tables_snapshot) - { - auto table_id = kv.second->getStorageID(); - kv.second->flushAndShutdown(); - } - - std::lock_guard lock(mutex); - loaded_tables.clear(); -} - -/** - * Returns an empty vector because the database is read-only and no tables can be backed up - */ -std::vector> DatabaseFilesystem::getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const -{ - return {}; -} - -/** - * - * Returns an empty iterator because the database does not have its own tables - * But only caches them for quick access - */ -DatabaseTablesIteratorPtr DatabaseFilesystem::getTablesIterator(ContextPtr, const FilterByNameFunction &) const -{ - return std::make_unique(Tables{}, getDatabaseName()); -} - -} diff --git a/src/Databases/DatabaseFilesystem.h b/src/Databases/DatabaseFilesystem.h deleted file mode 100644 index 7fe620401dc..00000000000 --- a/src/Databases/DatabaseFilesystem.h +++ /dev/null @@ -1,67 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -namespace DB -{ - -class Context; - -/** - * DatabaseFilesystem allows to interact with files stored on the local filesystem. - * Uses TableFunctionFile to implicitly load file when a user requests the table, - * and provides a read-only access to the data in the file. - * Tables are cached inside the database for quick access - * - * Used in clickhouse-local to access local files. - * For clickhouse-server requires allows to access file only from user_files directory. - */ -class DatabaseFilesystem : public IDatabase, protected WithContext -{ -public: - DatabaseFilesystem(const String & name, const String & path, ContextPtr context); - - String getEngineName() const override { return "Filesystem"; } - - bool isTableExist(const String & name, ContextPtr context) const override; - - StoragePtr getTable(const String & name, ContextPtr context) const override; - - StoragePtr tryGetTable(const String & name, ContextPtr context) const override; - - bool shouldBeEmptyOnDetach() const override { return false; } /// Contains only temporary tables. - - bool empty() const override; - - bool isReadOnly() const override { return true; } - - ASTPtr getCreateDatabaseQuery() const override; - - void shutdown() override; - - std::vector> getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const override; - - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr, const FilterByNameFunction &) const override; - -protected: - StoragePtr getTableImpl(const String & name, ContextPtr context) const; - - StoragePtr tryGetTableFromCache(const std::string & name) const; - - std::string getTablePath(const std::string & table_name) const; - - void addTable(const std::string & table_name, StoragePtr table_storage) const; - - bool checkTableFilePath(const std::string & table_path, ContextPtr context_, bool throw_on_error) const; - -private: - String path; - mutable Tables loaded_tables TSA_GUARDED_BY(mutex); - Poco::Logger * log; -}; - -} diff --git a/src/Databases/DatabaseHDFS.cpp b/src/Databases/DatabaseHDFS.cpp deleted file mode 100644 index 1a0145b9015..00000000000 --- a/src/Databases/DatabaseHDFS.cpp +++ /dev/null @@ -1,234 +0,0 @@ -#include "config.h" - -#if USE_HDFS - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -namespace fs = std::filesystem; - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; - extern const int UNKNOWN_TABLE; - extern const int BAD_ARGUMENTS; - extern const int FILE_DOESNT_EXIST; - extern const int UNACCEPTABLE_URL; - extern const int ACCESS_DENIED; - extern const int DATABASE_ACCESS_DENIED; - extern const int HDFS_ERROR; - extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; -} - -static constexpr std::string_view HDFS_HOST_REGEXP = "^hdfs://[^/]*"; - - -DatabaseHDFS::DatabaseHDFS(const String & name_, const String & source_url, ContextPtr context_) - : IDatabase(name_) - , WithContext(context_->getGlobalContext()) - , source(source_url) - , log(&Poco::Logger::get("DatabaseHDFS(" + name_ + ")")) -{ - if (!source.empty()) - { - if (!re2::RE2::FullMatch(source, std::string(HDFS_HOST_REGEXP))) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs host: {}. " - "It should have structure 'hdfs://:'", source); - - context_->getGlobalContext()->getRemoteHostFilter().checkURL(Poco::URI(source)); - } -} - -void DatabaseHDFS::addTable(const std::string & table_name, StoragePtr table_storage) const -{ - std::lock_guard lock(mutex); - auto [_, inserted] = loaded_tables.emplace(table_name, table_storage); - if (!inserted) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Table with name `{}` already exists in database `{}` (engine {})", - table_name, getDatabaseName(), getEngineName()); -} - -std::string DatabaseHDFS::getTablePath(const std::string & table_name) const -{ - if (table_name.starts_with("hdfs://")) - return table_name; - - if (source.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs url: {}. " - "It should have structure 'hdfs://:/path'", table_name); - - return fs::path(source) / table_name; -} - -bool DatabaseHDFS::checkUrl(const std::string & url, ContextPtr context_, bool throw_on_error) const -{ - try - { - checkHDFSURL(url); - context_->getGlobalContext()->getRemoteHostFilter().checkURL(Poco::URI(url)); - } - catch (...) - { - if (throw_on_error) - throw; - return false; - } - - return true; -} - -bool DatabaseHDFS::isTableExist(const String & name, ContextPtr context_) const -{ - std::lock_guard lock(mutex); - if (loaded_tables.find(name) != loaded_tables.end()) - return true; - - return checkUrl(name, context_, false); -} - -StoragePtr DatabaseHDFS::getTableImpl(const String & name, ContextPtr context_) const -{ - /// Check if the table exists in the loaded tables map. - { - std::lock_guard lock(mutex); - auto it = loaded_tables.find(name); - if (it != loaded_tables.end()) - return it->second; - } - - auto url = getTablePath(name); - - checkUrl(url, context_, true); - - auto args = makeASTFunction("hdfs", std::make_shared(url)); - - auto table_function = TableFunctionFactory::instance().get(args, context_); - if (!table_function) - return nullptr; - - /// TableFunctionHDFS throws exceptions, if table cannot be created. - auto table_storage = table_function->execute(args, context_, name); - if (table_storage) - addTable(name, table_storage); - - return table_storage; -} - -StoragePtr DatabaseHDFS::getTable(const String & name, ContextPtr context_) const -{ - /// Rethrow all exceptions from TableFunctionHDFS to show correct error to user. - if (auto storage = getTableImpl(name, context_)) - return storage; - - throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", - backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(name)); -} - -StoragePtr DatabaseHDFS::tryGetTable(const String & name, ContextPtr context_) const -{ - try - { - return getTableImpl(name, context_); - } - catch (const Exception & e) - { - // Ignore exceptions thrown by TableFunctionHDFS, which indicate that there is no table - if (e.code() == ErrorCodes::BAD_ARGUMENTS - || e.code() == ErrorCodes::ACCESS_DENIED - || e.code() == ErrorCodes::DATABASE_ACCESS_DENIED - || e.code() == ErrorCodes::FILE_DOESNT_EXIST - || e.code() == ErrorCodes::UNACCEPTABLE_URL - || e.code() == ErrorCodes::HDFS_ERROR - || e.code() == ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE) - { - return nullptr; - } - throw; - } - catch (const Poco::URISyntaxException &) - { - return nullptr; - } -} - -bool DatabaseHDFS::empty() const -{ - std::lock_guard lock(mutex); - return loaded_tables.empty(); -} - -ASTPtr DatabaseHDFS::getCreateDatabaseQuery() const -{ - const auto & settings = getContext()->getSettingsRef(); - ParserCreateQuery parser; - - const String query = fmt::format("CREATE DATABASE {} ENGINE = HDFS('{}')", backQuoteIfNeed(getDatabaseName()), source); - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth); - - if (const auto database_comment = getDatabaseComment(); !database_comment.empty()) - { - auto & ast_create_query = ast->as(); - ast_create_query.set(ast_create_query.comment, std::make_shared(database_comment)); - } - - return ast; -} - -void DatabaseHDFS::shutdown() -{ - Tables tables_snapshot; - { - std::lock_guard lock(mutex); - tables_snapshot = loaded_tables; - } - - for (const auto & kv : tables_snapshot) - { - auto table_id = kv.second->getStorageID(); - kv.second->flushAndShutdown(); - } - - std::lock_guard lock(mutex); - loaded_tables.clear(); -} - -/** - * Returns an empty vector because the database is read-only and no tables can be backed up - */ -std::vector> DatabaseHDFS::getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const -{ - return {}; -} - -/** - * - * Returns an empty iterator because the database does not have its own tables - * But only caches them for quick access - */ -DatabaseTablesIteratorPtr DatabaseHDFS::getTablesIterator(ContextPtr, const FilterByNameFunction &) const -{ - return std::make_unique(Tables{}, getDatabaseName()); -} - -} // DB - -#endif diff --git a/src/Databases/DatabaseHDFS.h b/src/Databases/DatabaseHDFS.h deleted file mode 100644 index 957b2080135..00000000000 --- a/src/Databases/DatabaseHDFS.h +++ /dev/null @@ -1,68 +0,0 @@ -#pragma once - -#include "config.h" - -#if USE_HDFS - -#include -#include -#include -#include -#include - -namespace DB -{ - -class Context; - -/** - * DatabaseHDFS allows to interact with files stored on the file system. - * Uses TableFunctionHDFS to implicitly load file when a user requests the table, - * and provides read-only access to the data in the file. - * Tables are cached inside the database for quick access. - */ -class DatabaseHDFS : public IDatabase, protected WithContext -{ -public: - DatabaseHDFS(const String & name, const String & source_url, ContextPtr context); - - String getEngineName() const override { return "S3"; } - - bool isTableExist(const String & name, ContextPtr context) const override; - - StoragePtr getTable(const String & name, ContextPtr context) const override; - - StoragePtr tryGetTable(const String & name, ContextPtr context) const override; - - bool shouldBeEmptyOnDetach() const override { return false; } /// Contains only temporary tables. - - bool empty() const override; - - bool isReadOnly() const override { return true; } - - ASTPtr getCreateDatabaseQuery() const override; - - void shutdown() override; - - std::vector> getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr, const FilterByNameFunction &) const override; - -protected: - StoragePtr getTableImpl(const String & name, ContextPtr context) const; - - void addTable(const std::string & table_name, StoragePtr table_storage) const; - - bool checkUrl(const std::string & url, ContextPtr context_, bool throw_on_error) const; - - std::string getTablePath(const std::string & table_name) const; - -private: - const String source; - - mutable Tables loaded_tables TSA_GUARDED_BY(mutex); - Poco::Logger * log; -}; - -} - -#endif diff --git a/src/Databases/DatabaseS3.cpp b/src/Databases/DatabaseS3.cpp deleted file mode 100644 index 11655f5f100..00000000000 --- a/src/Databases/DatabaseS3.cpp +++ /dev/null @@ -1,312 +0,0 @@ -#include "config.h" - -#if USE_AWS_S3 - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace fs = std::filesystem; - -namespace DB -{ - -static const std::unordered_set optional_configuration_keys = { - "url", - "access_key_id", - "secret_access_key", - "no_sign_request" -}; - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; - extern const int UNKNOWN_TABLE; - extern const int BAD_ARGUMENTS; - extern const int FILE_DOESNT_EXIST; - extern const int UNACCEPTABLE_URL; - extern const int S3_ERROR; - - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} - -DatabaseS3::DatabaseS3(const String & name_, const Configuration& config_, ContextPtr context_) - : IDatabase(name_) - , WithContext(context_->getGlobalContext()) - , config(config_) - , log(&Poco::Logger::get("DatabaseS3(" + name_ + ")")) -{ -} - -void DatabaseS3::addTable(const std::string & table_name, StoragePtr table_storage) const -{ - std::lock_guard lock(mutex); - auto [_, inserted] = loaded_tables.emplace(table_name, table_storage); - if (!inserted) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Table with name `{}` already exists in database `{}` (engine {})", - table_name, getDatabaseName(), getEngineName()); -} - -std::string DatabaseS3::getFullUrl(const std::string & name) const -{ - if (!config.url_prefix.empty()) - return fs::path(config.url_prefix) / name; - - return name; -} - -bool DatabaseS3::checkUrl(const std::string & url, ContextPtr context_, bool throw_on_error) const -{ - try - { - S3::URI uri(url); - context_->getGlobalContext()->getRemoteHostFilter().checkURL(uri.uri); - } - catch (...) - { - if (throw_on_error) - throw; - return false; - } - return true; -} - -bool DatabaseS3::isTableExist(const String & name, ContextPtr context_) const -{ - std::lock_guard lock(mutex); - if (loaded_tables.find(name) != loaded_tables.end()) - return true; - - return checkUrl(getFullUrl(name), context_, false); -} - -StoragePtr DatabaseS3::getTableImpl(const String & name, ContextPtr context_) const -{ - /// Check if the table exists in the loaded tables map. - { - std::lock_guard lock(mutex); - auto it = loaded_tables.find(name); - if (it != loaded_tables.end()) - return it->second; - } - - auto url = getFullUrl(name); - checkUrl(url, context_, /* throw_on_error */true); - - auto function = std::make_shared(); - function->name = "s3"; - function->arguments = std::make_shared(); - function->children.push_back(function->arguments); - - function->arguments->children.push_back(std::make_shared(url)); - if (config.no_sign_request) - { - function->arguments->children.push_back(std::make_shared("NOSIGN")); - } - else if (config.access_key_id.has_value() && config.secret_access_key.has_value()) - { - function->arguments->children.push_back(std::make_shared(config.access_key_id.value())); - function->arguments->children.push_back(std::make_shared(config.secret_access_key.value())); - } - - auto table_function = TableFunctionFactory::instance().get(function, context_); - if (!table_function) - return nullptr; - - /// TableFunctionS3 throws exceptions, if table cannot be created. - auto table_storage = table_function->execute(function, context_, name); - if (table_storage) - addTable(name, table_storage); - - return table_storage; -} - -StoragePtr DatabaseS3::getTable(const String & name, ContextPtr context_) const -{ - /// Rethrow all exceptions from TableFunctionS3 to show correct error to user. - if (auto storage = getTableImpl(name, context_)) - return storage; - - throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", - backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(name)); -} - -StoragePtr DatabaseS3::tryGetTable(const String & name, ContextPtr context_) const -{ - try - { - return getTableImpl(name, context_); - } - catch (const Exception & e) - { - /// Ignore exceptions thrown by TableFunctionS3, which indicate that there is no table. - if (e.code() == ErrorCodes::BAD_ARGUMENTS - || e.code() == ErrorCodes::S3_ERROR - || e.code() == ErrorCodes::FILE_DOESNT_EXIST - || e.code() == ErrorCodes::UNACCEPTABLE_URL) - { - return nullptr; - } - throw; - } - catch (const Poco::URISyntaxException &) - { - return nullptr; - } -} - -bool DatabaseS3::empty() const -{ - std::lock_guard lock(mutex); - return loaded_tables.empty(); -} - -ASTPtr DatabaseS3::getCreateDatabaseQuery() const -{ - const auto & settings = getContext()->getSettingsRef(); - ParserCreateQuery parser; - - std::string creation_args; - creation_args += fmt::format("'{}'", config.url_prefix); - if (config.no_sign_request) - creation_args += ", 'NOSIGN'"; - else if (config.access_key_id.has_value() && config.secret_access_key.has_value()) - creation_args += fmt::format(", '{}', '{}'", config.access_key_id.value(), config.secret_access_key.value()); - - const String query = fmt::format("CREATE DATABASE {} ENGINE = S3({})", backQuoteIfNeed(getDatabaseName()), creation_args); - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth); - - if (const auto database_comment = getDatabaseComment(); !database_comment.empty()) - { - auto & ast_create_query = ast->as(); - ast_create_query.set(ast_create_query.comment, std::make_shared(database_comment)); - } - - return ast; -} - -void DatabaseS3::shutdown() -{ - Tables tables_snapshot; - { - std::lock_guard lock(mutex); - tables_snapshot = loaded_tables; - } - - for (const auto & kv : tables_snapshot) - { - auto table_id = kv.second->getStorageID(); - kv.second->flushAndShutdown(); - } - - std::lock_guard lock(mutex); - loaded_tables.clear(); -} - -DatabaseS3::Configuration DatabaseS3::parseArguments(ASTs engine_args, ContextPtr context_) -{ - Configuration result; - - if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args, context_)) - { - auto & collection = *named_collection; - - validateNamedCollection(collection, {}, optional_configuration_keys); - - result.url_prefix = collection.getOrDefault("url", ""); - result.no_sign_request = collection.getOrDefault("no_sign_request", false); - - auto key_id = collection.getOrDefault("access_key_id", ""); - auto secret_key = collection.getOrDefault("secret_access_key", ""); - - if (!key_id.empty()) - result.access_key_id = key_id; - - if (!secret_key.empty()) - result.secret_access_key = secret_key; - } - else - { - const std::string supported_signature = - " - S3()\n" - " - S3('url')\n" - " - S3('url', 'NOSIGN')\n" - " - S3('url', 'access_key_id', 'secret_access_key')\n"; - const auto error_message = - fmt::format("Engine DatabaseS3 must have the following arguments signature\n{}", supported_signature); - - for (auto & arg : engine_args) - arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context_); - - if (engine_args.size() > 3) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, error_message.c_str()); - - if (engine_args.empty()) - return result; - - result.url_prefix = checkAndGetLiteralArgument(engine_args[0], "url"); - - // url, NOSIGN - if (engine_args.size() == 2) - { - auto second_arg = checkAndGetLiteralArgument(engine_args[1], "NOSIGN"); - if (boost::iequals(second_arg, "NOSIGN")) - result.no_sign_request = true; - else - throw Exception(ErrorCodes::BAD_ARGUMENTS, error_message.c_str()); - } - - // url, access_key_id, secret_access_key - if (engine_args.size() == 3) - { - auto key_id = checkAndGetLiteralArgument(engine_args[1], "access_key_id"); - auto secret_key = checkAndGetLiteralArgument(engine_args[2], "secret_access_key"); - - if (key_id.empty() || secret_key.empty() || boost::iequals(key_id, "NOSIGN")) - throw Exception(ErrorCodes::BAD_ARGUMENTS, error_message.c_str()); - - result.access_key_id = key_id; - result.secret_access_key = secret_key; - } - } - - return result; -} - -/** - * Returns an empty vector because the database is read-only and no tables can be backed up - */ -std::vector> DatabaseS3::getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const -{ - return {}; -} - -/** - * - * Returns an empty iterator because the database does not have its own tables - * But only caches them for quick access - */ -DatabaseTablesIteratorPtr DatabaseS3::getTablesIterator(ContextPtr, const FilterByNameFunction &) const -{ - return std::make_unique(Tables{}, getDatabaseName()); -} - -} - -#endif diff --git a/src/Databases/DatabaseS3.h b/src/Databases/DatabaseS3.h deleted file mode 100644 index 8297ae4e02d..00000000000 --- a/src/Databases/DatabaseS3.h +++ /dev/null @@ -1,81 +0,0 @@ -#pragma once - -#include "config.h" - -#if USE_AWS_S3 - -#include -#include -#include -#include -#include - -namespace DB -{ - -class Context; - -/** - * DatabaseS3 provides access to data stored in S3. - * Uses TableFunctionS3 to implicitly load file when a user requests the table, - * and provides read-only access to the data in the file. - * Tables are cached inside the database for quick access. - */ -class DatabaseS3 : public IDatabase, protected WithContext -{ -public: - struct Configuration - { - std::string url_prefix; - - bool no_sign_request = false; - - std::optional access_key_id; - std::optional secret_access_key; - }; - - DatabaseS3(const String & name, const Configuration& config, ContextPtr context); - - String getEngineName() const override { return "S3"; } - - bool isTableExist(const String & name, ContextPtr context) const override; - - StoragePtr getTable(const String & name, ContextPtr context) const override; - - StoragePtr tryGetTable(const String & name, ContextPtr context) const override; - - // Contains only temporary tables - bool shouldBeEmptyOnDetach() const override { return false; } - - bool empty() const override; - - bool isReadOnly() const override { return true; } - - ASTPtr getCreateDatabaseQuery() const override; - - void shutdown() override; - - std::vector> getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr, const FilterByNameFunction &) const override; - - static Configuration parseArguments(ASTs engine_args, ContextPtr context); - -protected: - StoragePtr getTableImpl(const String & name, ContextPtr context) const; - - void addTable(const std::string & table_name, StoragePtr table_storage) const; - - bool checkUrl(const std::string & url, ContextPtr context_, bool throw_on_error) const; - - std::string getFullUrl(const std::string & name) const; - -private: - const Configuration config; - - mutable Tables loaded_tables TSA_GUARDED_BY(mutex); - Poco::Logger * log; -}; - -} - -#endif diff --git a/src/Databases/DatabasesOverlay.cpp b/src/Databases/DatabasesOverlay.cpp deleted file mode 100644 index b44a9798072..00000000000 --- a/src/Databases/DatabasesOverlay.cpp +++ /dev/null @@ -1,266 +0,0 @@ -#include - -#include -#include -#include -#include - -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; - extern const int CANNOT_GET_CREATE_TABLE_QUERY; -} - -DatabasesOverlay::DatabasesOverlay(const String & name_, ContextPtr context_) - : IDatabase(name_), WithContext(context_->getGlobalContext()), log(&Poco::Logger::get("DatabaseOverlay(" + name_ + ")")) -{ -} - -DatabasesOverlay & DatabasesOverlay::registerNextDatabase(DatabasePtr database) -{ - databases.push_back(std::move(database)); - return *this; -} - -bool DatabasesOverlay::isTableExist(const String & table_name, ContextPtr context_) const -{ - for (const auto & db : databases) - { - if (db->isTableExist(table_name, context_)) - return true; - } - return false; -} - -StoragePtr DatabasesOverlay::tryGetTable(const String & table_name, ContextPtr context_) const -{ - StoragePtr result = nullptr; - for (const auto & db : databases) - { - result = db->tryGetTable(table_name, context_); - if (result) - break; - } - return result; -} - -void DatabasesOverlay::createTable(ContextPtr context_, const String & table_name, const StoragePtr & table, const ASTPtr & query) -{ - for (auto & db : databases) - { - if (!db->isReadOnly()) - { - db->createTable(context_, table_name, table, query); - return; - } - } - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "There is no databases for CREATE TABLE `{}` query in database `{}` (engine {})", - table_name, - getDatabaseName(), - getEngineName()); -} - -void DatabasesOverlay::dropTable(ContextPtr context_, const String & table_name, bool sync) -{ - for (auto & db : databases) - { - if (db->isTableExist(table_name, context_)) - { - db->dropTable(context_, table_name, sync); - return; - } - } - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "There is no databases for DROP TABLE `{}` query in database `{}` (engine {})", - table_name, - getDatabaseName(), - getEngineName()); -} - -void DatabasesOverlay::attachTable( - ContextPtr context_, const String & table_name, const StoragePtr & table, const String & relative_table_path) -{ - for (auto & db : databases) - { - try - { - db->attachTable(context_, table_name, table, relative_table_path); - return; - } - catch (...) - { - continue; - } - } - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "There is no databases for ATTACH TABLE `{}` query in database `{}` (engine {})", - table_name, - getDatabaseName(), - getEngineName()); -} - -StoragePtr DatabasesOverlay::detachTable(ContextPtr context_, const String & table_name) -{ - StoragePtr result = nullptr; - for (auto & db : databases) - { - if (db->isTableExist(table_name, context_)) - return db->detachTable(context_, table_name); - } - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "There is no databases for DETACH TABLE `{}` query in database `{}` (engine {})", - table_name, - getDatabaseName(), - getEngineName()); -} - -ASTPtr DatabasesOverlay::getCreateTableQueryImpl(const String & name, ContextPtr context_, bool throw_on_error) const -{ - ASTPtr result = nullptr; - for (const auto & db : databases) - { - result = db->tryGetCreateTableQuery(name, context_); - if (result) - break; - } - if (!result && throw_on_error) - throw Exception( - ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY, - "There is no metadata of table `{}` in database `{}` (engine {})", - name, - getDatabaseName(), - getEngineName()); - return result; -} - -/* - * DatabaseOverlay cannot be constructed by "CREATE DATABASE" query, as it is not a traditional ClickHouse database - * To use DatabaseOverlay, it must be constructed programmatically in code - */ -ASTPtr DatabasesOverlay::getCreateDatabaseQuery() const -{ - return std::make_shared(); -} - -String DatabasesOverlay::getTableDataPath(const String & table_name) const -{ - String result; - for (const auto & db : databases) - { - result = db->getTableDataPath(table_name); - if (!result.empty()) - break; - } - return result; -} - -String DatabasesOverlay::getTableDataPath(const ASTCreateQuery & query) const -{ - String result; - for (const auto & db : databases) - { - result = db->getTableDataPath(query); - if (!result.empty()) - break; - } - return result; -} - -UUID DatabasesOverlay::tryGetTableUUID(const String & table_name) const -{ - UUID result = UUIDHelpers::Nil; - for (const auto & db : databases) - { - result = db->tryGetTableUUID(table_name); - if (result != UUIDHelpers::Nil) - break; - } - return result; -} - -void DatabasesOverlay::drop(ContextPtr context_) -{ - for (auto & db : databases) - db->drop(context_); -} - -void DatabasesOverlay::alterTable(ContextPtr local_context, const StorageID & table_id, const StorageInMemoryMetadata & metadata) -{ - for (auto & db : databases) - { - if (!db->isReadOnly() && db->isTableExist(table_id.table_name, local_context)) - { - db->alterTable(local_context, table_id, metadata); - return; - } - } - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "There is no databases for ALTER TABLE `{}` query in database `{}` (engine {})", - table_id.table_name, - getDatabaseName(), - getEngineName()); -} - -std::vector> -DatabasesOverlay::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const -{ - std::vector> result; - for (const auto & db : databases) - { - auto db_backup = db->getTablesForBackup(filter, local_context); - result.insert(result.end(), std::make_move_iterator(db_backup.begin()), std::make_move_iterator(db_backup.end())); - } - return result; -} - -void DatabasesOverlay::createTableRestoredFromBackup( - const ASTPtr & create_table_query, - ContextMutablePtr local_context, - std::shared_ptr /*restore_coordination*/, - UInt64 /*timeout_ms*/) -{ - /// Creates a tables by executing a "CREATE TABLE" query. - InterpreterCreateQuery interpreter{create_table_query, local_context}; - interpreter.setInternal(true); - interpreter.execute(); -} - -bool DatabasesOverlay::empty() const -{ - for (const auto & db : databases) - { - if (!db->empty()) - return false; - } - return true; -} - -void DatabasesOverlay::shutdown() -{ - for (auto & db : databases) - db->shutdown(); -} - -DatabaseTablesIteratorPtr DatabasesOverlay::getTablesIterator(ContextPtr context_, const FilterByNameFunction & filter_by_table_name) const -{ - Tables tables; - for (const auto & db : databases) - { - for (auto table_it = db->getTablesIterator(context_, filter_by_table_name); table_it->isValid(); table_it->next()) - tables.insert({table_it->name(), table_it->table()}); - } - return std::make_unique(std::move(tables), getDatabaseName()); -} - -} diff --git a/src/Databases/DatabasesOverlay.h b/src/Databases/DatabasesOverlay.h deleted file mode 100644 index 0f31bbd6a47..00000000000 --- a/src/Databases/DatabasesOverlay.h +++ /dev/null @@ -1,66 +0,0 @@ -#pragma once - -#include -#include - -namespace DB -{ - -/** - * Implements the IDatabase interface and combines multiple other databases - * Searches for tables in each database in order until found, and delegates operations to the appropriate database - * Useful for combining databases - * - * Used in clickhouse-local to combine DatabaseFileSystem and DatabaseMemory - */ -class DatabasesOverlay : public IDatabase, protected WithContext -{ -public: - DatabasesOverlay(const String & name_, ContextPtr context_); - - /// Not thread-safe. Use only as factory to initialize database - DatabasesOverlay & registerNextDatabase(DatabasePtr database); - - String getEngineName() const override { return "Overlay"; } - -public: - bool isTableExist(const String & table_name, ContextPtr context) const override; - - StoragePtr tryGetTable(const String & table_name, ContextPtr context) const override; - - void createTable(ContextPtr context, const String & table_name, const StoragePtr & table, const ASTPtr & query) override; - - void dropTable(ContextPtr context, const String & table_name, bool sync) override; - - void attachTable(ContextPtr context, const String & table_name, const StoragePtr & table, const String & relative_table_path) override; - - StoragePtr detachTable(ContextPtr context, const String & table_name) override; - - ASTPtr getCreateTableQueryImpl(const String & name, ContextPtr context, bool throw_on_error) const override; - ASTPtr getCreateDatabaseQuery() const override; - - String getTableDataPath(const String & table_name) const override; - String getTableDataPath(const ASTCreateQuery & query) const override; - - UUID tryGetTableUUID(const String & table_name) const override; - - void drop(ContextPtr context) override; - - void alterTable(ContextPtr local_context, const StorageID & table_id, const StorageInMemoryMetadata & metadata) override; - - std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override; - - void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr restore_coordination, UInt64 timeout_ms) override; - - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; - - bool empty() const override; - - void shutdown() override; - -protected: - std::vector databases; - Poco::Logger * log; -}; - -} diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 6508e2ce060..53a2f372814 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -170,7 +170,7 @@ public: /// Get the table for work. Return nullptr if there is no table. virtual StoragePtr tryGetTable(const String & name, ContextPtr context) const = 0; - virtual StoragePtr getTable(const String & name, ContextPtr context) const; + StoragePtr getTable(const String & name, ContextPtr context) const; virtual UUID tryGetTableUUID(const String & /*table_name*/) const { return UUIDHelpers::Nil; } @@ -183,8 +183,6 @@ public: /// Is the database empty. virtual bool empty() const = 0; - virtual bool isReadOnly() const { return false; } - /// Add the table to the database. Record its presence in the metadata. virtual void createTable( ContextPtr /*context*/, diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 129323cd6b3..8d3fa91a7fe 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -338,17 +338,9 @@ DatabaseAndTable DatabaseCatalog::getTableImpl( database = it->second; } - StoragePtr table = nullptr; - try - { - table = database->getTable(table_id.table_name, context_); - } - catch (const Exception & e) - { - if (exception) - exception->emplace(e); - } - + auto table = database->tryGetTable(table_id.table_name, context_); + if (!table && exception) + exception->emplace(Exception(ErrorCodes::UNKNOWN_TABLE, "Table {} doesn't exist", table_id.getNameForLogs())); if (!table) database = nullptr; diff --git a/tests/config/config.d/named_collection.xml b/tests/config/config.d/named_collection.xml index 5b716a7b8da..2e49c0c596f 100644 --- a/tests/config/config.d/named_collection.xml +++ b/tests/config/config.d/named_collection.xml @@ -32,10 +32,5 @@ testtest auto - - http://localhost:11111/test/ - test - testtest - diff --git a/tests/queries/0_stateless/02707_clickhouse_local_implicit_file_table_function.reference b/tests/queries/0_stateless/02707_clickhouse_local_implicit_file_table_function.reference deleted file mode 100644 index ccc02ad4f34..00000000000 --- a/tests/queries/0_stateless/02707_clickhouse_local_implicit_file_table_function.reference +++ /dev/null @@ -1,9 +0,0 @@ -Test 1: check explicit and implicit call of the file table function -explicit: -4 -implicit: -4 -Test 2: check Filesystem database -4 -Test 3: check show database with Filesystem -test02707 diff --git a/tests/queries/0_stateless/02707_clickhouse_local_implicit_file_table_function.sh b/tests/queries/0_stateless/02707_clickhouse_local_implicit_file_table_function.sh deleted file mode 100755 index 7c9095b3d8b..00000000000 --- a/tests/queries/0_stateless/02707_clickhouse_local_implicit_file_table_function.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -dir=${CLICKHOUSE_TEST_UNIQUE_NAME} -[[ -d $dir ]] && rm -rd $dir -mkdir $dir - -# Create temporary csv file for tests -echo '"id","str","int","text"' > $dir/tmp.csv -echo '1,"abc",123,"abacaba"' >> $dir/tmp.csv -echo '2,"def",456,"bacabaa"' >> $dir/tmp.csv -echo '3,"story",78912,"acabaab"' >> $dir/tmp.csv -echo '4,"history",21321321,"cabaaba"' >> $dir/tmp.csv - -################# -echo "Test 1: check explicit and implicit call of the file table function" - -echo "explicit:" -$CLICKHOUSE_LOCAL -q "SELECT COUNT(*) FROM file('${dir}/tmp.csv')" -echo "implicit:" -$CLICKHOUSE_LOCAL -q "SELECT COUNT(*) FROM \"${dir}/tmp.csv\"" - -################# -echo "Test 2: check Filesystem database" -$CLICKHOUSE_LOCAL --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test; -CREATE DATABASE test ENGINE = Filesystem('${dir}'); -SELECT COUNT(*) FROM test.\`tmp.csv\`; -DROP DATABASE test; -""" - -################# -echo "Test 3: check show database with Filesystem" -$CLICKHOUSE_LOCAL --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test02707; -CREATE DATABASE test02707 ENGINE = Filesystem('${dir}'); -SHOW DATABASES; -DROP DATABASE test02707; -""" | grep "test02707" - -# Remove temporary dir with files -rm -rd $dir diff --git a/tests/queries/0_stateless/02722_database_filesystem.reference b/tests/queries/0_stateless/02722_database_filesystem.reference deleted file mode 100644 index c65dda7933a..00000000000 --- a/tests/queries/0_stateless/02722_database_filesystem.reference +++ /dev/null @@ -1,15 +0,0 @@ -Test 1: create filesystem database and check implicit calls -0 -test1 -4 -4 -4 -Test 2: check DatabaseFilesystem access rights and errors handling on server -OK -OK -OK -OK -OK -OK -OK -OK diff --git a/tests/queries/0_stateless/02722_database_filesystem.sh b/tests/queries/0_stateless/02722_database_filesystem.sh deleted file mode 100755 index 4ff659ee746..00000000000 --- a/tests/queries/0_stateless/02722_database_filesystem.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-parallel - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -# see 01658_read_file_to_stringcolumn.sh -CLICKHOUSE_USER_FILES_PATH=$(clickhouse-client --query "select _path, _file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') - -# Prepare data -unique_name=${CLICKHOUSE_TEST_UNIQUE_NAME} -user_files_tmp_dir=${CLICKHOUSE_USER_FILES_PATH}/${unique_name} -mkdir -p ${user_files_tmp_dir}/tmp/ -echo '"id","str","int","text"' > ${user_files_tmp_dir}/tmp.csv -echo '1,"abc",123,"abacaba"' >> ${user_files_tmp_dir}/tmp.csv -echo '2,"def",456,"bacabaa"' >> ${user_files_tmp_dir}/tmp.csv -echo '3,"story",78912,"acabaab"' >> ${user_files_tmp_dir}/tmp.csv -echo '4,"history",21321321,"cabaaba"' >> ${user_files_tmp_dir}/tmp.csv - -tmp_dir=${CLICKHOUSE_TEST_UNIQUE_NAME} -[[ -d $tmp_dir ]] && rm -rd $tmp_dir -mkdir $tmp_dir -cp ${user_files_tmp_dir}/tmp.csv ${tmp_dir}/tmp.csv -cp ${user_files_tmp_dir}/tmp.csv ${user_files_tmp_dir}/tmp/tmp.csv -cp ${user_files_tmp_dir}/tmp.csv ${user_files_tmp_dir}/tmp.myext - -################# -echo "Test 1: create filesystem database and check implicit calls" -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test1; -CREATE DATABASE test1 ENGINE = Filesystem; -""" -echo $? -${CLICKHOUSE_CLIENT} --query "SHOW DATABASES" | grep "test1" -${CLICKHOUSE_CLIENT} --query "SELECT COUNT(*) FROM test1.\`${unique_name}/tmp.csv\`;" -${CLICKHOUSE_CLIENT} --query "SELECT COUNT(*) FROM test1.\`${unique_name}/tmp/tmp.csv\`;" -${CLICKHOUSE_LOCAL} -q "SELECT COUNT(*) FROM \"${tmp_dir}/tmp.csv\"" - -################# -echo "Test 2: check DatabaseFilesystem access rights and errors handling on server" -# DATABASE_ACCESS_DENIED: Allows list files only inside user_files -${CLICKHOUSE_CLIENT} --query "SELECT COUNT(*) FROM test1.\`../tmp.csv\`;" 2>&1| grep -F "Code: 291" > /dev/null && echo "OK" -${CLICKHOUSE_CLIENT} --query "SELECT COUNT(*) FROM test1.\`/tmp/tmp.csv\`;" 2>&1| grep -F "Code: 291" > /dev/null && echo "OK" -${CLICKHOUSE_CLIENT} --multiline --multiquery --query """ -USE test1; -SELECT COUNT(*) FROM \"../${tmp_dir}/tmp.csv\"; -""" 2>&1| grep -F "Code: 291" > /dev/null && echo "OK" -${CLICKHOUSE_CLIENT} --query "SELECT COUNT(*) FROM test1.\`../../../../../../tmp.csv\`;" 2>&1| grep -F "Code: 291" > /dev/null && echo "OK" - -# BAD_ARGUMENTS: path should be inside user_files -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test2; -CREATE DATABASE test2 ENGINE = Filesystem('/tmp'); -""" 2>&1| grep -F "Code: 36" > /dev/null && echo "OK" - -# BAD_ARGUMENTS: .../user_files/relative_unknown_dir does not exists -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test2; -CREATE DATABASE test2 ENGINE = Filesystem('relative_unknown_dir'); -""" 2>&1| grep -F "Code: 36" > /dev/null && echo "OK" - -# FILE_DOESNT_EXIST: unknown file -${CLICKHOUSE_CLIENT} --query "SELECT COUNT(*) FROM test1.\`tmp2.csv\`;" 2>&1| grep -F "Code: 107" > /dev/null && echo "OK" - -# BAD_ARGUMENTS: Cannot determine the file format by it's extension -${CLICKHOUSE_CLIENT} --query "SELECT COUNT(*) FROM test1.\`${unique_name}/tmp.myext\`;" 2>&1| grep -F "Code: 36" > /dev/null && echo "OK" - -# Clean -${CLICKHOUSE_CLIENT} --query "DROP DATABASE test1;" -rm -rd $tmp_dir -rm -rd $user_files_tmp_dir diff --git a/tests/queries/0_stateless/02724_database_s3.reference b/tests/queries/0_stateless/02724_database_s3.reference deleted file mode 100644 index 425cca6a077..00000000000 --- a/tests/queries/0_stateless/02724_database_s3.reference +++ /dev/null @@ -1,21 +0,0 @@ -Test 1: select from s3 -1 2 3 -4 5 6 -7 8 9 -0 0 0 -test1 -10 11 12 -13 14 15 -16 17 18 -0 0 0 -10 11 12 -13 14 15 -16 17 18 -0 0 0 -10 11 12 -13 14 15 -16 17 18 -0 0 0 -Test 2: check exceptions -OK -OK diff --git a/tests/queries/0_stateless/02724_database_s3.sh b/tests/queries/0_stateless/02724_database_s3.sh deleted file mode 100755 index 79199b43571..00000000000 --- a/tests/queries/0_stateless/02724_database_s3.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest, no-parallel -# Tag no-fasttest: Depends on AWS - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -################# -echo "Test 1: select from s3" -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test1; -CREATE DATABASE test1 ENGINE = S3; -USE test1; -SELECT * FROM \"http://localhost:11111/test/a.tsv\" -""" -${CLICKHOUSE_CLIENT} -q "SHOW DATABASES;" | grep test1 - -# check credentials with absolute path -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test2; -CREATE DATABASE test2 ENGINE = S3('', 'test', 'testtest'); -USE test2; -SELECT * FROM \"http://localhost:11111/test/b.tsv\" -""" - -# check credentials with relative path -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test4; -CREATE DATABASE test4 ENGINE = S3('http://localhost:11111/test', 'test', 'testtest'); -USE test4; -SELECT * FROM \"b.tsv\" -""" - -# Check named collection loading -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test5; -CREATE DATABASE test5 ENGINE = S3(s3_conn_db); -SELECT * FROM test5.\`b.tsv\` -""" - -################# -echo "Test 2: check exceptions" -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test3; -CREATE DATABASE test3 ENGINE = S3; -USE test3; -SELECT * FROM \"http://localhost:11111/test/a.myext\" -""" 2>&1| grep -F "BAD_ARGUMENTS" > /dev/null && echo "OK" - -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -USE test3; -SELECT * FROM \"abacaba\" -""" 2>&1| grep -F "BAD_ARGUMENTS" > /dev/null && echo "OK" - -# Cleanup -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test1; -DROP DATABASE IF EXISTS test2; -DROP DATABASE IF EXISTS test3; -DROP DATABASE IF EXISTS test4; -DROP DATABASE IF EXISTS test5; -""" diff --git a/tests/queries/0_stateless/02725_database_hdfs.reference b/tests/queries/0_stateless/02725_database_hdfs.reference deleted file mode 100644 index ef8adae2bbc..00000000000 --- a/tests/queries/0_stateless/02725_database_hdfs.reference +++ /dev/null @@ -1,12 +0,0 @@ -Test 1: select from hdfs database -1 2 3 -test1 -1 2 3 -test2 -Test 2: check exceptions -OK0 -OK1 -OK2 -OK3 -OK4 -OK5 diff --git a/tests/queries/0_stateless/02725_database_hdfs.sh b/tests/queries/0_stateless/02725_database_hdfs.sh deleted file mode 100755 index a78f3e6bbdc..00000000000 --- a/tests/queries/0_stateless/02725_database_hdfs.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest, use-hdfs, no-parallel - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -# Prepare data -${CLICKHOUSE_CLIENT} -q "insert into table function hdfs('hdfs://localhost:12222/test_02725_1.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 1, 2, 3 settings hdfs_truncate_on_insert=1;" -${CLICKHOUSE_CLIENT} -q "insert into table function hdfs('hdfs://localhost:12222/test_02725_2.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 4, 5, 6 settings hdfs_truncate_on_insert=1;" - -################# -echo "Test 1: select from hdfs database" - -# Database without specific host -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test1; -CREATE DATABASE test1 ENGINE = HDFS; -USE test1; -SELECT * FROM \"hdfs://localhost:12222/test_02725_1.tsv\" -""" -${CLICKHOUSE_CLIENT} -q "SHOW DATABASES;" | grep test1 - -# Database with host -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test2; -CREATE DATABASE test2 ENGINE = HDFS('hdfs://localhost:12222'); -USE test2; -SELECT * FROM \"test_02725_1.tsv\" -""" -${CLICKHOUSE_CLIENT} -q "SHOW DATABASES;" | grep test2 - -################# -echo "Test 2: check exceptions" - -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test3; -CREATE DATABASE test3 ENGINE = HDFS('abacaba'); -""" 2>&1| grep -F "BAD_ARGUMENTS" > /dev/null && echo "OK0" - -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test4; -CREATE DATABASE test4 ENGINE = HDFS; -USE test4; -SELECT * FROM \"abacaba/file.tsv\" -""" 2>&1| grep -F "BAD_ARGUMENTS" > /dev/null && echo "OK1" - -${CLICKHOUSE_CLIENT} -q "SELECT * FROM test4.\`http://localhost:11111/test/a.tsv\`" 2>&1| grep -F "BAD_ARGUMENTS" > /dev/null && echo "OK2" -${CLICKHOUSE_CLIENT} --query "SELECT * FROM test4.\`hdfs://localhost:12222/file.myext\`" 2>&1| grep -F "BAD_ARGUMENTS" > /dev/null && echo "OK3" -${CLICKHOUSE_CLIENT} --query "SELECT * FROM test4.\`hdfs://localhost:12222/test_02725_3.tsv\`" 2>&1| grep -F "CANNOT_EXTRACT_TABLE_STRUCTURE" > /dev/null && echo "OK4" -${CLICKHOUSE_CLIENT} --query "SELECT * FROM test4.\`hdfs://localhost:12222\`" 2>&1| grep -F "BAD_ARGUMENTS" > /dev/null && echo "OK5" - - -# Cleanup -${CLICKHOUSE_CLIENT} --multiline --multiquery -q """ -DROP DATABASE IF EXISTS test1; -DROP DATABASE IF EXISTS test2; -DROP DATABASE IF EXISTS test3; -DROP DATABASE IF EXISTS test4; -""" \ No newline at end of file