thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-14 10:22:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor StorageFile construction

Browse Source
This commit is contained in:
Alexander Tokmakov 2019-10-30 17:17:55 +03:00
parent a3b4762cde
commit 9bbeb6359a
3 changed files with 97 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
dbms/src/Storages/StorageFile.cpp
View File

@ -26,7 +26,6 @@
#include <Poco/File.h>
#include <re2/re2.h>
#include <re2/stringpiece.h>
#include <filesystem>
namespace fs = std::filesystem;
@ -118,63 +117,52 @@ static void checkCreationIsAllowed(Context & context_global, const std::string &
}
}
StorageFile::StorageFile(
const std::string & table_path_,
int table_fd_,
const std::string & relative_table_dir_path,
const std::string & database_name_,
const std::string & table_name_,
const std::string & format_name_,
const ColumnsDescription & columns_,
const ConstraintsDescription & constraints_,
Context & context_)
:
table_name(table_name_), database_name(database_name_), format_name(format_name_), context_global(context_), table_fd(table_fd_)
{
setColumns(columns_);
setConstraints(constraints_);
if (table_fd < 0) /// Will use file
{
String table_dir_path = context_global.getPath() + relative_table_dir_path + "/";
use_table_fd = false;
if (!table_path_.empty()) /// Is user's file
{
Poco::Path poco_path = Poco::Path(table_path_);
if (poco_path.isRelative())
poco_path = Poco::Path(table_dir_path, poco_path);
const std::string path = poco_path.absolute().toString();
paths = listFilesWithRegexpMatching("/", path);
for (const auto & cur_path : paths)
checkCreationIsAllowed(context_global, table_dir_path, cur_path);
is_db_table = false;
}
else /// Is DB's file
{
if (relative_table_dir_path.empty())
throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME);
paths = {getTablePath(table_dir_path, format_name)};
is_db_table = true;
Poco::File(Poco::Path(paths.back()).parent()).createDirectories();
}
}
else /// Will use FD
StorageFile::StorageFile(int table_fd_, CommonArguments args)
: StorageFile(args)
{
if (context_global.getApplicationType() == Context::ApplicationType::SERVER)
throw Exception("Using file descriptor as source of storage isn't allowed for server daemons", ErrorCodes::DATABASE_ACCESS_DENIED);
is_db_table = false;
use_table_fd = true;
table_fd = table_fd_;
/// Save initial offset, it will be used for repeating SELECTs
/// If FD isn't seekable (lseek returns -1), then the second and subsequent SELECTs will fail.
table_fd_init_offset = lseek(table_fd, 0, SEEK_CUR);
}
StorageFile::StorageFile(const std::string & table_path_, const std::string & user_files_absolute_path, CommonArguments args)
: StorageFile(args)
{
is_db_table = false;
Poco::Path poco_path = Poco::Path(table_path_);
if (poco_path.isRelative())
poco_path = Poco::Path(user_files_absolute_path, poco_path);
const std::string path = poco_path.absolute().toString();
paths = listFilesWithRegexpMatching("/", path);
for (const auto & cur_path : paths)
checkCreationIsAllowed(context_global, user_files_absolute_path, cur_path);
}
StorageFile::StorageFile(const std::string & relative_table_dir_path, CommonArguments args)
: StorageFile(args)
{
if (relative_table_dir_path.empty())
throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME);
String table_dir_path = context_global.getPath() + relative_table_dir_path + "/";
Poco::File(table_dir_path).createDirectories();
paths = {getTablePath(table_dir_path, format_name)};
}
StorageFile::StorageFile(CommonArguments args)
: table_name(args.table_name), database_name(args.database_name), format_name(args.format_name), context_global(args.context)
{
setColumns(args.columns);
setConstraints(args.constraints);
}
class StorageFileBlockInputStream : public IBlockInputStream
{
@ -369,12 +357,15 @@ void registerStorageFile(StorageFactory & factory)
engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[0], args.local_context);
String format_name = engine_args[0]->as<ASTLiteral &>().value.safeGet<String>();
StorageFile::CommonArguments common_args{args.database_name, args.table_name, format_name,
args.columns, args.constraints,args.context};
if (engine_args.size() == 1) /// Table in database
return StorageFile::create(args.relative_data_path, common_args);
/// Will use FD if engine_args[1] is int literal or identifier with std* name
int source_fd = -1;
String source_path;
if (engine_args.size() >= 2)
{
/// Will use FD if engine_args[1] is int literal or identifier with std* name
if (auto opt_name = tryGetIdentifierName(engine_args[1]))
{
if (*opt_name == "stdin")
@ -396,14 +387,14 @@ void registerStorageFile(StorageFactory & factory)
source_fd = static_cast<int>(literal->value.get<UInt64>());
else if (type == Field::Types::String)
source_path = literal->value.get<String>();
}
else
throw Exception("Second argument must be path or file descriptor", ErrorCodes::BAD_ARGUMENTS);
}
return StorageFile::create(
source_path, source_fd,
args.relative_data_path,
args.database_name, args.table_name, format_name, args.columns, args.constraints,
args.context);
if (0 <= source_fd) /// File descriptor
return StorageFile::create(source_fd, common_args);
else /// User's file
return StorageFile::create(source_path, args.context.getUserFilesPath(), common_args);
});
}

35
dbms/src/Storages/StorageFile.h
View File

@ -42,27 +42,32 @@ public:
Strings getDataPaths() const override;
struct CommonArguments
{
const std::string & database_name;
const std::string & table_name;
const std::string & format_name;
const ColumnsDescription & columns;
const ConstraintsDescription & constraints;
Context & context;
};
protected:
friend class StorageFileBlockInputStream;
friend class StorageFileBlockOutputStream;
/** there are three options (ordered by priority):
- use specified file descriptor if (fd >= 0)
- use specified table_path if it isn't empty
- create own table inside data/db/table/
*/
StorageFile(
const std::string & table_path_,
int table_fd_,
const std::string & relative_table_dir_path,
const std::string & database_name_,
const std::string & table_name_,
const std::string & format_name_,
const ColumnsDescription & columns_,
const ConstraintsDescription & constraints_,
Context & context_);
/// From file descriptor
StorageFile(int table_fd_, CommonArguments args);
/// From user's file
StorageFile(const std::string & table_path_, const std::string & user_files_absolute_path, CommonArguments args);
/// From table in database
StorageFile(const std::string & relative_table_dir_path, CommonArguments args);
private:
explicit StorageFile(CommonArguments args);
std::string table_name;
std::string database_name;
std::string format_name;

12
dbms/src/TableFunctions/TableFunctionFile.cpp
View File

@ -8,15 +8,9 @@ namespace DB
StoragePtr TableFunctionFile::getStorage(
const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name) const
{
return StorageFile::create(source,
-1,
global_context.getUserFilesPath(),
getDatabaseName(),
table_name,
format,
columns,
ConstraintsDescription{},
global_context);
StorageFile::CommonArguments args{getDatabaseName(), table_name, format, columns,ConstraintsDescription{},global_context};
return StorageFile::create(source, global_context.getUserFilesPath(), args);
}
void registerTableFunctionFile(TableFunctionFactory & factory)