2017-04-01 09:19:00 +00:00
|
|
|
#include <Storages/StorageFile.h>
|
2017-12-30 00:36:06 +00:00
|
|
|
#include <Storages/StorageFactory.h>
|
2016-10-18 14:18:37 +00:00
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Interpreters/Context.h>
|
2017-12-30 00:36:06 +00:00
|
|
|
#include <Interpreters/evaluateConstantExpression.h>
|
|
|
|
|
|
|
|
#include <Parsers/ASTLiteral.h>
|
|
|
|
#include <Parsers/ASTIdentifier.h>
|
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <IO/ReadBufferFromFile.h>
|
|
|
|
#include <IO/WriteBufferFromFile.h>
|
|
|
|
#include <IO/WriteHelpers.h>
|
2017-12-30 00:36:06 +00:00
|
|
|
|
2018-06-10 19:22:49 +00:00
|
|
|
#include <Formats/FormatFactory.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <DataStreams/IProfilingBlockInputStream.h>
|
|
|
|
#include <DataStreams/IBlockOutputStream.h>
|
2016-10-18 14:18:37 +00:00
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Common/escapeForFileName.h>
|
2017-12-30 00:36:06 +00:00
|
|
|
#include <Common/typeid_cast.h>
|
2016-10-18 14:18:37 +00:00
|
|
|
|
2016-10-25 13:49:07 +00:00
|
|
|
#include <fcntl.h>
|
|
|
|
|
2018-04-06 09:53:29 +00:00
|
|
|
#include <Poco/Path.h>
|
2018-04-10 07:09:50 +00:00
|
|
|
#include <Poco/File.h>
|
2017-12-18 02:37:08 +00:00
|
|
|
|
2016-10-18 14:18:37 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2016-10-28 17:38:32 +00:00
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
extern const int CANNOT_WRITE_TO_FILE_DESCRIPTOR;
|
|
|
|
extern const int CANNOT_SEEK_THROUGH_FILE;
|
|
|
|
extern const int DATABASE_ACCESS_DENIED;
|
2017-12-30 00:36:06 +00:00
|
|
|
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
|
|
|
extern const int UNKNOWN_IDENTIFIER;
|
2017-11-03 19:53:10 +00:00
|
|
|
extern const int INCORRECT_FILE_NAME;
|
2018-04-10 08:54:31 +00:00
|
|
|
extern const int FILE_DOESNT_EXIST;
|
2017-11-03 19:53:10 +00:00
|
|
|
extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
|
2016-10-28 17:38:32 +00:00
|
|
|
};
|
|
|
|
|
2016-10-18 14:18:37 +00:00
|
|
|
|
|
|
|
static std::string getTablePath(const std::string & db_dir_path, const std::string & table_name, const std::string & format_name)
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
return db_dir_path + escapeForFileName(table_name) + "/data." + escapeForFileName(format_name);
|
2016-11-11 17:01:02 +00:00
|
|
|
}
|
|
|
|
|
2018-04-19 04:39:16 +00:00
|
|
|
/// Both db_dir_path and table_path must be converted to absolute paths (in particular, path cannot contain '..').
|
2018-04-19 04:36:08 +00:00
|
|
|
static void checkCreationIsAllowed(Context & context_global, const std::string & db_dir_path, const std::string & table_path, int table_fd)
|
2016-11-11 17:01:02 +00:00
|
|
|
{
|
2018-04-06 09:53:29 +00:00
|
|
|
if (context_global.getApplicationType() != Context::ApplicationType::SERVER)
|
|
|
|
return;
|
|
|
|
|
2018-04-06 13:43:29 +00:00
|
|
|
if (table_fd >= 0)
|
2018-04-06 10:11:39 +00:00
|
|
|
throw Exception("Using file descriptor as source of storage isn't allowed for server daemons", ErrorCodes::DATABASE_ACCESS_DENIED);
|
2018-04-06 13:43:29 +00:00
|
|
|
else if (!startsWith(table_path, db_dir_path))
|
2018-04-06 10:35:52 +00:00
|
|
|
throw Exception("Part path " + table_path + " is not inside " + db_dir_path, ErrorCodes::DATABASE_ACCESS_DENIED);
|
2018-04-10 07:09:50 +00:00
|
|
|
|
|
|
|
Poco::File table_path_poco_file = Poco::File(table_path);
|
|
|
|
if (!table_path_poco_file.exists())
|
2018-04-19 04:39:48 +00:00
|
|
|
throw Exception("File " + table_path + " is not exist", ErrorCodes::FILE_DOESNT_EXIST);
|
2018-04-10 07:09:50 +00:00
|
|
|
else if (table_path_poco_file.isDirectory())
|
|
|
|
throw Exception("File " + table_path + " must not be a directory", ErrorCodes::INCORRECT_FILE_NAME);
|
2016-10-18 14:18:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
StorageFile::StorageFile(
|
2017-04-01 07:20:54 +00:00
|
|
|
const std::string & table_path_,
|
|
|
|
int table_fd_,
|
|
|
|
const std::string & db_dir_path,
|
|
|
|
const std::string & table_name_,
|
|
|
|
const std::string & format_name_,
|
2018-03-06 20:18:34 +00:00
|
|
|
const ColumnsDescription & columns_,
|
2017-04-01 07:20:54 +00:00
|
|
|
Context & context_)
|
2018-03-06 20:18:34 +00:00
|
|
|
: IStorage(columns_),
|
2018-01-25 14:42:39 +00:00
|
|
|
table_name(table_name_), format_name(format_name_), context_global(context_), table_fd(table_fd_)
|
2016-10-18 14:18:37 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
if (table_fd < 0) /// Will use file
|
|
|
|
{
|
|
|
|
use_table_fd = false;
|
|
|
|
|
|
|
|
if (!table_path_.empty()) /// Is user's file
|
|
|
|
{
|
2018-04-06 10:35:52 +00:00
|
|
|
Poco::Path poco_path = Poco::Path(table_path_);
|
|
|
|
if (poco_path.isRelative())
|
|
|
|
poco_path = Poco::Path(db_dir_path, poco_path);
|
|
|
|
|
|
|
|
path = poco_path.absolute().toString();
|
2018-04-06 13:43:29 +00:00
|
|
|
checkCreationIsAllowed(context_global, db_dir_path, path, table_fd);
|
2017-04-01 07:20:54 +00:00
|
|
|
is_db_table = false;
|
|
|
|
}
|
|
|
|
else /// Is DB's file
|
|
|
|
{
|
2017-11-03 19:53:10 +00:00
|
|
|
if (db_dir_path.empty())
|
|
|
|
throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME);
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
path = getTablePath(db_dir_path, table_name, format_name);
|
|
|
|
is_db_table = true;
|
|
|
|
Poco::File(Poco::Path(path).parent()).createDirectories();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else /// Will use FD
|
|
|
|
{
|
2018-04-06 13:43:29 +00:00
|
|
|
checkCreationIsAllowed(context_global, db_dir_path, path, table_fd);
|
2018-04-06 09:53:29 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
is_db_table = false;
|
|
|
|
use_table_fd = true;
|
|
|
|
|
|
|
|
/// Save initial offset, it will be used for repeating SELECTs
|
|
|
|
/// If FD isn't seekable (lseek returns -1), then the second and subsequent SELECTs will fail.
|
|
|
|
table_fd_init_offset = lseek(table_fd, 0, SEEK_CUR);
|
|
|
|
}
|
2016-10-18 14:18:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
class StorageFileBlockInputStream : public IProfilingBlockInputStream
|
|
|
|
{
|
|
|
|
public:
|
2017-04-01 07:20:54 +00:00
|
|
|
StorageFileBlockInputStream(StorageFile & storage_, const Context & context, size_t max_block_size)
|
2017-07-28 17:34:02 +00:00
|
|
|
: storage(storage_)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
if (storage.use_table_fd)
|
|
|
|
{
|
2017-07-28 17:34:02 +00:00
|
|
|
storage.rwlock.lock();
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
/// We could use common ReadBuffer and WriteBuffer in storage to leverage cache
|
|
|
|
/// and add ability to seek unseekable files, but cache sync isn't supported.
|
|
|
|
|
|
|
|
if (storage.table_fd_was_used) /// We need seek to initial position
|
|
|
|
{
|
|
|
|
if (storage.table_fd_init_offset < 0)
|
|
|
|
throw Exception("File descriptor isn't seekable, inside " + storage.getName(), ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
|
|
|
|
|
|
|
|
/// ReadBuffer's seek() doesn't make sence, since cache is empty
|
|
|
|
if (lseek(storage.table_fd, storage.table_fd_init_offset, SEEK_SET) < 0)
|
|
|
|
throwFromErrno("Cannot seek file descriptor, inside " + storage.getName(), ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
|
|
|
|
}
|
|
|
|
|
|
|
|
storage.table_fd_was_used = true;
|
|
|
|
read_buf = std::make_unique<ReadBufferFromFileDescriptor>(storage.table_fd);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2017-07-28 17:34:02 +00:00
|
|
|
storage.rwlock.lock_shared();
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
read_buf = std::make_unique<ReadBufferFromFile>(storage.path);
|
|
|
|
}
|
|
|
|
|
2018-06-10 19:22:49 +00:00
|
|
|
reader = FormatFactory::instance().getInput(storage.format_name, *read_buf, storage.getSampleBlock(), context, max_block_size);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2017-07-28 17:34:02 +00:00
|
|
|
~StorageFileBlockInputStream() override
|
|
|
|
{
|
|
|
|
if (storage.use_table_fd)
|
|
|
|
storage.rwlock.unlock();
|
|
|
|
else
|
|
|
|
storage.rwlock.unlock_shared();
|
|
|
|
}
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
String getName() const override
|
|
|
|
{
|
|
|
|
return storage.getName();
|
|
|
|
}
|
|
|
|
|
|
|
|
Block readImpl() override
|
|
|
|
{
|
|
|
|
return reader->read();
|
|
|
|
}
|
|
|
|
|
2018-06-03 20:39:06 +00:00
|
|
|
Block getHeader() const override { return reader->getHeader(); }
|
2018-01-06 18:10:44 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void readPrefixImpl() override
|
|
|
|
{
|
|
|
|
reader->readPrefix();
|
|
|
|
}
|
|
|
|
|
|
|
|
void readSuffixImpl() override
|
|
|
|
{
|
|
|
|
reader->readSuffix();
|
|
|
|
}
|
2016-10-18 14:18:37 +00:00
|
|
|
|
|
|
|
private:
|
2017-04-01 07:20:54 +00:00
|
|
|
StorageFile & storage;
|
|
|
|
Block sample_block;
|
|
|
|
std::unique_ptr<ReadBufferFromFileDescriptor> read_buf;
|
|
|
|
BlockInputStreamPtr reader;
|
2016-10-18 14:18:37 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
BlockInputStreams StorageFile::read(
|
2017-12-01 21:13:25 +00:00
|
|
|
const Names & /*column_names*/,
|
|
|
|
const SelectQueryInfo & /*query_info*/,
|
2017-04-01 07:20:54 +00:00
|
|
|
const Context & context,
|
2017-12-01 21:13:25 +00:00
|
|
|
QueryProcessingStage::Enum & /*processed_stage*/,
|
2017-04-01 07:20:54 +00:00
|
|
|
size_t max_block_size,
|
2017-12-01 21:13:25 +00:00
|
|
|
unsigned /*num_streams*/)
|
2016-10-18 14:18:37 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
return BlockInputStreams(1, std::make_shared<StorageFileBlockInputStream>(*this, context, max_block_size));
|
2016-10-18 14:18:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
class StorageFileBlockOutputStream : public IBlockOutputStream
|
|
|
|
{
|
|
|
|
public:
|
2017-09-07 21:04:48 +00:00
|
|
|
explicit StorageFileBlockOutputStream(StorageFile & storage_)
|
2017-04-01 07:20:54 +00:00
|
|
|
: storage(storage_), lock(storage.rwlock)
|
|
|
|
{
|
|
|
|
if (storage.use_table_fd)
|
|
|
|
{
|
|
|
|
/** NOTE: Using real file binded to FD may be misleading:
|
|
|
|
* SELECT *; INSERT insert_data; SELECT *; last SELECT returns initil_fd_data + insert_data
|
|
|
|
* INSERT data; SELECT *; last SELECT returns only insert_data
|
|
|
|
*/
|
|
|
|
storage.table_fd_was_used = true;
|
|
|
|
write_buf = std::make_unique<WriteBufferFromFileDescriptor>(storage.table_fd);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
write_buf = std::make_unique<WriteBufferFromFile>(storage.path, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_APPEND | O_CREAT);
|
|
|
|
}
|
|
|
|
|
2018-06-10 19:22:49 +00:00
|
|
|
writer = FormatFactory::instance().getOutput(storage.format_name, *write_buf, storage.getSampleBlock(), storage.context_global);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2018-02-19 00:45:32 +00:00
|
|
|
Block getHeader() const override { return storage.getSampleBlock(); }
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void write(const Block & block) override
|
|
|
|
{
|
|
|
|
writer->write(block);
|
|
|
|
}
|
|
|
|
|
|
|
|
void writePrefix() override
|
|
|
|
{
|
|
|
|
writer->writePrefix();
|
|
|
|
}
|
|
|
|
|
|
|
|
void writeSuffix() override
|
|
|
|
{
|
|
|
|
writer->writeSuffix();
|
|
|
|
}
|
|
|
|
|
|
|
|
void flush() override
|
|
|
|
{
|
|
|
|
writer->flush();
|
|
|
|
}
|
2016-10-18 14:18:37 +00:00
|
|
|
|
|
|
|
private:
|
2017-04-01 07:20:54 +00:00
|
|
|
StorageFile & storage;
|
2017-07-28 17:34:02 +00:00
|
|
|
std::unique_lock<std::shared_mutex> lock;
|
2017-04-01 07:20:54 +00:00
|
|
|
std::unique_ptr<WriteBufferFromFileDescriptor> write_buf;
|
|
|
|
BlockOutputStreamPtr writer;
|
2016-10-18 14:18:37 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
BlockOutputStreamPtr StorageFile::write(
|
2017-12-01 21:13:25 +00:00
|
|
|
const ASTPtr & /*query*/,
|
|
|
|
const Settings & /*settings*/)
|
2016-10-18 14:18:37 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
return std::make_shared<StorageFileBlockOutputStream>(*this);
|
2016-10-18 14:18:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
void StorageFile::drop()
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
/// Extra actions are not required.
|
2016-10-18 14:18:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-12-01 21:13:25 +00:00
|
|
|
void StorageFile::rename(const String & new_path_to_db, const String & /*new_database_name*/, const String & new_table_name)
|
2016-10-18 14:18:37 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
if (!is_db_table)
|
|
|
|
throw Exception("Can't rename table '" + table_name + "' binded to user-defined file (or FD)", ErrorCodes::DATABASE_ACCESS_DENIED);
|
2016-10-18 14:18:37 +00:00
|
|
|
|
2017-07-28 17:34:02 +00:00
|
|
|
std::unique_lock<std::shared_mutex> lock(rwlock);
|
2016-10-18 14:18:37 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
std::string path_new = getTablePath(new_path_to_db, new_table_name, format_name);
|
|
|
|
Poco::File(Poco::Path(path_new).parent()).createDirectories();
|
|
|
|
Poco::File(path).renameTo(path_new);
|
2016-10-18 14:18:37 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
path = std::move(path_new);
|
2016-10-18 14:18:37 +00:00
|
|
|
}
|
|
|
|
|
2017-12-30 00:36:06 +00:00
|
|
|
|
|
|
|
void registerStorageFile(StorageFactory & factory)
|
|
|
|
{
|
|
|
|
factory.registerStorage("File", [](const StorageFactory::Arguments & args)
|
|
|
|
{
|
|
|
|
ASTs & engine_args = args.engine_args;
|
|
|
|
|
|
|
|
if (!(engine_args.size() == 1 || engine_args.size() == 2))
|
|
|
|
throw Exception(
|
|
|
|
"Storage File requires 1 or 2 arguments: name of used format and source.",
|
|
|
|
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
|
|
|
|
|
|
|
engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[0], args.local_context);
|
|
|
|
String format_name = static_cast<const ASTLiteral &>(*engine_args[0]).value.safeGet<String>();
|
|
|
|
|
|
|
|
int source_fd = -1;
|
|
|
|
String source_path;
|
|
|
|
if (engine_args.size() >= 2)
|
|
|
|
{
|
|
|
|
/// Will use FD if engine_args[1] is int literal or identifier with std* name
|
|
|
|
|
2018-04-19 05:32:09 +00:00
|
|
|
if (const ASTIdentifier * identifier = typeid_cast<const ASTIdentifier *>(engine_args[1].get()))
|
2017-12-30 00:36:06 +00:00
|
|
|
{
|
|
|
|
if (identifier->name == "stdin")
|
|
|
|
source_fd = STDIN_FILENO;
|
|
|
|
else if (identifier->name == "stdout")
|
|
|
|
source_fd = STDOUT_FILENO;
|
|
|
|
else if (identifier->name == "stderr")
|
|
|
|
source_fd = STDERR_FILENO;
|
|
|
|
else
|
|
|
|
throw Exception("Unknown identifier '" + identifier->name + "' in second arg of File storage constructor",
|
|
|
|
ErrorCodes::UNKNOWN_IDENTIFIER);
|
|
|
|
}
|
2018-04-19 05:32:09 +00:00
|
|
|
else if (const ASTLiteral * literal = typeid_cast<const ASTLiteral *>(engine_args[1].get()))
|
2017-12-30 00:36:06 +00:00
|
|
|
{
|
|
|
|
auto type = literal->value.getType();
|
|
|
|
if (type == Field::Types::Int64)
|
|
|
|
source_fd = static_cast<int>(literal->value.get<Int64>());
|
|
|
|
else if (type == Field::Types::UInt64)
|
|
|
|
source_fd = static_cast<int>(literal->value.get<UInt64>());
|
2018-04-19 05:32:09 +00:00
|
|
|
else if (type == Field::Types::String)
|
|
|
|
source_path = literal->value.get<String>();
|
2017-12-30 00:36:06 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return StorageFile::create(
|
|
|
|
source_path, source_fd,
|
2018-04-19 05:32:09 +00:00
|
|
|
args.data_path,
|
|
|
|
args.table_name, format_name, args.columns,
|
2017-12-30 00:36:06 +00:00
|
|
|
args.context);
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2016-10-18 14:18:37 +00:00
|
|
|
}
|