2021-05-17 11:02:35 +00:00
|
|
|
#include "StorageSQLite.h"
|
|
|
|
|
2021-07-08 13:27:30 +00:00
|
|
|
#if USE_SQLITE
|
2021-10-02 07:13:14 +00:00
|
|
|
#include <base/range.h>
|
2022-04-27 15:05:45 +00:00
|
|
|
#include <Common/logger_useful.h>
|
2021-10-15 20:18:20 +00:00
|
|
|
#include <Processors/Sources/SQLiteSource.h>
|
2021-07-14 13:13:37 +00:00
|
|
|
#include <Databases/SQLite/SQLiteUtils.h>
|
2021-05-17 11:02:35 +00:00
|
|
|
#include <DataTypes/DataTypeString.h>
|
2021-05-24 18:59:10 +00:00
|
|
|
#include <Interpreters/Context.h>
|
|
|
|
#include <Formats/FormatFactory.h>
|
2021-10-11 16:11:50 +00:00
|
|
|
#include <Processors/Formats/IOutputFormat.h>
|
2021-05-24 18:59:10 +00:00
|
|
|
#include <IO/Operators.h>
|
|
|
|
#include <IO/WriteHelpers.h>
|
2021-05-17 11:02:35 +00:00
|
|
|
#include <Interpreters/evaluateConstantExpression.h>
|
|
|
|
#include <Parsers/ASTLiteral.h>
|
2021-07-23 14:25:35 +00:00
|
|
|
#include <Processors/Sinks/SinkToStorage.h>
|
2021-05-17 11:02:35 +00:00
|
|
|
#include <Storages/StorageFactory.h>
|
|
|
|
#include <Storages/transformQueryForExternalDatabase.h>
|
2022-05-20 19:49:31 +00:00
|
|
|
#include <QueryPipeline/Pipe.h>
|
2021-07-13 19:34:31 +00:00
|
|
|
#include <Common/filesystemHelpers.h>
|
2021-07-10 08:05:23 +00:00
|
|
|
|
2021-05-17 11:02:35 +00:00
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
2021-07-10 08:05:23 +00:00
|
|
|
|
2021-05-17 11:02:35 +00:00
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
2021-07-10 11:00:59 +00:00
|
|
|
extern const int SQLITE_ENGINE_ERROR;
|
2021-05-17 11:02:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
StorageSQLite::StorageSQLite(
|
|
|
|
const StorageID & table_id_,
|
2021-07-10 08:05:23 +00:00
|
|
|
SQLitePtr sqlite_db_,
|
2021-07-26 07:05:28 +00:00
|
|
|
const String & database_path_,
|
2021-05-17 11:02:35 +00:00
|
|
|
const String & remote_table_name_,
|
|
|
|
const ColumnsDescription & columns_,
|
|
|
|
const ConstraintsDescription & constraints_,
|
2021-06-01 20:52:12 +00:00
|
|
|
ContextPtr context_)
|
2021-05-17 11:02:35 +00:00
|
|
|
: IStorage(table_id_)
|
2021-05-24 18:59:10 +00:00
|
|
|
, WithContext(context_->getGlobalContext())
|
2021-05-17 11:02:35 +00:00
|
|
|
, remote_table_name(remote_table_name_)
|
2021-07-26 07:05:28 +00:00
|
|
|
, database_path(database_path_)
|
2021-07-10 08:05:23 +00:00
|
|
|
, sqlite_db(sqlite_db_)
|
2021-09-28 22:17:26 +00:00
|
|
|
, log(&Poco::Logger::get("StorageSQLite (" + table_id_.table_name + ")"))
|
2021-05-17 11:02:35 +00:00
|
|
|
{
|
|
|
|
StorageInMemoryMetadata storage_metadata;
|
|
|
|
storage_metadata.setColumns(columns_);
|
|
|
|
storage_metadata.setConstraints(constraints_);
|
|
|
|
setInMemoryMetadata(storage_metadata);
|
|
|
|
}
|
2021-07-08 13:27:30 +00:00
|
|
|
|
|
|
|
|
2021-05-17 11:02:35 +00:00
|
|
|
Pipe StorageSQLite::read(
|
|
|
|
const Names & column_names,
|
2021-07-20 15:20:21 +00:00
|
|
|
const StorageSnapshotPtr & storage_snapshot,
|
2021-05-17 11:02:35 +00:00
|
|
|
SelectQueryInfo & query_info,
|
2021-05-24 18:59:10 +00:00
|
|
|
ContextPtr context_,
|
2021-05-17 11:02:35 +00:00
|
|
|
QueryProcessingStage::Enum,
|
|
|
|
size_t max_block_size,
|
|
|
|
unsigned int)
|
|
|
|
{
|
2021-07-26 07:05:28 +00:00
|
|
|
if (!sqlite_db)
|
|
|
|
sqlite_db = openSQLiteDB(database_path, getContext(), /* throw_on_error */true);
|
|
|
|
|
2021-07-20 15:20:21 +00:00
|
|
|
storage_snapshot->check(column_names);
|
2021-05-17 11:02:35 +00:00
|
|
|
|
|
|
|
String query = transformQueryForExternalDatabase(
|
|
|
|
query_info,
|
2021-07-20 15:20:21 +00:00
|
|
|
storage_snapshot->metadata->getColumns().getOrdinary(),
|
2021-05-17 11:02:35 +00:00
|
|
|
IdentifierQuotingStyle::DoubleQuotes,
|
2021-06-01 20:52:12 +00:00
|
|
|
"",
|
2021-05-17 11:02:35 +00:00
|
|
|
remote_table_name,
|
2021-05-24 18:59:10 +00:00
|
|
|
context_);
|
2021-09-28 22:17:26 +00:00
|
|
|
LOG_TRACE(log, "Query: {}", query);
|
2021-05-17 11:02:35 +00:00
|
|
|
|
|
|
|
Block sample_block;
|
|
|
|
for (const String & column_name : column_names)
|
|
|
|
{
|
2021-07-20 15:20:21 +00:00
|
|
|
auto column_data = storage_snapshot->metadata->getColumns().getPhysical(column_name);
|
2021-05-17 11:02:35 +00:00
|
|
|
sample_block.insert({column_data.type, column_data.name});
|
|
|
|
}
|
|
|
|
|
2021-08-11 16:44:34 +00:00
|
|
|
return Pipe(std::make_shared<SQLiteSource>(sqlite_db, query, sample_block, max_block_size));
|
2021-05-17 11:02:35 +00:00
|
|
|
}
|
|
|
|
|
2021-07-08 13:27:30 +00:00
|
|
|
|
2021-07-23 14:25:35 +00:00
|
|
|
class SQLiteSink : public SinkToStorage
|
2021-05-24 18:59:10 +00:00
|
|
|
{
|
|
|
|
public:
|
2021-07-23 14:25:35 +00:00
|
|
|
explicit SQLiteSink(
|
2021-05-24 18:59:10 +00:00
|
|
|
const StorageSQLite & storage_,
|
|
|
|
const StorageMetadataPtr & metadata_snapshot_,
|
2021-07-10 08:05:23 +00:00
|
|
|
StorageSQLite::SQLitePtr sqlite_db_,
|
|
|
|
const String & remote_table_name_)
|
2021-07-26 10:08:40 +00:00
|
|
|
: SinkToStorage(metadata_snapshot_->getSampleBlock())
|
2021-07-23 14:25:35 +00:00
|
|
|
, storage{storage_}
|
2021-05-24 18:59:10 +00:00
|
|
|
, metadata_snapshot(metadata_snapshot_)
|
2021-07-10 08:05:23 +00:00
|
|
|
, sqlite_db(sqlite_db_)
|
2021-05-24 18:59:10 +00:00
|
|
|
, remote_table_name(remote_table_name_)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2021-07-23 19:33:59 +00:00
|
|
|
String getName() const override { return "SQLiteSink"; }
|
2021-05-24 18:59:10 +00:00
|
|
|
|
2021-07-23 14:25:35 +00:00
|
|
|
void consume(Chunk chunk) override
|
2021-05-24 18:59:10 +00:00
|
|
|
{
|
2021-09-03 17:29:36 +00:00
|
|
|
auto block = getHeader().cloneWithColumns(chunk.getColumns());
|
2021-05-24 18:59:10 +00:00
|
|
|
WriteBufferFromOwnString sqlbuf;
|
2021-07-10 08:05:23 +00:00
|
|
|
|
2021-05-24 18:59:10 +00:00
|
|
|
sqlbuf << "INSERT INTO ";
|
2021-06-01 20:52:12 +00:00
|
|
|
sqlbuf << doubleQuoteString(remote_table_name);
|
2021-05-24 18:59:10 +00:00
|
|
|
sqlbuf << " (";
|
|
|
|
|
|
|
|
for (auto it = block.begin(); it != block.end(); ++it)
|
|
|
|
{
|
|
|
|
if (it != block.begin())
|
|
|
|
sqlbuf << ", ";
|
2021-06-01 20:52:12 +00:00
|
|
|
sqlbuf << quoteString(it->name);
|
2021-05-24 18:59:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
sqlbuf << ") VALUES ";
|
|
|
|
|
2021-10-11 16:11:50 +00:00
|
|
|
auto writer = FormatFactory::instance().getOutputFormat("Values", sqlbuf, metadata_snapshot->getSampleBlock(), storage.getContext());
|
2021-05-24 18:59:10 +00:00
|
|
|
writer->write(block);
|
|
|
|
|
|
|
|
sqlbuf << ";";
|
|
|
|
|
|
|
|
char * err_message = nullptr;
|
2021-07-10 08:05:23 +00:00
|
|
|
int status = sqlite3_exec(sqlite_db.get(), sqlbuf.str().c_str(), nullptr, nullptr, &err_message);
|
2021-05-24 18:59:10 +00:00
|
|
|
|
|
|
|
if (status != SQLITE_OK)
|
|
|
|
{
|
|
|
|
String err_msg(err_message);
|
|
|
|
sqlite3_free(err_message);
|
2021-07-10 11:00:59 +00:00
|
|
|
throw Exception(ErrorCodes::SQLITE_ENGINE_ERROR,
|
|
|
|
"Failed to execute sqlite INSERT query. Status: {}. Message: {}",
|
|
|
|
status, err_msg);
|
2021-05-24 18:59:10 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
const StorageSQLite & storage;
|
|
|
|
StorageMetadataPtr metadata_snapshot;
|
2021-07-10 08:05:23 +00:00
|
|
|
StorageSQLite::SQLitePtr sqlite_db;
|
|
|
|
String remote_table_name;
|
2021-05-24 18:59:10 +00:00
|
|
|
};
|
|
|
|
|
2021-07-08 13:27:30 +00:00
|
|
|
|
2021-07-23 14:25:35 +00:00
|
|
|
SinkToStoragePtr StorageSQLite::write(const ASTPtr & /* query */, const StorageMetadataPtr & metadata_snapshot, ContextPtr)
|
2021-05-24 18:59:10 +00:00
|
|
|
{
|
2021-07-26 07:05:28 +00:00
|
|
|
if (!sqlite_db)
|
|
|
|
sqlite_db = openSQLiteDB(database_path, getContext(), /* throw_on_error */true);
|
2021-07-23 14:25:35 +00:00
|
|
|
return std::make_shared<SQLiteSink>(*this, metadata_snapshot, sqlite_db, remote_table_name);
|
2021-05-24 18:59:10 +00:00
|
|
|
}
|
|
|
|
|
2021-07-10 08:05:23 +00:00
|
|
|
|
2021-05-17 11:02:35 +00:00
|
|
|
void registerStorageSQLite(StorageFactory & factory)
|
|
|
|
{
|
2021-07-10 08:05:23 +00:00
|
|
|
factory.registerStorage("SQLite", [](const StorageFactory::Arguments & args) -> StoragePtr
|
|
|
|
{
|
|
|
|
ASTs & engine_args = args.engine_args;
|
|
|
|
|
|
|
|
if (engine_args.size() != 2)
|
|
|
|
throw Exception("SQLite database requires 2 arguments: database path, table name",
|
|
|
|
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
|
|
|
|
|
|
|
for (auto & engine_arg : engine_args)
|
|
|
|
engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, args.getLocalContext());
|
|
|
|
|
|
|
|
const auto database_path = engine_args[0]->as<ASTLiteral &>().value.safeGet<String>();
|
|
|
|
const auto table_name = engine_args[1]->as<ASTLiteral &>().value.safeGet<String>();
|
|
|
|
|
2021-07-26 07:05:28 +00:00
|
|
|
auto sqlite_db = openSQLiteDB(database_path, args.getContext(), /* throw_on_error */!args.attach);
|
2021-07-13 12:38:37 +00:00
|
|
|
|
2022-04-19 20:47:29 +00:00
|
|
|
return std::make_shared<StorageSQLite>(args.table_id, sqlite_db, database_path,
|
2021-07-10 08:05:23 +00:00
|
|
|
table_name, args.columns, args.constraints, args.getContext());
|
|
|
|
},
|
|
|
|
{
|
|
|
|
.source_access_type = AccessType::SQLITE,
|
|
|
|
});
|
2021-05-17 11:02:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|
2021-07-08 13:27:30 +00:00
|
|
|
|
|
|
|
#endif
|