mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 00:30:49 +00:00
Fix tests
This commit is contained in:
parent
7df5c83046
commit
9fb86f134b
@ -1436,6 +1436,7 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des
|
||||
ConstraintsDescription{},
|
||||
String{},
|
||||
{},
|
||||
String{},
|
||||
};
|
||||
StoragePtr storage = std::make_shared<StorageFile>(in_file, global_context->getUserFilesPath(), args);
|
||||
storage->startup();
|
||||
|
@ -380,11 +380,11 @@ std::unique_ptr<ReadBuffer> createReadBuffer(
|
||||
int table_fd,
|
||||
const String & compression_method,
|
||||
ContextPtr context,
|
||||
const String & path_to_archive = "auto")
|
||||
const String & path_to_archive = "")
|
||||
{
|
||||
CompressionMethod method;
|
||||
|
||||
if (path_to_archive != "auto")
|
||||
if (!path_to_archive.empty())
|
||||
{
|
||||
auto reader = createArchiveReader(path_to_archive);
|
||||
std::unique_ptr<ReadBuffer> in = reader->readFile(current_path);
|
||||
@ -591,7 +591,7 @@ StorageFile::StorageFile(int table_fd_, CommonArguments args)
|
||||
StorageFile::StorageFile(const std::string & table_path_, const std::string & user_files_path, CommonArguments args)
|
||||
: StorageFile(args)
|
||||
{
|
||||
if (args.path_to_archive != "auto")
|
||||
if (!args.path_to_archive.empty())
|
||||
{
|
||||
paths_to_archive = getPathsList(args.path_to_archive, user_files_path, args.getContext(), total_bytes_to_read);
|
||||
paths = {table_path_};
|
||||
@ -600,6 +600,7 @@ StorageFile::StorageFile(const std::string & table_path_, const std::string & us
|
||||
{
|
||||
paths = getPathsList(table_path_, user_files_path, args.getContext(), total_bytes_to_read);
|
||||
}
|
||||
|
||||
is_db_table = false;
|
||||
is_path_with_globs = paths.size() > 1;
|
||||
if (!paths.empty())
|
||||
@ -822,8 +823,11 @@ public:
|
||||
if (!storage->use_table_fd)
|
||||
{
|
||||
size_t current_file = 0, current_archive = 0;
|
||||
if (files_info->files.size() == 1 && !files_info->paths_to_archive.empty())
|
||||
if (!files_info->paths_to_archive.empty())
|
||||
{
|
||||
if (files_info->files.size() != 1)
|
||||
throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Can only read a single file from archive");
|
||||
|
||||
current_archive = files_info->next_archive_to_read.fetch_add(1);
|
||||
if (current_archive >= files_info->paths_to_archive.size())
|
||||
return {};
|
||||
@ -1478,6 +1482,7 @@ void registerStorageFile(StorageFactory & factory)
|
||||
factory_args.constraints,
|
||||
factory_args.comment,
|
||||
{},
|
||||
{},
|
||||
};
|
||||
|
||||
ASTs & engine_args_ast = factory_args.engine_args;
|
||||
@ -1548,7 +1553,7 @@ void registerStorageFile(StorageFactory & factory)
|
||||
else if (type == Field::Types::UInt64)
|
||||
source_fd = static_cast<int>(literal->value.get<UInt64>());
|
||||
else if (type == Field::Types::String)
|
||||
source_path = literal->value.get<String>();
|
||||
StorageFile::parseFileSource(literal->value.get<String>(), source_path, storage_args.path_to_archive);
|
||||
else
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second argument must be path or file descriptor");
|
||||
}
|
||||
@ -1620,4 +1625,32 @@ void StorageFile::addColumnsToCache(
|
||||
schema_cache.addMany(cache_keys, columns);
|
||||
}
|
||||
|
||||
void StorageFile::parseFileSource(String source, String & filename, String & path_to_archive)
|
||||
{
|
||||
size_t pos = source.find("::");
|
||||
if (pos == String::npos)
|
||||
{
|
||||
filename = std::move(source);
|
||||
return;
|
||||
}
|
||||
|
||||
std::string_view path_to_archive_view = std::string_view{source}.substr(0, pos);
|
||||
while (path_to_archive_view.back() == ' ')
|
||||
path_to_archive_view.remove_suffix(1);
|
||||
|
||||
if (path_to_archive_view.empty())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path to archive is empty");
|
||||
|
||||
path_to_archive = path_to_archive_view;
|
||||
|
||||
std::string_view filename_view = std::string_view{source}.substr(pos + 2);
|
||||
while (filename_view.front() == ' ')
|
||||
filename_view.remove_prefix(1);
|
||||
|
||||
if (filename_view.empty())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Filename is empty");
|
||||
|
||||
filename = filename_view;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -23,7 +23,7 @@ public:
|
||||
const ConstraintsDescription & constraints;
|
||||
const String & comment;
|
||||
const std::string rename_after_processing;
|
||||
std::string path_to_archive = "auto";
|
||||
std::string path_to_archive;
|
||||
};
|
||||
|
||||
/// From file descriptor
|
||||
@ -93,6 +93,8 @@ public:
|
||||
|
||||
static SchemaCache & getSchemaCache(const ContextPtr & context);
|
||||
|
||||
static void parseFileSource(String source, String & filename, String & path_to_archive);
|
||||
|
||||
protected:
|
||||
friend class StorageFileSource;
|
||||
friend class StorageFileSink;
|
||||
|
@ -25,31 +25,7 @@ namespace ErrorCodes
|
||||
void ITableFunctionFileLike::parseFirstArguments(const ASTPtr & arg, const ContextPtr &)
|
||||
{
|
||||
String path = checkAndGetLiteralArgument<String>(arg, "source");
|
||||
size_t pos = path.find("::");
|
||||
if (pos == String::npos)
|
||||
{
|
||||
filename = path;
|
||||
}
|
||||
else
|
||||
{
|
||||
std::string_view path_to_archive_view = std::string_view{path}.substr(0, pos);
|
||||
while (path_to_archive_view.back() == ' ')
|
||||
path_to_archive_view.remove_suffix(1);
|
||||
|
||||
if (path_to_archive_view.empty())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path to archive is empty");
|
||||
|
||||
path_to_archive = path_to_archive_view;
|
||||
|
||||
std::string_view filename_view = std::string_view{path}.substr(pos + 2);
|
||||
while (filename_view.front() == ' ')
|
||||
filename_view.remove_prefix(1);
|
||||
|
||||
if (filename_view.empty())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Filename is empty");
|
||||
|
||||
filename = filename_view;
|
||||
}
|
||||
StorageFile::parseFileSource(std::move(path), filename, path_to_archive);
|
||||
}
|
||||
|
||||
String ITableFunctionFileLike::getFormatFromFirstArgument()
|
||||
|
@ -42,7 +42,7 @@ protected:
|
||||
virtual String getFormatFromFirstArgument();
|
||||
|
||||
String filename;
|
||||
String path_to_archive = "auto";
|
||||
String path_to_archive;
|
||||
String format = "auto";
|
||||
String structure = "auto";
|
||||
String compression_method = "auto";
|
||||
|
@ -39,6 +39,8 @@ void TableFunctionFile::parseFirstArguments(const ASTPtr & arg, const ContextPtr
|
||||
fd = STDOUT_FILENO;
|
||||
else if (filename == "stderr")
|
||||
fd = STDERR_FILENO;
|
||||
else
|
||||
StorageFile::parseFileSource(std::move(filename), filename, path_to_archive);
|
||||
}
|
||||
else if (type == Field::Types::Int64 || type == Field::Types::UInt64)
|
||||
{
|
||||
@ -78,6 +80,7 @@ StoragePtr TableFunctionFile::getStorage(const String & source,
|
||||
global_context->getSettingsRef().rename_files_after_processing,
|
||||
path_to_archive,
|
||||
};
|
||||
|
||||
if (fd >= 0)
|
||||
return std::make_shared<StorageFile>(fd, args);
|
||||
|
||||
@ -98,6 +101,7 @@ ColumnsDescription TableFunctionFile::getActualTableStructure(ContextPtr context
|
||||
paths = StorageFile::getPathsList(filename, context->getUserFilesPath(), context, total_bytes_to_read);
|
||||
else
|
||||
paths_to_archives = StorageFile::getPathsList(path_to_archive, context->getUserFilesPath(), context, total_bytes_to_read);
|
||||
|
||||
return StorageFile::getTableStructureFromFile(format, paths, compression_method, std::nullopt, context, paths_to_archives);
|
||||
}
|
||||
|
||||
|
@ -1,7 +1,12 @@
|
||||
1 2
|
||||
3 4
|
||||
1 2
|
||||
3 4
|
||||
1
|
||||
3
|
||||
1
|
||||
3
|
||||
1
|
||||
3
|
||||
1
|
||||
3
|
||||
|
||||
|
@ -4,13 +4,24 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
echo -e "1,2\n3,4" >${CLICKHOUSE_TEST_UNIQUE_NAME}_data.csv
|
||||
zip ${CLICKHOUSE_TEST_UNIQUE_NAME}_archive1.zip ${CLICKHOUSE_TEST_UNIQUE_NAME}_data.csv > /dev/null
|
||||
zip ${CLICKHOUSE_TEST_UNIQUE_NAME}_archive2.zip ${CLICKHOUSE_TEST_UNIQUE_NAME}_data.csv > /dev/null
|
||||
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS 02661_archive_table"
|
||||
|
||||
$CLICKHOUSE_LOCAL --query "SELECT * FROM file('${CLICKHOUSE_TEST_UNIQUE_NAME}_archive1.zip :: ${CLICKHOUSE_TEST_UNIQUE_NAME}_data.csv')"
|
||||
$CLICKHOUSE_LOCAL --query "SELECT c1 FROM file('${CLICKHOUSE_TEST_UNIQUE_NAME}_archive{1..2}.zip :: ${CLICKHOUSE_TEST_UNIQUE_NAME}_data.csv')"
|
||||
user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}')
|
||||
|
||||
echo -e "1,2\n3,4" > ${CLICKHOUSE_TEST_UNIQUE_NAME}_data.csv
|
||||
zip ${user_files_path}/${CLICKHOUSE_TEST_UNIQUE_NAME}_archive1.zip ${CLICKHOUSE_TEST_UNIQUE_NAME}_data.csv > /dev/null
|
||||
zip ${user_files_path}/${CLICKHOUSE_TEST_UNIQUE_NAME}_archive2.zip ${CLICKHOUSE_TEST_UNIQUE_NAME}_data.csv > /dev/null
|
||||
|
||||
function read_archive_file() {
|
||||
$CLICKHOUSE_LOCAL --query "SELECT $1 FROM file('${user_files_path}/$2')"
|
||||
$CLICKHOUSE_CLIENT --query "CREATE TABLE 02661_archive_table Engine=File('CSV', '${user_files_path}/$2')"
|
||||
$CLICKHOUSE_CLIENT --query "SELECT $1 FROM 02661_archive_table"
|
||||
$CLICKHOUSE_CLIENT --query "DROP TABLE 02661_archive_table"
|
||||
}
|
||||
|
||||
read_archive_file "*" "${CLICKHOUSE_TEST_UNIQUE_NAME}_archive1.zip :: ${CLICKHOUSE_TEST_UNIQUE_NAME}_data.csv"
|
||||
read_archive_file "c1" "${CLICKHOUSE_TEST_UNIQUE_NAME}_archive{1..2}.zip :: ${CLICKHOUSE_TEST_UNIQUE_NAME}_data.csv"
|
||||
|
||||
rm ${CLICKHOUSE_TEST_UNIQUE_NAME}_data.csv
|
||||
rm ${CLICKHOUSE_TEST_UNIQUE_NAME}_archive1.zip
|
||||
rm ${CLICKHOUSE_TEST_UNIQUE_NAME}_archive2.zip
|
||||
rm ${user_files_path}/${CLICKHOUSE_TEST_UNIQUE_NAME}_archive1.zip
|
||||
rm ${user_files_path}/${CLICKHOUSE_TEST_UNIQUE_NAME}_archive2.zip
|
||||
|
Loading…
Reference in New Issue
Block a user