thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 09:32:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tests, url table function

Browse Source
This commit is contained in:
kssenii 2021-09-08 22:28:22 +03:00
parent ab6937ee45
commit 41bc68e82e
8 changed files with 86 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/Storages/StorageURL.cpp
View File

@ -389,7 +389,16 @@ URLBasedDataSourceConfiguration StorageURL::getConfiguration(ASTs & args, Contex
if (with_named_collection)
{
if (!storage_specific_args.empty())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Illegal arguments");
{
String illegal_args;
for (const auto & arg : storage_specific_args)
{
if (!illegal_args.empty())
illegal_args += ", ";
illegal_args += arg.first;
}
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown arguements {} for table function URL", illegal_args);
}
}
else
{

16
src/TableFunctions/ITableFunctionFileLike.h
View File

@ -12,6 +12,15 @@ class Context;
*/
class ITableFunctionFileLike : public ITableFunction
{
protected:
void parseArguments(const ASTPtr & ast_function, ContextPtr context) override;
String filename;
String format;
String structure;
String compression_method = "auto";
private:
StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns) const override;
@ -21,13 +30,6 @@ private:
ColumnsDescription getActualTableStructure(ContextPtr context) const override;
void parseArguments(const ASTPtr & ast_function, ContextPtr context) override;
bool hasStaticStructure() const override { return true; }
String filename;
String format;
String structure;
String compression_method = "auto";
};
}

36
src/TableFunctions/TableFunctionURL.cpp
View File

@ -4,6 +4,7 @@
#include <Access/AccessFlags.h>
#include <Formats/FormatFactory.h>
#include <Poco/URI.h>
#include <Parsers/ASTFunction.h>
#include <Storages/ColumnsDescription.h>
#include <Storages/StorageURL.h>
#include <TableFunctions/TableFunctionFactory.h>
@ -12,6 +13,41 @@
namespace DB
{
void TableFunctionURL::parseArguments(const ASTPtr & ast_function, ContextPtr context)
{
const auto & func_args = ast_function->as<ASTFunction &>();
if (!func_args.arguments)
throw Exception("Table function 'URL' must have arguments.", ErrorCodes::BAD_ARGUMENTS);
auto [common_configuration, storage_specific_args, with_named_collection] = getURLBasedDataSourceConfiguration(func_args.arguments->children, context);
URLBasedDataSourceConfiguration configuration(common_configuration);
if (with_named_collection)
{
if (!storage_specific_args.empty())
{
String illegal_args;
for (const auto & arg : storage_specific_args)
{
if (!illegal_args.empty())
illegal_args += ", ";
illegal_args += arg.first;
}
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown arguements {} for table function URL", illegal_args);
}
filename = configuration.url;
format = configuration.format;
structure = configuration.structure;
compression_method = configuration.compression_method;
}
else
{
ITableFunctionFileLike::parseArguments(ast_function, context);
}
}
StoragePtr TableFunctionURL::getStorage(
const String & source, const String & format_, const ColumnsDescription & columns, ContextPtr global_context,
const std::string & table_name, const String & compression_method_) const

3
src/TableFunctions/TableFunctionURL.h
View File

@ -19,6 +19,9 @@ public:
return name;
}
protected:
void parseArguments(const ASTPtr & ast_function, ContextPtr context) override;
private:
StoragePtr getStorage(
const String & source, const String & format_, const ColumnsDescription & columns, ContextPtr global_context,

2
tests/integration/test_allowed_url_from_config/test.py
View File

@ -2,7 +2,7 @@ import pytest
from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=['configs/config_with_hosts.xml'])
node1 = cluster.add_instance('node1', main_configs=['configs/config_with_hosts.xml', 'configs/named_collections.xml'])
node2 = cluster.add_instance('node2', main_configs=['configs/config_with_only_primary_hosts.xml'])
node3 = cluster.add_instance('node3', main_configs=['configs/config_with_only_regexp_hosts.xml'])
node4 = cluster.add_instance('node4', main_configs=[]) # No `remote_url_allow_hosts` at all.

2
tests/integration/test_redirect_url_storage/test.py
View File

@ -92,3 +92,5 @@ def test_predefined_connection_configuration(started_cluster):
node1.query(
"create table WebHDFSStorageWithRedirect (id UInt32, name String, weight Float64) ENGINE = URL(url1, url='http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', format='TSV')")
assert node1.query("SET max_http_get_redirects=1; select * from WebHDFSStorageWithRedirect") == "1\tMark\t72.53\n"
result = node1.query("SET max_http_get_redirects=1; select * from url(url1, url='http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', format='TSV', structure='id UInt32, name String, weight Float64')")
assert(result == "1\tMark\t72.53\n")

9
tests/integration/test_storage_s3/configs/named_collections.xml Normal file
View File

@ -0,0 +1,9 @@
<yandex>
<named_collections>
<s3_conf1>
<url>http://minio1:9001/root/test_table</url>
<access_key_id>minio</access_key_id>
<secret_access_key>minio123</secret_access_key>
</s3_conf1>
</named_collections>
</yandex>

17
tests/integration/test_storage_s3/test.py
View File

@ -87,7 +87,7 @@ def started_cluster():
cluster = ClickHouseCluster(__file__)
cluster.add_instance("restricted_dummy", main_configs=["configs/config_for_test_remote_host_filter.xml"],
with_minio=True)
cluster.add_instance("dummy", with_minio=True, main_configs=["configs/defaultS3.xml"])
cluster.add_instance("dummy", with_minio=True, main_configs=["configs/defaultS3.xml", "configs/named_collections.xml"])
cluster.add_instance("s3_max_redirects", with_minio=True, main_configs=["configs/defaultS3.xml"],
user_configs=["configs/s3_max_redirects.xml"])
logging.info("Starting cluster...")
@ -735,3 +735,18 @@ def test_truncate_table(started_cluster):
assert(len(list(minio.list_objects(started_cluster.minio_bucket, 'truncate/'))) == 0)
assert instance.query("SELECT * FROM {}".format(name)) == ""
def test_predefined_connection_configuration(started_cluster):
bucket = started_cluster.minio_bucket
instance = started_cluster.instances["dummy"] # type: ClickHouseInstance
name = "test_table"
instance.query("drop table if exists {}".format(name))
instance.query("CREATE TABLE {} (id UInt32) ENGINE = S3(s3_conf1, format='CSV')".format(name))
instance.query("INSERT INTO {} SELECT number FROM numbers(10)".format(name))
result = instance.query("SELECT * FROM {}".format(name))
assert result == instance.query("SELECT number FROM numbers(10)")
result = instance.query("SELECT * FROM s3(s3_conf1, format='CSV', structure='id UInt32')")
assert result == instance.query("SELECT number FROM numbers(10)")