ClickHouse/src/Storages/ExternalDataSourceConfiguration.h

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

93 lines
2.5 KiB
C++
Raw Normal View History

2021-09-01 17:59:11 +00:00
#pragma once
#include <Interpreters/Context.h>
#include <Poco/Util/AbstractConfiguration.h>
2022-04-03 22:33:59 +00:00
#include <Storages/StorageS3Settings.h>
#include <IO/HTTPHeaderEntries.h>
2021-09-01 17:59:11 +00:00
namespace DB
{
2023-02-01 13:54:03 +00:00
#define EMPTY_SETTINGS(M, ALIAS)
DECLARE_SETTINGS_TRAITS(EmptySettingsTraits, EMPTY_SETTINGS)
2021-12-27 14:41:37 +00:00
struct EmptySettings : public BaseSettings<EmptySettingsTraits> {};
2021-09-01 17:59:11 +00:00
struct ExternalDataSourceConfiguration
{
String host;
2021-09-02 13:01:26 +00:00
UInt16 port = 0;
Improvements for `parallel_distributed_insert_select` (and related) (#34728) * Add a warning if parallel_distributed_insert_select was ignored Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com> * Respect max_distributed_depth for parallel_distributed_insert_select Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com> * Print warning for non applied parallel_distributed_insert_select only for initial query Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com> * Remove Cluster::getHashOfAddresses() Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com> * Forbid parallel_distributed_insert_select for remote()/cluster() with different addresses Before it uses empty cluster name (getClusterName()) which is not correct, compare all addresses instead. Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com> * Fix max_distributed_depth check max_distributed_depth=1 must mean not more then one distributed query, not two, since max_distributed_depth=0 means no limit, and distribute_depth is 0 for the first query. Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com> * Fix INSERT INTO remote()/cluster() with parallel_distributed_insert_select Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com> * Add a test for parallel_distributed_insert_select with cluster()/remote() Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com> * Return <remote> instead of empty cluster name in Distributed engine Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com> * Make user with sharding_key and w/o in remote()/cluster() identical Before with sharding_key the user was "default", while w/o it it was empty. Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
2022-03-08 14:24:39 +00:00
String username = "default";
2021-09-01 17:59:11 +00:00
String password;
2022-08-03 19:44:08 +00:00
String quota_key;
2021-09-01 17:59:11 +00:00
String database;
String table;
2021-09-01 23:17:15 +00:00
String schema;
2021-09-01 17:59:11 +00:00
2021-09-07 11:17:25 +00:00
std::vector<std::pair<String, UInt16>> addresses; /// Failover replicas.
String addresses_expr;
2021-09-07 11:17:25 +00:00
2021-09-01 23:17:15 +00:00
String toString() const;
2021-09-15 22:45:43 +00:00
void set(const ExternalDataSourceConfiguration & conf);
2021-09-01 17:59:11 +00:00
};
2021-09-01 23:17:15 +00:00
using StorageSpecificArgs = std::vector<std::pair<String, ASTPtr>>;
2021-09-01 17:59:11 +00:00
2021-12-27 14:41:37 +00:00
struct ExternalDataSourceInfo
2021-09-22 15:10:25 +00:00
{
ExternalDataSourceConfiguration configuration;
2021-12-27 14:41:37 +00:00
SettingsChanges settings_changes;
2021-09-22 15:10:25 +00:00
};
2021-12-29 10:02:18 +00:00
using HasConfigKeyFunc = std::function<bool(const String &)>;
2022-01-10 11:00:03 +00:00
template <typename T = EmptySettingsTraits>
std::optional<ExternalDataSourceInfo> getExternalDataSourceConfiguration(
2021-12-29 10:02:18 +00:00
const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix,
2022-01-10 11:00:03 +00:00
ContextPtr context, HasConfigKeyFunc has_config_key, const BaseSettings<T> & settings = {});
2021-09-02 13:01:26 +00:00
/// Highest priority is 0, the bigger the number in map, the less the priority.
using ExternalDataSourcesConfigurationByPriority = std::map<size_t, std::vector<ExternalDataSourceConfiguration>>;
struct ExternalDataSourcesByPriority
{
String database;
String table;
String schema;
ExternalDataSourcesConfigurationByPriority replicas_configurations;
};
ExternalDataSourcesByPriority
2021-12-29 10:02:18 +00:00
getExternalDataSourceConfigurationByPriority(const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix, ContextPtr context, HasConfigKeyFunc has_config_key);
2021-09-01 17:59:11 +00:00
2021-09-07 11:17:25 +00:00
struct URLBasedDataSourceConfiguration
{
String url;
2022-05-27 17:43:34 +00:00
String endpoint;
String format = "auto";
2021-09-07 11:17:25 +00:00
String compression_method = "auto";
String structure = "auto";
2021-09-07 11:17:25 +00:00
String user;
String password;
HTTPHeaderEntries headers;
2021-10-28 12:44:12 +00:00
String http_method;
2021-09-15 22:45:43 +00:00
void set(const URLBasedDataSourceConfiguration & conf);
2021-09-07 11:17:25 +00:00
};
2021-09-22 15:10:25 +00:00
struct URLBasedDataSourceConfig
{
URLBasedDataSourceConfiguration configuration;
};
std::optional<URLBasedDataSourceConfig> getURLBasedDataSourceConfiguration(
const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix, ContextPtr context);
2021-09-01 17:59:11 +00:00
}