2018-03-13 23:44:23 +00:00
|
|
|
#include <Poco/Util/AbstractConfiguration.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Common/Macros.h>
|
|
|
|
#include <Common/Exception.h>
|
2019-11-05 13:05:48 +00:00
|
|
|
#include <IO/WriteHelpers.h>
|
2020-09-26 19:18:28 +00:00
|
|
|
#include <common/logger_useful.h>
|
2014-08-11 15:59:01 +00:00
|
|
|
|
2018-03-13 23:44:23 +00:00
|
|
|
|
2014-08-11 15:59:01 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2016-01-11 21:46:36 +00:00
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
extern const int SYNTAX_ERROR;
|
2016-01-11 21:46:36 +00:00
|
|
|
}
|
|
|
|
|
2020-09-26 19:18:28 +00:00
|
|
|
Macros::Macros(const Poco::Util::AbstractConfiguration & config, const String & root_key, Poco::Logger * log)
|
2014-08-11 15:59:01 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
Poco::Util::AbstractConfiguration::Keys keys;
|
|
|
|
config.keys(root_key, keys);
|
|
|
|
for (const String & key : keys)
|
|
|
|
{
|
|
|
|
macros[key] = config.getString(root_key + "." + key);
|
2020-09-26 19:18:28 +00:00
|
|
|
if (key == "database" || key == "table" || key == "uuid")
|
|
|
|
{
|
2020-09-27 11:10:45 +00:00
|
|
|
if (log)
|
|
|
|
LOG_WARNING(log,
|
|
|
|
"Config file contains '{}' macro. This macro has special meaning "
|
|
|
|
"and it's explicit definition is not recommended. Implicit unfolding for "
|
|
|
|
"'database', 'table' and 'uuid' macros will be disabled.",
|
|
|
|
key);
|
2020-09-26 19:18:28 +00:00
|
|
|
enable_special_macros = false;
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2014-08-11 15:59:01 +00:00
|
|
|
}
|
|
|
|
|
2020-07-10 09:19:32 +00:00
|
|
|
String Macros::expand(const String & s,
|
2020-09-23 18:28:59 +00:00
|
|
|
MacroExpansionInfo & info) const
|
2014-08-11 15:59:01 +00:00
|
|
|
{
|
2020-09-26 19:18:28 +00:00
|
|
|
/// Do not allow recursion if we expand only special macros, because it will be infinite recursion
|
|
|
|
assert(info.level == 0 || !info.expand_special_macros_only);
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
if (s.find('{') == String::npos)
|
|
|
|
return s;
|
2014-08-11 15:59:01 +00:00
|
|
|
|
2020-09-23 18:28:59 +00:00
|
|
|
if (info.level && s.size() > 65536)
|
2017-04-01 07:20:54 +00:00
|
|
|
throw Exception("Too long string while expanding macros", ErrorCodes::SYNTAX_ERROR);
|
2014-10-15 19:38:46 +00:00
|
|
|
|
2020-09-23 18:28:59 +00:00
|
|
|
if (info.level >= 10)
|
2017-04-01 07:20:54 +00:00
|
|
|
throw Exception("Too deep recursion while expanding macros: '" + s + "'", ErrorCodes::SYNTAX_ERROR);
|
2014-10-15 19:38:46 +00:00
|
|
|
|
2020-09-26 19:18:28 +00:00
|
|
|
/// If config file contains explicit special macro, then we do not expand it in this mode.
|
|
|
|
if (!enable_special_macros && info.expand_special_macros_only)
|
|
|
|
return s;
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
String res;
|
|
|
|
size_t pos = 0;
|
|
|
|
while (true)
|
|
|
|
{
|
|
|
|
size_t begin = s.find('{', pos);
|
2014-08-11 15:59:01 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
if (begin == String::npos)
|
|
|
|
{
|
|
|
|
res.append(s, pos, String::npos);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
res.append(s, pos, begin - pos);
|
|
|
|
}
|
2014-08-11 15:59:01 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
++begin;
|
|
|
|
size_t end = s.find('}', begin);
|
|
|
|
if (end == String::npos)
|
|
|
|
throw Exception("Unbalanced { and } in string with macros: '" + s + "'", ErrorCodes::SYNTAX_ERROR);
|
2014-08-11 15:59:01 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
String macro_name = s.substr(begin, end - begin);
|
2018-10-01 09:01:50 +00:00
|
|
|
auto it = macros.find(macro_name);
|
2014-08-11 15:59:01 +00:00
|
|
|
|
2018-10-01 09:01:50 +00:00
|
|
|
/// Prefer explicit macros over implicit.
|
2020-09-26 19:18:28 +00:00
|
|
|
if (it != macros.end() && !info.expand_special_macros_only)
|
2018-10-01 09:01:50 +00:00
|
|
|
res += it->second;
|
2020-09-26 19:18:28 +00:00
|
|
|
else if (macro_name == "database" && !info.table_id.database_name.empty())
|
|
|
|
{
|
|
|
|
res += info.table_id.database_name;
|
|
|
|
info.expanded_database = true;
|
|
|
|
}
|
|
|
|
else if (macro_name == "table" && !info.table_id.table_name.empty())
|
|
|
|
{
|
|
|
|
res += info.table_id.table_name;
|
|
|
|
info.expanded_table = true;
|
|
|
|
}
|
2020-09-10 14:50:10 +00:00
|
|
|
else if (macro_name == "uuid")
|
|
|
|
{
|
2020-09-26 19:18:28 +00:00
|
|
|
if (info.table_id.uuid == UUIDHelpers::Nil)
|
2020-09-10 14:50:10 +00:00
|
|
|
throw Exception("Macro 'uuid' and empty arguments of ReplicatedMergeTree "
|
|
|
|
"are supported only for ON CLUSTER queries with Atomic database engine",
|
|
|
|
ErrorCodes::SYNTAX_ERROR);
|
2020-09-23 18:28:59 +00:00
|
|
|
/// For ON CLUSTER queries we don't want to require all macros definitions in initiator's config.
|
|
|
|
/// However, initiator must check that for cross-replication cluster zookeeper_path does not contain {uuid} macro.
|
|
|
|
/// It becomes impossible to check if {uuid} is contained inside some unknown macro.
|
|
|
|
if (info.level)
|
|
|
|
throw Exception("Macro 'uuid' should not be inside another macro", ErrorCodes::SYNTAX_ERROR);
|
2020-09-26 19:18:28 +00:00
|
|
|
res += toString(info.table_id.uuid);
|
2020-09-23 18:28:59 +00:00
|
|
|
info.expanded_uuid = true;
|
|
|
|
}
|
2020-09-26 19:18:28 +00:00
|
|
|
else if (info.ignore_unknown || info.expand_special_macros_only)
|
2020-09-23 18:28:59 +00:00
|
|
|
{
|
2020-09-26 19:18:28 +00:00
|
|
|
if (info.expand_special_macros_only)
|
|
|
|
res += '{';
|
2020-09-23 18:28:59 +00:00
|
|
|
res += macro_name;
|
2020-09-26 19:18:28 +00:00
|
|
|
if (info.expand_special_macros_only)
|
|
|
|
res += '}';
|
2020-09-23 18:28:59 +00:00
|
|
|
info.has_unknown = true;
|
2020-09-10 14:50:10 +00:00
|
|
|
}
|
2018-09-27 10:01:10 +00:00
|
|
|
else
|
2019-11-05 13:05:48 +00:00
|
|
|
throw Exception("No macro '" + macro_name +
|
2020-07-10 09:19:32 +00:00
|
|
|
"' in config while processing substitutions in '" + s + "' at '"
|
|
|
|
+ toString(begin) + "' or macro is not supported here", ErrorCodes::SYNTAX_ERROR);
|
2014-08-11 15:59:01 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
pos = end + 1;
|
|
|
|
}
|
2014-08-11 15:59:01 +00:00
|
|
|
|
2020-09-23 18:28:59 +00:00
|
|
|
++info.level;
|
2020-09-26 19:18:28 +00:00
|
|
|
if (info.expand_special_macros_only)
|
|
|
|
return res;
|
|
|
|
|
2020-09-23 18:28:59 +00:00
|
|
|
return expand(res, info);
|
2014-08-11 15:59:01 +00:00
|
|
|
}
|
|
|
|
|
2019-10-09 01:14:57 +00:00
|
|
|
String Macros::getValue(const String & key) const
|
|
|
|
{
|
|
|
|
if (auto it = macros.find(key); it != macros.end())
|
|
|
|
return it->second;
|
|
|
|
throw Exception("No macro " + key + " in config", ErrorCodes::SYNTAX_ERROR);
|
|
|
|
}
|
|
|
|
|
2020-09-23 18:28:59 +00:00
|
|
|
|
|
|
|
String Macros::expand(const String & s) const
|
|
|
|
{
|
|
|
|
MacroExpansionInfo info;
|
|
|
|
return expand(s, info);
|
|
|
|
}
|
|
|
|
|
2020-07-10 09:19:32 +00:00
|
|
|
String Macros::expand(const String & s, const StorageID & table_id, bool allow_uuid) const
|
2018-10-01 09:01:50 +00:00
|
|
|
{
|
2020-09-23 18:28:59 +00:00
|
|
|
MacroExpansionInfo info;
|
2020-09-26 19:18:28 +00:00
|
|
|
info.table_id = table_id;
|
|
|
|
if (!allow_uuid)
|
|
|
|
info.table_id.uuid = UUIDHelpers::Nil;
|
2020-09-23 18:28:59 +00:00
|
|
|
return expand(s, info);
|
2018-10-01 09:01:50 +00:00
|
|
|
}
|
|
|
|
|
2018-04-20 16:08:27 +00:00
|
|
|
Names Macros::expand(const Names & source_names, size_t level) const
|
2018-04-20 13:48:42 +00:00
|
|
|
{
|
2018-04-20 16:08:27 +00:00
|
|
|
Names result_names;
|
|
|
|
result_names.reserve(source_names.size());
|
2018-04-20 13:48:42 +00:00
|
|
|
|
2020-09-23 18:28:59 +00:00
|
|
|
MacroExpansionInfo info;
|
2018-04-20 16:08:27 +00:00
|
|
|
for (const String & name : source_names)
|
2020-09-23 18:28:59 +00:00
|
|
|
{
|
|
|
|
info.level = level;
|
|
|
|
result_names.push_back(expand(name, info));
|
|
|
|
}
|
2018-05-07 02:01:11 +00:00
|
|
|
|
2018-04-20 16:08:27 +00:00
|
|
|
return result_names;
|
2018-04-20 13:48:42 +00:00
|
|
|
}
|
2014-08-11 15:59:01 +00:00
|
|
|
}
|