thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 09:32:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:ClickHouse/ClickHouse into feature-server-iface-metrics

Browse Source
This commit is contained in:
Alexey Milovidov 2023-12-30 18:38:45 +01:00
commit 68cc3eaa96
18 changed files with 12 additions and 292 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
docs/en/sql-reference/functions/ip-address-functions.md
View File

@ -501,41 +501,3 @@ Result:
│ 0 │
└────────────────────────────────────────────────────────────────────┘
```
## reverseDNSQuery
Performs a reverse DNS query to get the PTR records associated with the IP address.
**Syntax**
``` sql
reverseDNSQuery(address)
```
This function performs reverse DNS resolutions on both IPv4 and IPv6.
**Arguments**
- `address` — An IPv4 or IPv6 address. [String](../../sql-reference/data-types/string.md).
**Returned value**
- Associated domains (PTR records).
Type: Type: [Array(String)](../../sql-reference/data-types/array.md).
**Example**
Query:
``` sql
SELECT reverseDNSQuery('192.168.0.2');
```
Result:
``` text
┌─reverseDNSQuery('192.168.0.2')────────────┐
│ ['test2.example.com','test3.example.com'] │
└───────────────────────────────────────────┘
```

4
programs/local/LocalServer.cpp
View File

@ -744,7 +744,7 @@ void LocalServer::processConfig()
LOG_DEBUG(log, "Loading metadata from {}", path);
auto startup_system_tasks = loadMetadataSystem(global_context);
attachSystemTablesLocal(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE));
attachSystemTablesServer(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE), false);
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA));
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
waitLoad(TablesLoaderForegroundPoolId, startup_system_tasks);
@ -763,7 +763,7 @@ void LocalServer::processConfig()
}
else if (!config().has("no-system-tables"))
{
attachSystemTablesLocal(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE));
attachSystemTablesServer(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE), false);
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA));
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
}

118
src/Functions/reverseDNSQuery.cpp
View File

@ -1,118 +0,0 @@
#include <Columns/ColumnString.h>
#include <Columns/ColumnArray.h>
#include <Functions/FunctionFactory.h>
#include <Functions/IFunction.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeArray.h>
#include <Common/DNSResolver.h>
#include <Poco/Net/IPAddress.h>
#include <Interpreters/Context.h>
#include <Poco/Util/AbstractConfiguration.h>
namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int BAD_ARGUMENTS;
extern const int FUNCTION_NOT_ALLOWED;
}
class ReverseDNSQuery : public IFunction
{
public:
static constexpr auto name = "reverseDNSQuery";
static constexpr auto allow_function_config_name = "allow_reverse_dns_query_function";
static FunctionPtr create(ContextPtr)
{
return std::make_shared<ReverseDNSQuery>();
}
String getName() const override
{
return name;
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & data_type, size_t input_rows_count) const override
{
if (!Context::getGlobalContextInstance()->getConfigRef().getBool(allow_function_config_name, false))
{
throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Function {} is not allowed because {} is not set", name, allow_function_config_name);
}
if (arguments.empty())
{
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires at least one argument", name);
}
auto res_type = getReturnTypeImpl({data_type});
if (input_rows_count == 0u)
{
return res_type->createColumnConstWithDefaultValue(input_rows_count);
}
if (!isString(arguments[0].type))
{
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function {} requires the input column to be of type String", name);
}
auto input_column = arguments[0].column;
auto ip_address = Poco::Net::IPAddress(input_column->getDataAt(0).toString());
auto ptr_records = DNSResolver::instance().reverseResolve(ip_address);
if (ptr_records.empty())
return res_type->createColumnConstWithDefaultValue(input_rows_count);
Array res;
for (const auto & ptr_record : ptr_records)
{
res.push_back(ptr_record);
}
return res_type->createColumnConst(input_rows_count, res);
}
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override
{
return false;
}
size_t getNumberOfArguments() const override
{
return 1u;
}
DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override
{
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
}
};
REGISTER_FUNCTION(ReverseDNSQuery)
{
factory.registerFunction<ReverseDNSQuery>(
FunctionDocumentation{
.description = R"(Performs a reverse DNS query to get the PTR records associated with the IP address)",
.syntax = "reverseDNSQuery(address)",
.arguments = {{"address", "An IPv4 or IPv6 address. [String](../../sql-reference/data-types/string.md)"}},
.returned_value = "Associated domains (PTR records). [String](../../sql-reference/data-types/string.md).",
.examples = {{"",
"SELECT reverseDNSQuery('192.168.0.2');",
R"(
┌─reverseDNSQuery('192.168.0.2')────────────┐
│ ['test2.example.com','test3.example.com'] │
└───────────────────────────────────────────┘
)"}}
}
);
}
}

7
src/Storages/System/attachSystemTables.cpp
View File

@ -109,7 +109,7 @@
namespace DB
{
void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database)
void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, bool has_zookeeper)
{
attach<StorageSystemOne>(context, system_database, "one");
attach<StorageSystemNumbers>(context, system_database, "numbers", false);
@ -170,11 +170,6 @@ void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database)
#if USE_ROCKSDB
attach<StorageSystemRocksDB>(context, system_database, "rocksdb");
#endif
}
void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, bool has_zookeeper)
{
attachSystemTablesLocal(context, system_database);
attach<StorageSystemParts>(context, system_database, "parts");
attach<StorageSystemProjectionParts>(context, system_database, "projection_parts");

1
src/Storages/System/attachSystemTables.h
View File

@ -10,7 +10,6 @@ class AsynchronousMetrics;
class IDatabase;
void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, bool has_zookeeper);
void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database);
void attachSystemTablesAsync(ContextPtr context, IDatabase & system_database, AsynchronousMetrics & async_metrics);
}

3
tests/config/config.d/reverse_dns_query_function.xml
View File

@ -1,3 +0,0 @@
<clickhouse>
<allow_reverse_dns_query_function>1</allow_reverse_dns_query_function>
</clickhouse>

0
tests/integration/test_reverse_dns_query/__init__.py
View File

3
tests/integration/test_reverse_dns_query/configs/config.xml
View File

@ -1,3 +0,0 @@
<clickhouse>
<disable_internal_dns_cache>1</disable_internal_dns_cache>
</clickhouse>

5
tests/integration/test_reverse_dns_query/configs/listen_host.xml
View File

@ -1,5 +0,0 @@
<clickhouse>
<listen_host>::</listen_host>
<listen_host>0.0.0.0</listen_host>
<listen_try>1</listen_try>
</clickhouse>

3
tests/integration/test_reverse_dns_query/configs/reverse_dns_function.xml
View File

@ -1,3 +0,0 @@
<clickhouse>
<allow_reverse_dns_query_function>1</allow_reverse_dns_query_function>
</clickhouse>

8
tests/integration/test_reverse_dns_query/coredns_config/Corefile
View File

@ -1,8 +0,0 @@
. {
hosts /example.com {
reload "20ms"
fallthrough
}
forward . 127.0.0.11
log
}

1
tests/integration/test_reverse_dns_query/coredns_config/example.com
View File

@ -1 +0,0 @@
filled in runtime, but needs to exist in order to be volume mapped in docker

74
tests/integration/test_reverse_dns_query/test.py
View File

@ -1,74 +0,0 @@
import pytest
import socket
from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check
from time import sleep
import os
DOCKER_COMPOSE_PATH = get_docker_compose_path()
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
cluster = ClickHouseCluster(__file__)
ch_server = cluster.add_instance(
"clickhouse-server",
with_coredns=True,
main_configs=[
"configs/config.xml",
"configs/reverse_dns_function.xml",
"configs/listen_host.xml",
],
)
@pytest.fixture(scope="module")
def started_cluster():
global cluster
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def check_ptr_record(ip, hostname):
try:
host, aliaslist, ipaddrlist = socket.gethostbyaddr(ip)
if hostname.lower() == host.lower():
return True
except socket.herror:
pass
return False
def setup_dns_server(ip):
domains_string = "test.example.com"
example_file_path = f'{ch_server.env_variables["COREDNS_CONFIG_DIR"]}/example.com'
run_and_check(f"echo '{ip} {domains_string}' > {example_file_path}", shell=True)
# DNS server takes time to reload the configuration.
for try_num in range(10):
if all(check_ptr_record(ip, host) for host in domains_string.split()):
break
sleep(1)
def setup_ch_server(dns_server_ip):
ch_server.exec_in_container(
(["bash", "-c", f"echo 'nameserver {dns_server_ip}' > /etc/resolv.conf"])
)
ch_server.exec_in_container(
(["bash", "-c", "echo 'options ndots:0' >> /etc/resolv.conf"])
)
ch_server.query("SYSTEM DROP DNS CACHE")
def test_reverse_dns_query(started_cluster):
dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
random_ipv6 = "4ae8:fa0f:ee1d:68c5:0b76:1b79:7ae6:1549" # https://commentpicker.com/ip-address-generator.php
setup_dns_server(random_ipv6)
setup_ch_server(dns_server_ip)
for _ in range(0, 200):
response = ch_server.query(f"select reverseDNSQuery('{random_ipv6}')")
assert response == "['test.example.com']\n"

14
tests/queries/0_stateless/02483_test_reverse_dns_resolution.reference
View File

@ -1,14 +0,0 @@
-- { echoOn }
-- Expect dns.google on both queries
select reverseDNSQuery('8.8.8.8');
['dns.google']
select reverseDNSQuery('2001:4860:4860::8888');
['dns.google']
-- Expect empty response
select reverseDNSQuery('');
[]
-- Expect error, invalid column type
select reverseDNSQuery(1); -- {serverError 36}
-- Expect error, wrong number of arguments
select reverseDNSQuery(); -- {serverError 42}
select reverseDNSQuery(1, 2); -- {serverError 42}

14
tests/queries/0_stateless/02483_test_reverse_dns_resolution.sql
View File

@ -1,14 +0,0 @@
-- { echoOn }
-- Expect dns.google on both queries
select reverseDNSQuery('8.8.8.8');
select reverseDNSQuery('2001:4860:4860::8888');
-- Expect empty response
select reverseDNSQuery('');
-- Expect error, invalid column type
select reverseDNSQuery(1); -- {serverError 36}
-- Expect error, wrong number of arguments
select reverseDNSQuery(); -- {serverError 42}
select reverseDNSQuery(1, 2); -- {serverError 42}

1
tests/queries/0_stateless/02956_clickhouse_local_system_parts.reference Normal file
View File

@ -0,0 +1 @@
test all_1_1_0 1

8
tests/queries/0_stateless/02956_clickhouse_local_system_parts.sh Executable file
View File

@ -0,0 +1,8 @@
#!/usr/bin/env bash
# Tags: no-fasttest
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_LOCAL --multiquery "CREATE TABLE test (x UInt8) ENGINE = MergeTree ORDER BY (); INSERT INTO test SELECT 1; SELECT table, name, rows FROM system.parts WHERE database = currentDatabase();"

2
utils/check-style/aspell-ignore/en/aspell-dict.txt
View File

@ -2198,8 +2198,6 @@ retentions
rethrow
retransmit
retriable
retuned
reverseDNSQuery
reverseUTF
rightPad
rightPadUTF