Initialize custom TLDs in clickhouse-local

programs/local/LocalServer.cpp

@@ -18,7 +18,9 @@
 #include <Common/Macros.h>
 #include <Common/Config/ConfigProcessor.h>
 #include <Common/ThreadStatus.h>
+#include <Common/TLDListsHolder.h>
 #include <Common/quoteString.h>
+#include <Common/randomSeed.h>
 #include <loggers/Loggers.h>
 #include <IO/ReadBufferFromFile.h>
 #include <IO/ReadBufferFromString.h>
@@ -35,7 +37,6 @@
 #include <Formats/registerFormats.h>
 #include <boost/program_options/options_description.hpp>
 #include <base/argsToConfig.h>
-#include <Common/randomSeed.h>
 #include <filesystem>
 
 namespace fs = std::filesystem;
@@ -283,6 +284,11 @@ void LocalServer::tryInitPath()
     global_context->setFlagsPath(path + "flags");
 
     global_context->setUserFilesPath(""); // user's files are everywhere
+
+    /// top_level_domains_lists
+    const std::string & top_level_domains_path = config().getString("top_level_domains_path", path + "top_level_domains/");
+    if (!top_level_domains_path.empty())
+        TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", config());
 }
 
 
@@ -671,6 +677,7 @@ void LocalServer::addOptions(OptionsDescription & options_description)
 
         ("no-system-tables", "do not attach system tables (better startup time)")
         ("path", po::value<std::string>(), "Storage path")
+        ("top_level_domains_path", po::value<std::string>(), "Path to lists with custom TLDs")
         ;
 }
 

tests/queries/0_stateless/02110_clickhouse_local_custom_tld.config.xml

@@ -0,0 +1,22 @@
+<clickhouse>
+    <profiles>
+        <default></default>
+    </profiles>
+    <users>
+        <default>
+            <password></password>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <profile>default</profile>
+            <quota>default</quota>
+        </default>
+    </users>
+    <quotas>
+        <default></default>
+    </quotas>
+
+    <top_level_domains_lists>
+        <public_suffix_list>02110_clickhouse_local_custom_tld.tld.dat</public_suffix_list>
+    </top_level_domains_lists>
+</clickhouse>

tests/queries/0_stateless/02110_clickhouse_local_custom_tld.reference

@@ -0,0 +1 @@
+biz.ss	kernel.biz.ss

tests/queries/0_stateless/02110_clickhouse_local_custom_tld.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+function clickhouse_local()
+{
+    local opts=(
+        --config "$CURDIR/$(basename "${BASH_SOURCE[0]}" .sh).config.xml"
+        --top_level_domains_path "$CURDIR"
+    )
+    $CLICKHOUSE_LOCAL "${opts[@]}" "$@"
+}
+
+# -- biz.ss is not in the default TLD list, hence:
+clickhouse_local -q "
+    select
+        cutToFirstSignificantSubdomain('foo.kernel.biz.ss'),
+        cutToFirstSignificantSubdomainCustom('foo.kernel.biz.ss', 'public_suffix_list')
+" |& grep -v -e 'Processing configuration file'

tests/queries/0_stateless/02110_clickhouse_local_custom_tld.tld.dat

@@ -0,0 +1 @@
+biz.ss