2016-10-25 12:14:27 +00:00
# include "LocalServer.h"
2017-03-14 18:39:06 +00:00
2023-02-25 02:20:47 +00:00
# include <sys/resource.h>
2024-07-25 14:55:23 +00:00
# include <Common/Config/getLocalConfigPath.h>
2023-02-25 02:20:47 +00:00
# include <Common/logger_useful.h>
2023-06-29 18:33:48 +00:00
# include <Common/formatReadable.h>
2024-04-27 18:01:54 +00:00
# include <Core/UUID.h>
2023-06-29 18:33:48 +00:00
# include <base/getMemoryAmount.h>
2016-10-25 12:14:27 +00:00
# include <Poco/Util/XMLConfiguration.h>
2016-12-13 18:51:19 +00:00
# include <Poco/String.h>
2018-01-15 14:58:21 +00:00
# include <Poco/Logger.h>
# include <Poco/NullChannel.h>
2021-10-30 12:28:50 +00:00
# include <Poco/SimpleFileChannel.h>
2023-12-30 03:35:00 +00:00
# include <Databases/registerDatabases.h>
2023-06-20 22:29:46 +00:00
# include <Databases/DatabaseFilesystem.h>
2018-03-23 20:46:43 +00:00
# include <Databases/DatabaseMemory.h>
2024-08-08 08:07:39 +00:00
# include <Databases/DatabaseAtomic.h>
2023-06-20 22:29:46 +00:00
# include <Databases/DatabasesOverlay.h>
2017-04-01 09:19:00 +00:00
# include <Storages/System/attachSystemTables.h>
2021-09-09 09:37:51 +00:00
# include <Storages/System/attachInformationSchemaTables.h>
2022-10-19 10:38:20 +00:00
# include <Interpreters/DatabaseCatalog.h>
# include <Interpreters/JIT/CompiledExpressionCache.h>
2017-04-01 09:19:00 +00:00
# include <Interpreters/ProcessList.h>
# include <Interpreters/loadMetadata.h>
2024-01-09 06:33:48 +00:00
# include <Interpreters/registerInterpreters.h>
2022-03-10 04:35:01 +00:00
# include <Access/AccessControl.h>
2023-11-06 13:08:27 +00:00
# include <Common/PoolId.h>
2017-04-01 09:19:00 +00:00
# include <Common/Exception.h>
# include <Common/Macros.h>
2018-02-28 20:34:25 +00:00
# include <Common/Config/ConfigProcessor.h>
2019-01-28 11:18:00 +00:00
# include <Common/ThreadStatus.h>
2021-10-25 18:12:45 +00:00
# include <Common/TLDListsHolder.h>
2019-10-08 18:42:22 +00:00
# include <Common/quoteString.h>
2023-04-13 16:53:16 +00:00
# include <Common/ThreadPool.h>
2024-06-26 00:29:04 +00:00
# include <Common/CurrentMetrics.h>
2023-09-19 21:39:59 +00:00
# include <Loggers/OwnFormattingChannel.h>
# include <Loggers/OwnPatternFormatter.h>
2020-10-14 08:50:36 +00:00
# include <IO/ReadBufferFromFile.h>
2017-04-01 09:19:00 +00:00
# include <IO/ReadBufferFromString.h>
2021-09-04 18:19:01 +00:00
# include <IO/UseSSL.h>
2023-05-03 16:56:45 +00:00
# include <IO/SharedThreadPools.h>
2022-01-10 19:01:41 +00:00
# include <Parsers/ASTInsertQuery.h>
2022-04-27 15:05:45 +00:00
# include <Common/ErrorHandlers.h>
2023-12-12 14:03:49 +00:00
# include <Functions/UserDefined/IUserDefinedSQLObjectsStorage.h>
2017-04-21 17:47:27 +00:00
# include <Functions/registerFunctions.h>
2017-05-05 20:39:25 +00:00
# include <AggregateFunctions/registerAggregateFunctions.h>
2017-06-10 09:04:31 +00:00
# include <TableFunctions/registerTableFunctions.h>
2017-12-30 00:36:06 +00:00
# include <Storages/registerStorages.h>
2018-11-28 11:37:12 +00:00
# include <Dictionaries/registerDictionaries.h>
2019-11-27 09:39:44 +00:00
# include <Disks/registerDisks.h>
2020-10-29 03:39:43 +00:00
# include <Formats/registerFormats.h>
2018-04-20 19:31:19 +00:00
# include <boost/program_options/options_description.hpp>
2021-10-02 07:13:14 +00:00
# include <base/argsToConfig.h>
2020-06-24 19:03:28 +00:00
# include <filesystem>
2023-06-20 22:29:46 +00:00
# include "config.h"
2023-01-30 19:00:48 +00:00
# if USE_AZURE_BLOB_STORAGE
# include <azure / storage / common / internal / xml_wrapper.hpp>
# endif
2024-06-26 00:29:04 +00:00
2021-05-16 22:06:09 +00:00
namespace fs = std : : filesystem ;
2016-10-25 12:14:27 +00:00
2024-06-26 00:29:04 +00:00
namespace CurrentMetrics
{
extern const Metric MemoryTracking ;
}
2021-07-11 23:17:14 +00:00
2016-10-25 12:14:27 +00:00
namespace DB
{
2024-11-07 19:55:04 +00:00
2024-09-18 12:20:53 +00:00
namespace Setting
{
extern const SettingsBool allow_introspection_functions ;
2024-11-07 19:55:04 +00:00
extern const SettingsBool implicit_select ;
2024-09-18 12:20:53 +00:00
extern const SettingsLocalFSReadMethod storage_file_read_method ;
}
2016-10-25 12:14:27 +00:00
2024-10-16 19:13:26 +00:00
namespace ServerSetting
{
2024-11-15 17:21:53 +00:00
extern const ServerSettingsUInt32 allowed_feature_tier ;
2024-10-16 19:13:26 +00:00
extern const ServerSettingsDouble cache_size_to_ram_max_ratio ;
extern const ServerSettingsUInt64 compiled_expression_cache_elements_size ;
extern const ServerSettingsUInt64 compiled_expression_cache_size ;
extern const ServerSettingsUInt64 database_catalog_drop_table_concurrency ;
extern const ServerSettingsString default_database ;
extern const ServerSettingsString index_mark_cache_policy ;
extern const ServerSettingsUInt64 index_mark_cache_size ;
extern const ServerSettingsDouble index_mark_cache_size_ratio ;
extern const ServerSettingsString index_uncompressed_cache_policy ;
extern const ServerSettingsUInt64 index_uncompressed_cache_size ;
extern const ServerSettingsDouble index_uncompressed_cache_size_ratio ;
extern const ServerSettingsUInt64 io_thread_pool_queue_size ;
extern const ServerSettingsString mark_cache_policy ;
extern const ServerSettingsUInt64 mark_cache_size ;
extern const ServerSettingsDouble mark_cache_size_ratio ;
extern const ServerSettingsUInt64 max_active_parts_loading_thread_pool_size ;
extern const ServerSettingsUInt64 max_io_thread_pool_free_size ;
extern const ServerSettingsUInt64 max_io_thread_pool_size ;
extern const ServerSettingsUInt64 max_outdated_parts_loading_thread_pool_size ;
extern const ServerSettingsUInt64 max_parts_cleaning_thread_pool_size ;
extern const ServerSettingsUInt64 max_server_memory_usage ;
extern const ServerSettingsDouble max_server_memory_usage_to_ram_ratio ;
extern const ServerSettingsUInt64 max_thread_pool_free_size ;
extern const ServerSettingsUInt64 max_thread_pool_size ;
extern const ServerSettingsUInt64 max_unexpected_parts_loading_thread_pool_size ;
extern const ServerSettingsUInt64 mmap_cache_size ;
extern const ServerSettingsBool show_addresses_in_stack_traces ;
extern const ServerSettingsUInt64 thread_pool_queue_size ;
extern const ServerSettingsString uncompressed_cache_policy ;
extern const ServerSettingsUInt64 uncompressed_cache_size ;
extern const ServerSettingsDouble uncompressed_cache_size_ratio ;
2024-11-19 16:15:12 +00:00
extern const ServerSettingsString primary_index_cache_policy ;
extern const ServerSettingsUInt64 primary_index_cache_size ;
extern const ServerSettingsDouble primary_index_cache_size_ratio ;
2024-10-16 19:13:26 +00:00
}
2016-11-11 17:01:02 +00:00
namespace ErrorCodes
{
2020-06-24 22:07:01 +00:00
extern const int BAD_ARGUMENTS ;
2017-04-01 07:20:54 +00:00
extern const int CANNOT_LOAD_CONFIG ;
2020-10-23 09:53:35 +00:00
extern const int FILE_ALREADY_EXISTS ;
2016-11-11 17:01:02 +00:00
}
2023-06-24 19:41:33 +00:00
void applySettingsOverridesForLocal ( ContextMutablePtr context )
{
2024-07-15 12:28:12 +00:00
Settings settings = context - > getSettingsCopy ( ) ;
2023-06-24 19:41:33 +00:00
2024-09-18 12:20:53 +00:00
settings [ Setting : : allow_introspection_functions ] = true ;
settings [ Setting : : storage_file_read_method ] = LocalFSReadMethod : : mmap ;
2024-11-07 19:55:04 +00:00
settings [ Setting : : implicit_select ] = true ;
2023-06-24 19:41:33 +00:00
context - > setSettings ( settings ) ;
}
2016-11-11 17:01:02 +00:00
2024-06-25 14:23:37 +00:00
Poco : : Util : : LayeredConfiguration & LocalServer : : getClientConfiguration ( )
{
return config ( ) ;
}
2021-10-04 15:23:08 +00:00
void LocalServer : : processError ( const String & ) const
2021-08-21 10:55:54 +00:00
{
2021-09-05 09:27:27 +00:00
if ( ignore_error )
2021-08-21 15:29:28 +00:00
return ;
2021-09-11 11:34:22 +00:00
if ( is_interactive )
2021-08-21 10:55:54 +00:00
{
2021-10-04 15:23:08 +00:00
String message ;
2021-09-11 11:34:22 +00:00
if ( server_exception )
{
2021-10-04 15:23:08 +00:00
message = getExceptionMessage ( * server_exception , print_stack_trace , true ) ;
2021-09-11 11:34:22 +00:00
}
2021-10-04 15:23:08 +00:00
else if ( client_exception )
2021-09-11 11:34:22 +00:00
{
2021-10-04 15:23:08 +00:00
message = client_exception - > message ( ) ;
2021-09-11 11:34:22 +00:00
}
2021-10-04 15:23:08 +00:00
fmt : : print ( stderr , " Received exception: \n {} \n " , message ) ;
fmt : : print ( stderr , " \n " ) ;
2021-08-21 10:55:54 +00:00
}
2021-09-11 11:34:22 +00:00
else
2021-08-21 10:55:54 +00:00
{
2021-09-11 11:34:22 +00:00
if ( server_exception )
server_exception - > rethrow ( ) ;
if ( client_exception )
client_exception - > rethrow ( ) ;
2021-08-21 10:55:54 +00:00
}
}
2016-11-11 17:01:02 +00:00
2016-10-25 12:14:27 +00:00
void LocalServer : : initialize ( Poco : : Util : : Application & self )
{
2017-04-01 07:20:54 +00:00
Poco : : Util : : Application : : initialize ( self ) ;
2018-01-15 14:58:21 +00:00
2024-07-25 14:55:23 +00:00
const char * home_path_cstr = getenv ( " HOME " ) ; // NOLINT(concurrency-mt-unsafe)
if ( home_path_cstr )
home_path = home_path_cstr ;
2019-06-14 14:00:37 +00:00
/// Load config files if exists
2024-07-25 14:55:23 +00:00
std : : string config_path ;
if ( getClientConfiguration ( ) . has ( " config-file " ) )
config_path = getClientConfiguration ( ) . getString ( " config-file " ) ;
else if ( config_path . empty ( ) & & fs : : exists ( " config.xml " ) )
config_path = " config.xml " ;
else if ( config_path . empty ( ) )
config_path = getLocalConfigPath ( home_path ) . value_or ( " " ) ;
if ( fs : : exists ( config_path ) )
2019-06-14 14:00:37 +00:00
{
2024-08-11 11:26:45 +00:00
ConfigProcessor config_processor ( config_path ) ;
2024-04-07 09:51:45 +00:00
ConfigProcessor : : setConfigPath ( fs : : path ( config_path ) . parent_path ( ) ) ;
2019-06-14 14:00:37 +00:00
auto loaded_config = config_processor . loadConfig ( ) ;
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . add ( loaded_config . configuration . duplicate ( ) , PRIO_DEFAULT , false ) ;
2019-06-14 14:00:37 +00:00
}
2022-04-29 07:41:10 +00:00
2024-06-26 00:29:04 +00:00
server_settings . loadSettingsFromConfig ( config ( ) ) ;
2022-04-29 07:41:10 +00:00
GlobalThreadPool : : initialize (
2024-10-16 19:13:26 +00:00
server_settings [ ServerSetting : : max_thread_pool_size ] ,
server_settings [ ServerSetting : : max_thread_pool_free_size ] ,
server_settings [ ServerSetting : : thread_pool_queue_size ] ) ;
2022-04-29 07:41:10 +00:00
2023-01-30 19:00:48 +00:00
# if USE_AZURE_BLOB_STORAGE
/// See the explanation near the same line in Server.cpp
GlobalThreadPool : : instance ( ) . addOnDestroyCallback ( [ ]
{
Azure : : Storage : : _internal : : XmlGlobalDeinitialize ( ) ;
} ) ;
# endif
2023-06-06 12:42:56 +00:00
getIOThreadPool ( ) . initialize (
2024-10-16 19:13:26 +00:00
server_settings [ ServerSetting : : max_io_thread_pool_size ] ,
server_settings [ ServerSetting : : max_io_thread_pool_free_size ] ,
server_settings [ ServerSetting : : io_thread_pool_queue_size ] ) ;
2023-05-03 16:56:45 +00:00
2024-10-16 19:13:26 +00:00
const size_t active_parts_loading_threads = server_settings [ ServerSetting : : max_active_parts_loading_thread_pool_size ] ;
2023-06-06 12:42:56 +00:00
getActivePartsLoadingThreadPool ( ) . initialize (
active_parts_loading_threads ,
0 , // We don't need any threads one all the parts will be loaded
active_parts_loading_threads ) ;
2024-10-16 19:13:26 +00:00
const size_t outdated_parts_loading_threads = server_settings [ ServerSetting : : max_outdated_parts_loading_thread_pool_size ] ;
2023-06-06 12:42:56 +00:00
getOutdatedPartsLoadingThreadPool ( ) . initialize (
outdated_parts_loading_threads ,
2023-05-03 16:56:45 +00:00
0 , // We don't need any threads one all the parts will be loaded
2023-06-06 12:42:56 +00:00
outdated_parts_loading_threads ) ;
getOutdatedPartsLoadingThreadPool ( ) . setMaxTurboThreads ( active_parts_loading_threads ) ;
2024-10-16 19:13:26 +00:00
const size_t unexpected_parts_loading_threads = server_settings [ ServerSetting : : max_unexpected_parts_loading_thread_pool_size ] ;
2024-05-08 13:04:16 +00:00
getUnexpectedPartsLoadingThreadPool ( ) . initialize (
unexpected_parts_loading_threads ,
0 , // We don't need any threads one all the parts will be loaded
unexpected_parts_loading_threads ) ;
getUnexpectedPartsLoadingThreadPool ( ) . setMaxTurboThreads ( active_parts_loading_threads ) ;
2024-10-16 19:13:26 +00:00
const size_t cleanup_threads = server_settings [ ServerSetting : : max_parts_cleaning_thread_pool_size ] ;
2023-06-06 12:42:56 +00:00
getPartsCleaningThreadPool ( ) . initialize (
cleanup_threads ,
0 , // We don't need any threads one all the parts will be deleted
cleanup_threads ) ;
2024-07-26 17:08:07 +00:00
getDatabaseCatalogDropTablesThreadPool ( ) . initialize (
2024-10-16 19:13:26 +00:00
server_settings [ ServerSetting : : database_catalog_drop_table_concurrency ] ,
2024-07-26 17:08:07 +00:00
0 , // We don't need any threads if there are no DROP queries.
2024-10-16 19:13:26 +00:00
server_settings [ ServerSetting : : database_catalog_drop_table_concurrency ] ) ;
2016-10-25 12:14:27 +00:00
}
2016-10-31 14:31:26 +00:00
2021-09-09 12:43:12 +00:00
static DatabasePtr createMemoryDatabaseIfNotExists ( ContextPtr context , const String & database_name )
2018-04-20 15:32:40 +00:00
{
2021-09-09 12:43:12 +00:00
DatabasePtr system_database = DatabaseCatalog : : instance ( ) . tryGetDatabase ( database_name ) ;
if ( ! system_database )
{
/// TODO: add attachTableDelayed into DatabaseMemory to speedup loading
system_database = std : : make_shared < DatabaseMemory > ( database_name , context ) ;
DatabaseCatalog : : instance ( ) . attachDatabase ( database_name , system_database ) ;
}
return system_database ;
2016-10-31 14:31:26 +00:00
}
2024-08-08 08:07:39 +00:00
static DatabasePtr createClickHouseLocalDatabaseOverlay ( const String & name_ , ContextPtr context )
2023-06-20 22:29:46 +00:00
{
2024-08-08 08:07:39 +00:00
auto overlay = std : : make_shared < DatabasesOverlay > ( name_ , context ) ;
overlay - > registerNextDatabase ( std : : make_shared < DatabaseAtomic > ( name_ , fs : : weakly_canonical ( context - > getPath ( ) ) , UUIDHelpers : : generateV4 ( ) , context ) ) ;
overlay - > registerNextDatabase ( std : : make_shared < DatabaseFilesystem > ( name_ , " " , context ) ) ;
return overlay ;
2023-06-20 22:29:46 +00:00
}
2021-09-09 12:43:12 +00:00
2016-12-13 18:51:19 +00:00
/// If path is specified and not empty, will try to setup server environment and load existing metadata
void LocalServer : : tryInitPath ( )
{
2020-06-24 19:03:28 +00:00
std : : string path ;
2024-06-25 14:23:37 +00:00
if ( getClientConfiguration ( ) . has ( " path " ) )
2020-06-24 19:03:28 +00:00
{
// User-supplied path.
2024-06-25 14:23:37 +00:00
path = getClientConfiguration ( ) . getString ( " path " ) ;
2020-06-24 19:03:28 +00:00
Poco : : trimInPlace ( path ) ;
2016-12-13 18:51:19 +00:00
2020-06-24 19:03:28 +00:00
if ( path . empty ( ) )
{
throw Exception ( ErrorCodes : : BAD_ARGUMENTS ,
2020-08-08 01:21:04 +00:00
" Cannot work with empty storage path that is explicitly specified "
2020-06-24 22:07:01 +00:00
" by the --path option. Please check the program options and "
2020-06-24 19:03:28 +00:00
" correct the --path. " ) ;
}
}
else
2018-04-20 15:32:40 +00:00
{
2020-10-26 07:59:15 +00:00
// The path is not provided explicitly - use a unique path in the system temporary directory
2024-04-30 02:45:05 +00:00
// (or in the current dir if a temporary doesn't exist)
2024-01-23 17:04:50 +00:00
LoggerRawPtr log = & logger ( ) ;
2020-10-23 09:53:35 +00:00
std : : filesystem : : path parent_folder ;
std : : filesystem : : path default_path ;
2018-04-20 15:32:40 +00:00
2020-10-23 09:53:35 +00:00
try
2020-06-24 19:03:28 +00:00
{
2020-10-23 09:53:35 +00:00
// try to guess a tmp folder name, and check if it's a directory (throw exception otherwise)
parent_folder = std : : filesystem : : temp_directory_path ( ) ;
}
2023-02-25 02:20:47 +00:00
catch ( const fs : : filesystem_error & e )
2020-10-23 09:53:35 +00:00
{
2023-02-25 02:20:47 +00:00
// The tmp folder doesn't exist? Is it a misconfiguration? Or chroot?
2020-10-23 09:53:35 +00:00
LOG_DEBUG ( log , " Can not get temporary folder: {} " , e . what ( ) ) ;
parent_folder = std : : filesystem : : current_path ( ) ;
std : : filesystem : : is_directory ( parent_folder ) ; // that will throw an exception if it's not a directory
LOG_DEBUG ( log , " Will create working directory inside current directory: {} " , parent_folder . string ( ) ) ;
}
2020-10-26 07:59:15 +00:00
/// we can have another clickhouse-local running simultaneously, even with the same PID (for ex. - several dockers mounting the same folder)
2020-10-23 09:53:35 +00:00
/// or it can be some leftovers from other clickhouse-local runs
/// as we can't accurately distinguish those situations we don't touch any existent folders
/// we just try to pick some free name for our working folder
2024-04-27 18:01:54 +00:00
default_path = parent_folder / fmt : : format ( " clickhouse-local-{} " , UUIDHelpers : : generateV4 ( ) ) ;
2020-10-23 09:53:35 +00:00
2024-04-27 18:01:54 +00:00
if ( fs : : exists ( default_path ) )
throw Exception ( ErrorCodes : : FILE_ALREADY_EXISTS , " Unsuccessful attempt to set up the working directory: {} already exists. " , default_path . string ( ) ) ;
2020-06-24 19:03:28 +00:00
2024-04-27 18:01:54 +00:00
/// The directory can be created lazily during the runtime.
2020-06-24 19:03:28 +00:00
temporary_directory_to_delete = default_path ;
path = default_path . string ( ) ;
2020-10-23 09:53:35 +00:00
LOG_DEBUG ( log , " Working directory created: {} " , path ) ;
2018-04-20 15:32:40 +00:00
}
2016-12-13 18:51:19 +00:00
2024-04-27 16:23:14 +00:00
global_context - > setPath ( fs : : path ( path ) / " " ) ;
2020-10-20 15:57:53 +00:00
2024-04-27 16:47:03 +00:00
global_context - > setTemporaryStoragePath ( fs : : path ( path ) / " tmp " / " " , 0 ) ;
global_context - > setFlagsPath ( fs : : path ( path ) / " flags " / " " ) ;
2020-10-20 15:57:53 +00:00
2024-04-27 16:23:14 +00:00
global_context - > setUserFilesPath ( " " ) ; /// user's files are everywhere
2021-10-25 18:12:45 +00:00
2024-08-11 07:30:30 +00:00
std : : string user_scripts_path = getClientConfiguration ( ) . getString ( " user_scripts_path " , fs : : path ( path ) / " user_scripts " / " " ) ;
2023-07-27 05:06:16 +00:00
global_context - > setUserScriptsPath ( user_scripts_path ) ;
2024-08-11 07:30:30 +00:00
/// Set path for filesystem caches
String filesystem_caches_path ( getClientConfiguration ( ) . getString ( " filesystem_caches_path " , fs : : path ( path ) / " cache " / " " ) ) ;
if ( ! filesystem_caches_path . empty ( ) )
global_context - > setFilesystemCachesPath ( filesystem_caches_path ) ;
2021-10-25 18:12:45 +00:00
/// top_level_domains_lists
2024-06-25 14:23:37 +00:00
const std : : string & top_level_domains_path = getClientConfiguration ( ) . getString ( " top_level_domains_path " , fs : : path ( path ) / " top_level_domains/ " ) ;
2021-10-25 18:12:45 +00:00
if ( ! top_level_domains_path . empty ( ) )
2024-06-25 14:23:37 +00:00
TLDListsHolder : : getInstance ( ) . parseConfig ( fs : : path ( top_level_domains_path ) / " " , getClientConfiguration ( ) ) ;
2016-12-13 18:51:19 +00:00
}
2021-07-11 23:17:14 +00:00
void LocalServer : : cleanup ( )
2020-03-18 00:57:00 +00:00
{
2021-10-16 11:28:57 +00:00
try
2020-03-18 00:57:00 +00:00
{
2021-10-16 11:28:57 +00:00
connection . reset ( ) ;
2021-09-04 18:19:01 +00:00
2024-01-17 13:28:10 +00:00
/// Suggestions are loaded async in a separate thread and it can use global context.
/// We should reset it before resetting global_context.
if ( suggest )
suggest . reset ( ) ;
2024-01-17 13:54:18 +00:00
2024-07-25 14:36:32 +00:00
client_context . reset ( ) ;
2021-10-16 11:28:57 +00:00
if ( global_context )
{
global_context - > shutdown ( ) ;
global_context . reset ( ) ;
}
2021-09-04 18:19:01 +00:00
2022-09-05 21:10:03 +00:00
/// thread status should be destructed before shared context because it relies on process list.
2021-10-16 11:28:57 +00:00
status . reset ( ) ;
2021-09-04 18:19:01 +00:00
2021-10-16 11:28:57 +00:00
// Delete the temporary directory if needed.
if ( temporary_directory_to_delete )
{
2024-04-27 16:33:34 +00:00
LOG_DEBUG ( & logger ( ) , " Removing temporary directory: {} " , temporary_directory_to_delete - > string ( ) ) ;
fs : : remove_all ( * temporary_directory_to_delete ) ;
2021-10-16 11:28:57 +00:00
temporary_directory_to_delete . reset ( ) ;
}
}
catch ( . . . )
2021-07-11 23:17:14 +00:00
{
2021-10-16 11:28:57 +00:00
tryLogCurrentException ( __PRETTY_FUNCTION__ ) ;
2020-03-18 00:57:00 +00:00
}
}
2021-07-11 23:17:14 +00:00
std : : string LocalServer : : getInitialCreateTableQuery ( )
2016-10-25 12:14:27 +00:00
{
2024-06-25 14:23:37 +00:00
if ( ! getClientConfiguration ( ) . has ( " table-structure " ) & & ! getClientConfiguration ( ) . has ( " table-file " ) & & ! getClientConfiguration ( ) . has ( " table-data-format " ) & & ( ! isRegularFile ( STDIN_FILENO ) | | queries . empty ( ) ) )
2022-01-25 07:55:09 +00:00
return { } ;
2024-06-25 14:23:37 +00:00
auto table_name = backQuoteIfNeed ( getClientConfiguration ( ) . getString ( " table-name " , " table " ) ) ;
auto table_structure = getClientConfiguration ( ) . getString ( " table-structure " , " auto " ) ;
2017-04-01 07:20:54 +00:00
2021-07-11 23:17:14 +00:00
String table_file ;
2024-06-25 14:23:37 +00:00
if ( ! getClientConfiguration ( ) . has ( " table-file " ) | | getClientConfiguration ( ) . getString ( " table-file " ) = = " - " )
2021-07-11 23:17:14 +00:00
{
/// Use Unix tools stdin naming convention
table_file = " stdin " ;
2017-04-01 07:20:54 +00:00
}
2021-07-11 23:17:14 +00:00
else
{
/// Use regular file
2024-06-25 14:23:37 +00:00
auto file_name = getClientConfiguration ( ) . getString ( " table-file " ) ;
2022-01-20 12:55:28 +00:00
table_file = quoteString ( file_name ) ;
2021-07-11 23:17:14 +00:00
}
2024-03-23 01:42:22 +00:00
String data_format = backQuoteIfNeed ( default_input_format ) ;
2022-01-20 12:55:28 +00:00
2021-12-15 11:30:57 +00:00
if ( table_structure = = " auto " )
table_structure = " " ;
else
table_structure = " ( " + table_structure + " ) " ;
2024-08-08 08:07:39 +00:00
return fmt : : format ( " CREATE TEMPORARY TABLE {} {} ENGINE = File({}, {}); " ,
2021-07-11 23:17:14 +00:00
table_name , table_structure , data_format , table_file ) ;
}
static ConfigurationPtr getConfigurationFromXMLString ( const char * xml_data )
{
std : : stringstream ss { std : : string { xml_data } } ; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
Poco : : XML : : InputSource input_source { ss } ;
return { new Poco : : Util : : XMLConfiguration { & input_source } } ;
}
void LocalServer : : setupUsers ( )
{
static const char * minimal_default_user_xml =
2021-10-25 18:15:41 +00:00
" <clickhouse> "
2021-07-11 23:17:14 +00:00
" <profiles> "
" <default></default> "
" </profiles> "
" <users> "
" <default> "
" <password></password> "
" <networks> "
" <ip>::/0</ip> "
" </networks> "
" <profile>default</profile> "
" <quota>default</quota> "
2024-07-02 09:52:51 +00:00
" <named_collection_control>1</named_collection_control> "
2021-07-11 23:17:14 +00:00
" </default> "
" </users> "
" <quotas> "
" <default></default> "
" </quotas> "
2021-10-25 18:15:41 +00:00
" </clickhouse> " ;
2021-07-11 23:17:14 +00:00
ConfigurationPtr users_config ;
2022-03-10 04:35:01 +00:00
auto & access_control = global_context - > getAccessControl ( ) ;
2024-06-25 14:23:37 +00:00
access_control . setNoPasswordAllowed ( getClientConfiguration ( ) . getBool ( " allow_no_password " , true ) ) ;
access_control . setPlaintextPasswordAllowed ( getClientConfiguration ( ) . getBool ( " allow_plaintext_password " , true ) ) ;
if ( getClientConfiguration ( ) . has ( " config-file " ) | | fs : : exists ( " config.xml " ) )
2020-10-14 08:50:36 +00:00
{
2024-06-25 14:23:37 +00:00
String config_path = getClientConfiguration ( ) . getString ( " config-file " , " " ) ;
bool has_user_directories = getClientConfiguration ( ) . has ( " user_directories " ) ;
2022-08-01 14:03:36 +00:00
const auto config_dir = fs : : path { config_path } . remove_filename ( ) . string ( ) ;
2024-06-25 14:23:37 +00:00
String users_config_path = getClientConfiguration ( ) . getString ( " users_config " , " " ) ;
2022-08-01 14:03:36 +00:00
if ( users_config_path . empty ( ) & & has_user_directories )
{
2024-06-25 14:23:37 +00:00
users_config_path = getClientConfiguration ( ) . getString ( " user_directories.users_xml.path " ) ;
2022-08-01 14:05:50 +00:00
if ( fs : : path ( users_config_path ) . is_relative ( ) & & fs : : exists ( fs : : path ( config_dir ) / users_config_path ) )
users_config_path = fs : : path ( config_dir ) / users_config_path ;
2022-08-01 14:03:36 +00:00
}
if ( users_config_path . empty ( ) )
2022-07-22 02:46:36 +00:00
users_config = getConfigurationFromXMLString ( minimal_default_user_xml ) ;
else
2022-07-20 04:38:36 +00:00
{
2022-07-22 02:46:36 +00:00
ConfigProcessor config_processor ( users_config_path ) ;
const auto loaded_config = config_processor . loadConfig ( ) ;
users_config = loaded_config . configuration ;
2022-07-20 04:38:36 +00:00
}
2021-07-11 23:17:14 +00:00
}
else
users_config = getConfigurationFromXMLString ( minimal_default_user_xml ) ;
if ( users_config )
global_context - > setUsersConfig ( users_config ) ;
else
2023-01-23 21:13:58 +00:00
throw Exception ( ErrorCodes : : CANNOT_LOAD_CONFIG , " Can't load config for users " ) ;
2021-07-11 23:17:14 +00:00
}
2017-04-01 07:20:54 +00:00
2021-08-23 08:50:12 +00:00
void LocalServer : : connect ( )
{
2024-06-25 14:23:37 +00:00
connection_parameters = ConnectionParameters ( getClientConfiguration ( ) , " localhost " ) ;
2024-03-25 15:39:33 +00:00
2024-07-01 00:38:42 +00:00
/// This is needed for table function input(...).
2024-03-25 15:39:33 +00:00
ReadBuffer * in ;
2024-06-25 14:23:37 +00:00
auto table_file = getClientConfiguration ( ) . getString ( " table-file " , " - " ) ;
2024-03-25 15:39:33 +00:00
if ( table_file = = " - " | | table_file = = " stdin " )
{
in = & std_in ;
}
else
{
input = std : : make_unique < ReadBufferFromFile > ( table_file ) ;
in = input . get ( ) ;
}
2022-03-05 06:22:56 +00:00
connection = LocalConnection : : createConnection (
2024-07-25 14:36:32 +00:00
connection_parameters , client_context , in , need_render_progress , need_render_profile_events , server_display_name ) ;
2021-08-23 08:50:12 +00:00
}
2017-04-01 07:20:54 +00:00
2021-09-04 18:19:01 +00:00
int LocalServer : : main ( const std : : vector < std : : string > & /*args*/ )
2021-07-31 12:34:29 +00:00
try
2016-10-25 12:14:27 +00:00
{
2021-09-04 18:19:01 +00:00
UseSSL use_ssl ;
2022-09-05 21:10:03 +00:00
thread_status . emplace ( ) ;
2022-07-29 23:50:45 +00:00
2024-10-16 19:13:26 +00:00
StackTrace : : setShowAddresses ( server_settings [ ServerSetting : : show_addresses_in_stack_traces ] ) ;
2022-07-29 23:50:45 +00:00
2021-10-02 08:10:34 +00:00
setupSignalHandler ( ) ;
2021-07-31 12:34:29 +00:00
2021-12-06 18:27:06 +00:00
std : : cout < < std : : fixed < < std : : setprecision ( 3 ) ;
std : : cerr < < std : : fixed < < std : : setprecision ( 3 ) ;
2023-02-25 02:20:47 +00:00
/// Try to increase limit on number of open files.
{
rlimit rlim ;
if ( getrlimit ( RLIMIT_NOFILE , & rlim ) )
throw Poco : : Exception ( " Cannot getrlimit " ) ;
if ( rlim . rlim_cur < rlim . rlim_max )
{
2024-06-25 14:23:37 +00:00
rlim . rlim_cur = getClientConfiguration ( ) . getUInt ( " max_open_files " , static_cast < unsigned > ( rlim . rlim_max ) ) ;
2023-02-25 02:20:47 +00:00
int rc = setrlimit ( RLIMIT_NOFILE , & rlim ) ;
if ( rc ! = 0 )
std : : cerr < < fmt : : format ( " Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {} " , rlim . rlim_cur , errnoToString ( ) ) < < ' \n ' ;
}
}
2021-10-29 12:04:08 +00:00
is_interactive = stdin_is_a_tty
2024-06-25 14:23:37 +00:00
& & ( getClientConfiguration ( ) . hasOption ( " interactive " )
| | ( queries . empty ( ) & & ! getClientConfiguration ( ) . has ( " table-structure " ) & & queries_files . empty ( ) & & ! getClientConfiguration ( ) . has ( " table-file " ) ) ) ;
2024-03-18 01:16:52 +00:00
2021-10-01 13:47:39 +00:00
if ( ! is_interactive )
2021-09-29 19:17:26 +00:00
{
/// We will terminate process on error
static KillingErrorHandler error_handler ;
Poco : : ErrorHandler : : set ( & error_handler ) ;
}
2021-07-31 12:34:29 +00:00
2024-01-09 06:33:48 +00:00
registerInterpreters ( ) ;
2021-07-31 12:34:29 +00:00
/// Don't initialize DateLUT
2017-04-21 17:47:27 +00:00
registerFunctions ( ) ;
2017-05-05 20:39:25 +00:00
registerAggregateFunctions ( ) ;
2024-11-15 20:28:54 +00:00
registerTableFunctions ( ) ;
2023-12-30 03:35:00 +00:00
registerDatabases ( ) ;
2024-11-15 20:28:54 +00:00
registerStorages ( ) ;
registerDictionaries ( ) ;
2022-11-19 08:09:24 +00:00
registerDisks ( /* global_skip_access_check= */ true ) ;
2020-10-29 03:39:43 +00:00
registerFormats ( ) ;
2017-04-21 17:47:27 +00:00
2021-07-31 12:34:29 +00:00
processConfig ( ) ;
2024-04-27 16:33:34 +00:00
2024-04-27 16:36:46 +00:00
SCOPE_EXIT ( { cleanup ( ) ; } ) ;
2024-04-27 16:33:34 +00:00
2024-07-16 01:08:37 +00:00
initTTYBuffer ( toProgressOption ( getClientConfiguration ( ) . getString ( " progress " , " default " ) ) ,
2024-05-13 10:50:52 +00:00
toProgressOption ( config ( ) . getString ( " progress-table " , " default " ) ) ) ;
2024-07-22 01:15:27 +00:00
initKeystrokeInterceptor ( ) ;
2024-02-22 18:17:15 +00:00
ASTAlterCommand : : setFormatAlterCommandsWithParentheses ( true ) ;
2022-11-12 02:55:26 +00:00
2023-07-27 05:06:16 +00:00
/// try to load user defined executable functions, throw on error and die
try
{
2024-06-25 14:23:37 +00:00
global_context - > loadOrReloadUserDefinedExecutableFunctions ( getClientConfiguration ( ) ) ;
2023-07-27 05:06:16 +00:00
}
catch ( . . . )
{
tryLogCurrentException ( & logger ( ) , " Caught exception while loading user defined executable functions. " ) ;
throw ;
}
2024-07-25 14:36:32 +00:00
/// Must be called after we stopped initializing the global context and changing its settings.
/// After this point the global context must be stayed almost unchanged till shutdown,
/// and all necessary changes must be made to the client context instead.
createClientContext ( ) ;
2021-07-31 12:34:29 +00:00
if ( is_interactive )
{
2021-09-04 18:19:01 +00:00
clearTerminal ( ) ;
showClientVersion ( ) ;
std : : cerr < < std : : endl ;
2021-11-14 07:19:59 +00:00
}
2021-08-23 08:50:12 +00:00
connect ( ) ;
2021-09-04 18:19:01 +00:00
2021-12-10 18:46:26 +00:00
String initial_query = getInitialCreateTableQuery ( ) ;
if ( ! initial_query . empty ( ) )
processQueryText ( initial_query ) ;
2024-03-18 01:16:52 +00:00
# if defined(FUZZING_MODE)
runLibFuzzer ( ) ;
# else
2021-10-29 12:04:08 +00:00
if ( is_interactive & & ! delayed_interactive )
2021-07-31 12:34:29 +00:00
{
2021-08-18 21:01:17 +00:00
runInteractive ( ) ;
2021-07-31 12:34:29 +00:00
}
else
{
runNonInteractive ( ) ;
2021-10-29 12:04:08 +00:00
if ( delayed_interactive )
runInteractive ( ) ;
2021-07-31 12:34:29 +00:00
}
2021-10-07 18:01:36 +00:00
# endif
2024-03-18 01:16:52 +00:00
2021-07-31 12:34:29 +00:00
return Application : : EXIT_OK ;
}
2021-10-16 11:28:57 +00:00
catch ( const DB : : Exception & e )
2021-07-31 12:34:29 +00:00
{
2024-06-25 14:23:37 +00:00
bool need_print_stack_trace = getClientConfiguration ( ) . getBool ( " stacktrace " , false ) ;
2021-10-25 08:00:55 +00:00
std : : cerr < < getExceptionMessage ( e , need_print_stack_trace , true ) < < std : : endl ;
2024-11-07 20:28:06 +00:00
auto code = DB : : getCurrentExceptionCode ( ) ;
return static_cast < UInt8 > ( code ) ? code : 1 ;
2021-10-16 11:28:57 +00:00
}
catch ( . . . )
{
2024-11-07 20:28:06 +00:00
std : : cerr < < DB : : getCurrentExceptionMessage ( true ) < < ' \n ' ;
auto code = DB : : getCurrentExceptionCode ( ) ;
return static_cast < UInt8 > ( code ) ? code : 1 ;
2021-07-23 20:54:49 +00:00
}
2022-04-05 13:38:44 +00:00
void LocalServer : : updateLoggerLevel ( const String & logs_level )
2022-04-05 12:46:18 +00:00
{
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setString ( " logger.level " , logs_level ) ;
updateLevels ( getClientConfiguration ( ) , logger ( ) ) ;
2022-04-05 12:46:18 +00:00
}
2021-07-23 20:54:49 +00:00
void LocalServer : : processConfig ( )
{
2024-06-25 14:23:37 +00:00
if ( ! queries . empty ( ) & & getClientConfiguration ( ) . has ( " queries-file " ) )
2023-05-28 22:25:40 +00:00
throw Exception ( ErrorCodes : : BAD_ARGUMENTS , " Options '--query' and '--queries-file' cannot be specified at the same time " ) ;
2024-06-25 14:23:37 +00:00
pager = getClientConfiguration ( ) . getString ( " pager " , " " ) ;
2023-10-16 20:26:49 +00:00
2024-06-25 14:23:37 +00:00
delayed_interactive = getClientConfiguration ( ) . has ( " interactive " ) & & ( ! queries . empty ( ) | | getClientConfiguration ( ) . has ( " queries-file " ) ) ;
2023-09-11 09:32:41 +00:00
if ( ! is_interactive | | delayed_interactive )
2021-07-23 20:54:49 +00:00
{
2024-06-25 14:23:37 +00:00
echo_queries = getClientConfiguration ( ) . hasOption ( " echo " ) | | getClientConfiguration ( ) . hasOption ( " verbose " ) ;
ignore_error = getClientConfiguration ( ) . getBool ( " ignore-error " , false ) ;
2021-07-23 20:54:49 +00:00
}
2022-02-12 19:26:01 +00:00
2024-06-25 14:23:37 +00:00
print_stack_trace = getClientConfiguration ( ) . getBool ( " stacktrace " , false ) ;
2023-07-26 15:25:57 +00:00
const std : : string clickhouse_dialect { " clickhouse " } ;
2024-06-25 14:23:37 +00:00
load_suggestions = ( is_interactive | | delayed_interactive ) & & ! getClientConfiguration ( ) . getBool ( " disable_suggestion " , false )
& & getClientConfiguration ( ) . getString ( " dialect " , clickhouse_dialect ) = = clickhouse_dialect ;
wait_for_suggestions_to_load = getClientConfiguration ( ) . getBool ( " wait_for_suggestions_to_load " , false ) ;
2021-07-23 20:54:49 +00:00
2024-06-25 14:23:37 +00:00
auto logging = ( getClientConfiguration ( ) . has ( " logger.console " )
| | getClientConfiguration ( ) . has ( " logger.level " )
| | getClientConfiguration ( ) . has ( " log-level " )
| | getClientConfiguration ( ) . has ( " send_logs_level " )
| | getClientConfiguration ( ) . has ( " logger.log " ) ) ;
2021-10-30 12:28:50 +00:00
2024-06-25 14:23:37 +00:00
auto level = getClientConfiguration ( ) . getString ( " log-level " , " trace " ) ;
2021-10-30 12:28:50 +00:00
2024-06-25 14:23:37 +00:00
if ( getClientConfiguration ( ) . has ( " server_logs_file " ) )
2021-10-30 12:28:50 +00:00
{
2022-04-05 12:46:18 +00:00
auto poco_logs_level = Poco : : Logger : : parseLevel ( level ) ;
Poco : : Logger : : root ( ) . setLevel ( poco_logs_level ) ;
2023-09-19 21:39:59 +00:00
Poco : : AutoPtr < OwnPatternFormatter > pf = new OwnPatternFormatter ;
Poco : : AutoPtr < OwnFormattingChannel > log = new OwnFormattingChannel ( pf , new Poco : : SimpleFileChannel ( server_logs_file ) ) ;
Poco : : Logger : : root ( ) . setChannel ( log ) ;
2021-10-30 12:28:50 +00:00
}
2023-11-19 15:09:52 +00:00
else
2021-10-30 12:28:50 +00:00
{
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setString ( " logger " , " logger " ) ;
2023-11-19 15:09:52 +00:00
auto log_level_default = logging ? level : " fatal " ;
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setString ( " logger.level " , getClientConfiguration ( ) . getString ( " log-level " , getClientConfiguration ( ) . getString ( " send_logs_level " , log_level_default ) ) ) ;
buildLoggers ( getClientConfiguration ( ) , logger ( ) , " clickhouse-local " ) ;
2021-10-30 12:28:50 +00:00
}
2021-07-23 20:54:49 +00:00
shared_context = Context : : createShared ( ) ;
global_context = Context : : createGlobal ( shared_context . get ( ) ) ;
global_context - > makeGlobalContext ( ) ;
global_context - > setApplicationType ( Context : : ApplicationType : : LOCAL ) ;
tryInitPath ( ) ;
2024-01-23 17:04:50 +00:00
LoggerRawPtr log = & logger ( ) ;
2021-07-23 20:54:49 +00:00
2017-04-01 07:20:54 +00:00
/// Maybe useless
2024-06-25 14:23:37 +00:00
if ( getClientConfiguration ( ) . has ( " macros " ) )
global_context - > setMacros ( std : : make_unique < Macros > ( getClientConfiguration ( ) , " macros " , log ) ) ;
2017-04-01 07:20:54 +00:00
2024-05-13 07:21:01 +00:00
setDefaultFormatsAndCompressionFromConfiguration ( ) ;
2021-08-18 14:39:04 +00:00
2021-03-11 20:41:10 +00:00
/// Sets external authenticators config (LDAP, Kerberos).
2024-06-25 14:23:37 +00:00
global_context - > setExternalAuthenticatorsConfig ( getClientConfiguration ( ) ) ;
2020-06-10 22:48:15 +00:00
2017-04-01 07:20:54 +00:00
setupUsers ( ) ;
/// Limit on total number of concurrently executing queries.
2020-03-28 03:02:26 +00:00
/// There is no need for concurrent queries, override max_concurrent_queries.
2020-10-22 07:37:03 +00:00
global_context - > getProcessList ( ) . setMaxSize ( 0 ) ;
2017-04-01 07:20:54 +00:00
2023-08-22 15:44:26 +00:00
const size_t physical_server_memory = getMemoryAmount ( ) ;
2024-06-26 00:29:04 +00:00
2024-10-16 19:13:26 +00:00
size_t max_server_memory_usage = server_settings [ ServerSetting : : max_server_memory_usage ] ;
double max_server_memory_usage_to_ram_ratio = server_settings [ ServerSetting : : max_server_memory_usage_to_ram_ratio ] ;
2024-06-26 00:29:04 +00:00
size_t default_max_server_memory_usage = static_cast < size_t > ( physical_server_memory * max_server_memory_usage_to_ram_ratio ) ;
if ( max_server_memory_usage = = 0 )
{
max_server_memory_usage = default_max_server_memory_usage ;
LOG_INFO ( log , " Setting max_server_memory_usage was set to {} "
" ({} available * {:.2f} max_server_memory_usage_to_ram_ratio) " ,
formatReadableSizeWithBinarySuffix ( max_server_memory_usage ) ,
formatReadableSizeWithBinarySuffix ( physical_server_memory ) ,
max_server_memory_usage_to_ram_ratio ) ;
}
else if ( max_server_memory_usage > default_max_server_memory_usage )
{
max_server_memory_usage = default_max_server_memory_usage ;
LOG_INFO ( log , " Setting max_server_memory_usage was lowered to {} "
" because the system has low amount of memory. The amount was "
" calculated as {} available "
" * {:.2f} max_server_memory_usage_to_ram_ratio " ,
formatReadableSizeWithBinarySuffix ( max_server_memory_usage ) ,
formatReadableSizeWithBinarySuffix ( physical_server_memory ) ,
max_server_memory_usage_to_ram_ratio ) ;
}
total_memory_tracker . setHardLimit ( max_server_memory_usage ) ;
total_memory_tracker . setDescription ( " (total) " ) ;
total_memory_tracker . setMetric ( CurrentMetrics : : MemoryTracking ) ;
2024-10-16 19:13:26 +00:00
const double cache_size_to_ram_max_ratio = server_settings [ ServerSetting : : cache_size_to_ram_max_ratio ] ;
2023-08-22 15:44:26 +00:00
const size_t max_cache_size = static_cast < size_t > ( physical_server_memory * cache_size_to_ram_max_ratio ) ;
2023-06-29 18:33:48 +00:00
2024-10-16 19:13:26 +00:00
String uncompressed_cache_policy = server_settings [ ServerSetting : : uncompressed_cache_policy ] ;
size_t uncompressed_cache_size = server_settings [ ServerSetting : : uncompressed_cache_size ] ;
double uncompressed_cache_size_ratio = server_settings [ ServerSetting : : uncompressed_cache_size_ratio ] ;
2023-06-29 18:33:48 +00:00
if ( uncompressed_cache_size > max_cache_size )
{
uncompressed_cache_size = max_cache_size ;
LOG_INFO ( log , " Lowered uncompressed cache size to {} because the system has limited RAM " , formatReadableSizeWithBinarySuffix ( uncompressed_cache_size ) ) ;
}
2023-08-21 19:08:34 +00:00
global_context - > setUncompressedCache ( uncompressed_cache_policy , uncompressed_cache_size , uncompressed_cache_size_ratio ) ;
2017-04-01 07:20:54 +00:00
2024-10-16 19:13:26 +00:00
String mark_cache_policy = server_settings [ ServerSetting : : mark_cache_policy ] ;
size_t mark_cache_size = server_settings [ ServerSetting : : mark_cache_size ] ;
double mark_cache_size_ratio = server_settings [ ServerSetting : : mark_cache_size_ratio ] ;
2023-06-29 18:47:03 +00:00
if ( ! mark_cache_size )
LOG_ERROR ( log , " Too low mark cache size will lead to severe performance degradation. " ) ;
2023-06-29 18:33:48 +00:00
if ( mark_cache_size > max_cache_size )
{
mark_cache_size = max_cache_size ;
LOG_INFO ( log , " Lowered mark cache size to {} because the system has limited RAM " , formatReadableSizeWithBinarySuffix ( mark_cache_size ) ) ;
}
2023-08-21 19:08:34 +00:00
global_context - > setMarkCache ( mark_cache_policy , mark_cache_size , mark_cache_size_ratio ) ;
2017-04-01 07:20:54 +00:00
2024-10-16 19:13:26 +00:00
String index_uncompressed_cache_policy = server_settings [ ServerSetting : : index_uncompressed_cache_policy ] ;
size_t index_uncompressed_cache_size = server_settings [ ServerSetting : : index_uncompressed_cache_size ] ;
double index_uncompressed_cache_size_ratio = server_settings [ ServerSetting : : index_uncompressed_cache_size_ratio ] ;
2023-06-29 18:33:48 +00:00
if ( index_uncompressed_cache_size > max_cache_size )
{
index_uncompressed_cache_size = max_cache_size ;
2024-08-19 17:28:27 +00:00
LOG_INFO ( log , " Lowered index uncompressed cache size to {} because the system has limited RAM " , formatReadableSizeWithBinarySuffix ( index_uncompressed_cache_size ) ) ;
2023-06-29 18:33:48 +00:00
}
2023-08-21 19:08:34 +00:00
global_context - > setIndexUncompressedCache ( index_uncompressed_cache_policy , index_uncompressed_cache_size , index_uncompressed_cache_size_ratio ) ;
2021-08-21 17:27:22 +00:00
2024-10-16 19:13:26 +00:00
String index_mark_cache_policy = server_settings [ ServerSetting : : index_mark_cache_policy ] ;
size_t index_mark_cache_size = server_settings [ ServerSetting : : index_mark_cache_size ] ;
double index_mark_cache_size_ratio = server_settings [ ServerSetting : : index_mark_cache_size_ratio ] ;
2023-06-29 18:33:48 +00:00
if ( index_mark_cache_size > max_cache_size )
{
index_mark_cache_size = max_cache_size ;
2024-08-19 17:28:27 +00:00
LOG_INFO ( log , " Lowered index mark cache size to {} because the system has limited RAM " , formatReadableSizeWithBinarySuffix ( index_mark_cache_size ) ) ;
2023-06-29 18:33:48 +00:00
}
2023-08-21 19:08:34 +00:00
global_context - > setIndexMarkCache ( index_mark_cache_policy , index_mark_cache_size , index_mark_cache_size_ratio ) ;
2021-08-21 17:27:22 +00:00
2024-11-19 16:15:12 +00:00
String primary_index_cache_policy = server_settings [ ServerSetting : : primary_index_cache_policy ] ;
size_t primary_index_cache_size = server_settings [ ServerSetting : : primary_index_cache_size ] ;
double primary_index_cache_size_ratio = server_settings [ ServerSetting : : primary_index_cache_size_ratio ] ;
if ( primary_index_cache_size > max_cache_size )
{
primary_index_cache_size = max_cache_size ;
2024-11-22 17:21:38 +00:00
LOG_INFO ( log , " Lowered primary index cache size to {} because the system has limited RAM " , formatReadableSizeWithBinarySuffix ( primary_index_cache_size ) ) ;
2024-11-19 16:15:12 +00:00
}
global_context - > setPrimaryIndexCache ( primary_index_cache_policy , primary_index_cache_size , primary_index_cache_size_ratio ) ;
2024-10-16 19:13:26 +00:00
size_t mmap_cache_size = server_settings [ ServerSetting : : mmap_cache_size ] ;
2023-06-29 18:33:48 +00:00
if ( mmap_cache_size > max_cache_size )
{
mmap_cache_size = max_cache_size ;
2024-08-19 17:28:27 +00:00
LOG_INFO ( log , " Lowered mmap file cache size to {} because the system has limited RAM " , formatReadableSizeWithBinarySuffix ( mmap_cache_size ) ) ;
2023-06-29 18:33:48 +00:00
}
2023-06-29 13:02:11 +00:00
global_context - > setMMappedFileCache ( mmap_cache_size ) ;
2021-03-28 01:10:30 +00:00
2023-06-29 13:02:11 +00:00
/// Initialize a dummy query cache.
global_context - > setQueryCache ( 0 , 0 , 0 , 0 ) ;
2023-06-29 18:55:09 +00:00
2024-11-15 17:21:53 +00:00
/// Initialize allowed tiers
global_context - > getAccessControl ( ) . setAllowTierSettings ( server_settings [ ServerSetting : : allowed_feature_tier ] ) ;
2022-10-19 10:27:00 +00:00
# if USE_EMBEDDED_COMPILER
2024-10-16 19:13:26 +00:00
size_t compiled_expression_cache_max_size_in_bytes = server_settings [ ServerSetting : : compiled_expression_cache_size ] ;
size_t compiled_expression_cache_max_elements = server_settings [ ServerSetting : : compiled_expression_cache_elements_size ] ;
2023-06-29 10:19:40 +00:00
CompiledExpressionCacheFactory : : instance ( ) . init ( compiled_expression_cache_max_size_in_bytes , compiled_expression_cache_max_elements ) ;
2022-10-19 10:27:00 +00:00
# endif
2021-03-28 01:10:30 +00:00
2023-06-24 19:41:33 +00:00
/// NOTE: it is important to apply any overrides before
/// setDefaultProfiles() calls since it will copy current context (i.e.
/// there is separate context for Buffer tables).
2024-04-24 21:08:49 +00:00
adjustSettings ( ) ;
2023-06-24 19:41:33 +00:00
applySettingsOverridesForLocal ( global_context ) ;
applyCmdOptions ( global_context ) ;
2018-02-01 13:52:29 +00:00
/// Load global settings from default_profile and system_profile.
2024-06-25 14:23:37 +00:00
global_context - > setDefaultProfiles ( getClientConfiguration ( ) ) ;
2017-04-01 07:20:54 +00:00
2024-07-27 12:42:23 +00:00
/// Command-line parameters can override settings from the default profile.
applyCmdSettings ( global_context ) ;
2024-07-25 14:36:32 +00:00
2021-08-26 11:01:15 +00:00
/// We load temporary database first, because projections need it.
DatabaseCatalog : : instance ( ) . initializeAndLoadTemporaryDatabase ( ) ;
2024-10-16 19:13:26 +00:00
std : : string default_database = server_settings [ ServerSetting : : default_database ] ;
2024-08-08 08:07:39 +00:00
{
DatabasePtr database = createClickHouseLocalDatabaseOverlay ( default_database , global_context ) ;
if ( UUID uuid = database - > getUUID ( ) ; uuid ! = UUIDHelpers : : Nil )
DatabaseCatalog : : instance ( ) . addUUIDMapping ( uuid ) ;
DatabaseCatalog : : instance ( ) . attachDatabase ( default_database , database ) ;
}
2020-10-22 07:37:03 +00:00
global_context - > setCurrentDatabase ( default_database ) ;
2017-04-01 07:20:54 +00:00
2024-06-25 14:23:37 +00:00
if ( getClientConfiguration ( ) . has ( " path " ) )
2017-04-01 07:20:54 +00:00
{
2021-01-16 15:08:21 +00:00
String path = global_context - > getPath ( ) ;
2024-05-03 01:30:28 +00:00
fs : : create_directories ( fs : : path ( path ) ) ;
2021-01-16 15:08:21 +00:00
2018-04-20 15:32:40 +00:00
/// Lock path directory before read
2021-07-23 20:54:49 +00:00
status . emplace ( fs : : path ( path ) / " status " , StatusFile : : write_full_info ) ;
2018-04-20 15:32:40 +00:00
2020-08-13 19:41:06 +00:00
LOG_DEBUG ( log , " Loading metadata from {} " , path ) ;
2024-09-22 12:06:59 +00:00
auto load_system_metadata_tasks = loadMetadataSystem ( global_context ) ;
2023-12-29 20:25:22 +00:00
attachSystemTablesServer ( global_context , * createMemoryDatabaseIfNotExists ( global_context , DatabaseCatalog : : SYSTEM_DATABASE ) , false ) ;
2021-09-09 09:37:51 +00:00
attachInformationSchema ( global_context , * createMemoryDatabaseIfNotExists ( global_context , DatabaseCatalog : : INFORMATION_SCHEMA ) ) ;
attachInformationSchema ( global_context , * createMemoryDatabaseIfNotExists ( global_context , DatabaseCatalog : : INFORMATION_SCHEMA_UPPERCASE ) ) ;
2024-09-22 12:06:59 +00:00
waitLoad ( TablesLoaderForegroundPoolId , load_system_metadata_tasks ) ;
2022-09-19 14:10:00 +00:00
2024-06-25 14:23:37 +00:00
if ( ! getClientConfiguration ( ) . has ( " only-system-tables " ) )
2022-09-19 14:10:00 +00:00
{
2023-02-17 16:57:49 +00:00
DatabaseCatalog : : instance ( ) . createBackgroundTasks ( ) ;
2023-05-01 11:56:00 +00:00
waitLoad ( loadMetadata ( global_context ) ) ;
2023-09-12 06:00:40 +00:00
DatabaseCatalog : : instance ( ) . startupBackgroundTasks ( ) ;
2022-09-19 14:10:00 +00:00
}
2021-07-23 20:54:49 +00:00
2022-09-24 21:24:39 +00:00
/// For ClickHouse local if path is not set the loader will be disabled.
2023-12-12 14:03:49 +00:00
global_context - > getUserDefinedSQLObjectsStorage ( ) . loadObjects ( ) ;
2022-09-24 21:24:39 +00:00
2020-05-23 22:24:01 +00:00
LOG_DEBUG ( log , " Loaded metadata. " ) ;
2017-04-01 07:20:54 +00:00
}
2024-06-25 14:23:37 +00:00
else if ( ! getClientConfiguration ( ) . has ( " no-system-tables " ) )
2017-06-15 20:08:26 +00:00
{
2023-12-29 20:25:22 +00:00
attachSystemTablesServer ( global_context , * createMemoryDatabaseIfNotExists ( global_context , DatabaseCatalog : : SYSTEM_DATABASE ) , false ) ;
2021-09-09 09:37:51 +00:00
attachInformationSchema ( global_context , * createMemoryDatabaseIfNotExists ( global_context , DatabaseCatalog : : INFORMATION_SCHEMA ) ) ;
attachInformationSchema ( global_context , * createMemoryDatabaseIfNotExists ( global_context , DatabaseCatalog : : INFORMATION_SCHEMA_UPPERCASE ) ) ;
2017-06-15 20:08:26 +00:00
}
2021-09-30 13:51:15 +00:00
2024-06-25 14:23:37 +00:00
server_display_name = getClientConfiguration ( ) . getString ( " display_name " , " " ) ;
prompt_by_server_display_name = getClientConfiguration ( ) . getRawString ( " prompt_by_server_display_name.default " , " :) " ) ;
2017-10-13 19:13:41 +00:00
}
2021-12-06 18:27:06 +00:00
[[ maybe_unused ]] static std : : string getHelpHeader ( )
2018-04-20 19:31:19 +00:00
{
return
" usage: clickhouse-local [initial table definition] [--query <query>] \n "
" clickhouse-local allows to execute SQL queries on your data files via single command line call. "
" To do so, initially you need to define your data source and its format. "
" After you can execute your SQL queries in usual manner. \n "
" There are two ways to define initial table keeping your data. "
" Either just in first query like this: \n "
" CREATE TABLE <table> (<structure>) ENGINE = File(<input-format>, <file>); \n "
" Either through corresponding command line parameters --table --structure --input-format and --file. " ;
}
2021-07-11 23:17:14 +00:00
2021-12-06 18:27:06 +00:00
[[ maybe_unused ]] static std : : string getHelpFooter ( )
2018-04-20 19:31:19 +00:00
{
return
" Example printing memory used by each Unix user: \n "
" ps aux | tail -n +2 | awk '{ printf( \" %s \\ t%s \\ n \" , $1, $4) }' | "
" clickhouse-local -S \" user String, mem Float64 \" -q "
" \" SELECT user, round(sum(mem), 2) as mem_total FROM table GROUP BY user ORDER "
" BY mem_total DESC FORMAT PrettyCompact \" " ;
}
2021-07-11 23:17:14 +00:00
2024-04-26 12:05:53 +00:00
void LocalServer : : printHelpMessage ( const OptionsDescription & options_description , bool verbose )
2018-04-20 19:31:19 +00:00
{
2021-07-11 11:36:27 +00:00
std : : cout < < getHelpHeader ( ) < < " \n " ;
std : : cout < < options_description . main_description . value ( ) < < " \n " ;
2024-04-26 11:19:22 +00:00
if ( verbose )
std : : cout < < " All settings are documented at https://clickhouse.com/docs/en/operations/settings/settings. \n \n " ;
2021-07-11 11:36:27 +00:00
std : : cout < < getHelpFooter ( ) < < " \n " ;
2023-12-24 10:07:26 +00:00
std : : cout < < " In addition, --param_name=value can be specified for substitution of parameters for parametrized queries. \n " ;
2024-02-11 19:00:37 +00:00
std : : cout < < " \n See also: https://clickhouse.com/docs/en/operations/utilities/clickhouse-local/ \n " ;
2021-07-11 11:36:27 +00:00
}
2018-05-23 16:12:46 +00:00
2018-04-20 19:31:19 +00:00
2021-10-14 13:34:05 +00:00
void LocalServer : : addOptions ( OptionsDescription & options_description )
2021-07-11 11:36:27 +00:00
{
options_description . main_description - > add_options ( )
2018-04-20 19:31:19 +00:00
( " table,N " , po : : value < std : : string > ( ) , " name of the initial table " )
2024-08-20 02:29:30 +00:00
( " copy " , " shortcut for format conversion, equivalent to: --query 'SELECT * FROM table' " )
2021-08-20 21:19:06 +00:00
2018-04-20 19:31:19 +00:00
/// If structure argument is omitted then initial query is not generated
( " structure,S " , po : : value < std : : string > ( ) , " structure of the initial table (list of column and type names) " )
2024-02-28 04:16:34 +00:00
( " file,F " , po : : value < std : : string > ( ) , " path to file with data of the initial table (stdin if not specified) " )
2021-08-20 21:19:06 +00:00
2018-04-20 19:31:19 +00:00
( " input-format " , po : : value < std : : string > ( ) , " input format of the initial table data " )
2020-06-28 23:40:43 +00:00
( " logger.console " , po : : value < bool > ( ) - > implicit_value ( true ) , " Log to console " )
2019-06-14 14:00:37 +00:00
( " logger.log " , po : : value < std : : string > ( ) , " Log file name " )
( " logger.level " , po : : value < std : : string > ( ) , " Log level " )
2021-07-29 12:48:07 +00:00
2021-01-10 11:12:02 +00:00
( " no-system-tables " , " do not attach system tables (better startup time) " )
2021-10-15 15:29:27 +00:00
( " path " , po : : value < std : : string > ( ) , " Storage path " )
2022-09-19 14:10:00 +00:00
( " only-system-tables " , " attach only system tables from specified path " )
2021-10-25 18:12:45 +00:00
( " top_level_domains_path " , po : : value < std : : string > ( ) , " Path to lists with custom TLDs " )
2019-04-25 14:08:20 +00:00
;
2021-07-11 11:36:27 +00:00
}
2018-04-20 19:31:19 +00:00
2021-07-11 23:17:14 +00:00
void LocalServer : : applyCmdSettings ( ContextMutablePtr context )
{
context - > applySettingsChanges ( cmd_settings . changes ( ) ) ;
}
2018-04-20 19:31:19 +00:00
2021-07-11 23:17:14 +00:00
void LocalServer : : applyCmdOptions ( ContextMutablePtr context )
{
2024-06-25 14:23:37 +00:00
context - > setDefaultFormat ( getClientConfiguration ( ) . getString ( " output-format " , getClientConfiguration ( ) . getString ( " format " , is_interactive ? " PrettyCompact " : " TSV " ) ) ) ;
2021-07-11 23:17:14 +00:00
applyCmdSettings ( context ) ;
}
2018-04-20 19:31:19 +00:00
2021-07-11 23:17:14 +00:00
2024-07-25 14:36:32 +00:00
void LocalServer : : createClientContext ( )
{
/// In case of clickhouse-local it's necessary to use a separate context for client-related purposes.
/// We can't just change the global context because it is used in background tasks (for example, in merges)
/// which don't expect that the global context can suddenly change.
client_context = Context : : createCopy ( global_context ) ;
initClientContext ( ) ;
}
2022-02-10 09:43:08 +00:00
void LocalServer : : processOptions ( const OptionsDescription & , const CommandLineOptions & options , const std : : vector < Arguments > & , const std : : vector < Arguments > & )
2021-07-11 11:36:27 +00:00
{
2018-04-20 19:31:19 +00:00
if ( options . count ( " table " ) )
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setString ( " table-name " , options [ " table " ] . as < std : : string > ( ) ) ;
2018-04-20 19:31:19 +00:00
if ( options . count ( " file " ) )
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setString ( " table-file " , options [ " file " ] . as < std : : string > ( ) ) ;
2018-04-20 19:31:19 +00:00
if ( options . count ( " structure " ) )
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setString ( " table-structure " , options [ " structure " ] . as < std : : string > ( ) ) ;
2021-08-20 21:19:06 +00:00
if ( options . count ( " no-system-tables " ) )
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setBool ( " no-system-tables " , true ) ;
2022-09-19 14:10:00 +00:00
if ( options . count ( " only-system-tables " ) )
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setBool ( " only-system-tables " , true ) ;
2023-09-11 12:12:02 +00:00
if ( options . count ( " database " ) )
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setString ( " default_database " , options [ " database " ] . as < std : : string > ( ) ) ;
2021-08-20 21:19:06 +00:00
2018-04-20 19:31:19 +00:00
if ( options . count ( " input-format " ) )
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setString ( " table-data-format " , options [ " input-format " ] . as < std : : string > ( ) ) ;
2018-04-20 19:31:19 +00:00
if ( options . count ( " output-format " ) )
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setString ( " output-format " , options [ " output-format " ] . as < std : : string > ( ) ) ;
2018-04-20 19:31:19 +00:00
2020-06-28 23:40:43 +00:00
if ( options . count ( " logger.console " ) )
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setBool ( " logger.console " , options [ " logger.console " ] . as < bool > ( ) ) ;
2019-06-14 14:00:37 +00:00
if ( options . count ( " logger.log " ) )
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setString ( " logger.log " , options [ " logger.log " ] . as < std : : string > ( ) ) ;
2019-06-14 14:00:37 +00:00
if ( options . count ( " logger.level " ) )
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setString ( " logger.level " , options [ " logger.level " ] . as < std : : string > ( ) ) ;
2022-03-29 11:33:17 +00:00
if ( options . count ( " send_logs_level " ) )
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setString ( " send_logs_level " , options [ " send_logs_level " ] . as < std : : string > ( ) ) ;
2024-04-09 19:51:27 +00:00
if ( options . count ( " wait_for_suggestions_to_load " ) )
2024-06-25 14:23:37 +00:00
getClientConfiguration ( ) . setBool ( " wait_for_suggestions_to_load " , true ) ;
2024-08-19 22:54:09 +00:00
if ( options . count ( " copy " ) )
{
if ( ! queries . empty ( ) )
throw Exception ( ErrorCodes : : BAD_ARGUMENTS , " Options '--copy' and '--query' cannot be specified at the same time " ) ;
queries . emplace_back ( " SELECT * FROM table " ) ;
}
2018-04-20 19:31:19 +00:00
}
2022-03-01 09:22:12 +00:00
void LocalServer : : readArguments ( int argc , char * * argv , Arguments & common_arguments , std : : vector < Arguments > & , std : : vector < Arguments > & )
{
for ( int arg_num = 1 ; arg_num < argc ; + + arg_num )
{
2023-05-17 03:43:35 +00:00
std : : string_view arg = argv [ arg_num ] ;
2024-03-18 01:16:52 +00:00
2023-12-24 10:07:26 +00:00
/// Parameter arg after underline.
if ( arg . starts_with ( " --param_ " ) )
{
auto param_continuation = arg . substr ( strlen ( " --param_ " ) ) ;
auto equal_pos = param_continuation . find_first_of ( ' = ' ) ;
if ( equal_pos = = std : : string : : npos )
{
/// param_name value
+ + arg_num ;
if ( arg_num > = argc )
throw Exception ( ErrorCodes : : BAD_ARGUMENTS , " Parameter requires value " ) ;
arg = argv [ arg_num ] ;
query_parameters . emplace ( String ( param_continuation ) , String ( arg ) ) ;
}
else
{
if ( equal_pos = = 0 )
throw Exception ( ErrorCodes : : BAD_ARGUMENTS , " Parameter name cannot be empty " ) ;
/// param_name=value
query_parameters . emplace ( param_continuation . substr ( 0 , equal_pos ) , param_continuation . substr ( equal_pos + 1 ) ) ;
}
}
2023-05-17 03:43:35 +00:00
else
2024-03-18 01:16:52 +00:00
{
2023-05-17 03:43:35 +00:00
common_arguments . emplace_back ( arg ) ;
2024-03-18 01:16:52 +00:00
}
2022-03-01 09:22:12 +00:00
}
}
2018-05-11 14:35:32 +00:00
}
2024-03-11 12:54:34 +00:00
# pragma clang diagnostic ignored "-Wunused-function"
# pragma clang diagnostic ignored "-Wmissing-declarations"
2019-12-15 06:34:43 +00:00
2017-12-02 02:47:12 +00:00
int mainEntryClickHouseLocal ( int argc , char * * argv )
{
try
{
2021-12-25 19:00:28 +00:00
DB : : LocalServer app ;
2017-12-02 02:47:12 +00:00
app . init ( argc , argv ) ;
return app . run ( ) ;
}
2021-10-14 13:34:05 +00:00
catch ( const DB : : Exception & e )
2021-10-12 14:08:47 +00:00
{
2021-10-14 13:34:05 +00:00
std : : cerr < < DB : : getExceptionMessage ( e , false ) < < std : : endl ;
auto code = DB : : getCurrentExceptionCode ( ) ;
2024-11-07 20:28:06 +00:00
return static_cast < UInt8 > ( code ) ? code : 1 ;
2021-10-12 14:08:47 +00:00
}
2021-10-15 15:29:27 +00:00
catch ( const boost : : program_options : : error & e )
{
std : : cerr < < " Bad arguments: " < < e . what ( ) < < std : : endl ;
return DB : : ErrorCodes : : BAD_ARGUMENTS ;
}
2017-12-02 02:47:12 +00:00
catch ( . . . )
{
2018-08-23 00:14:26 +00:00
std : : cerr < < DB : : getCurrentExceptionMessage ( true ) < < ' \n ' ;
2017-12-02 02:47:12 +00:00
auto code = DB : : getCurrentExceptionCode ( ) ;
2024-11-07 20:28:06 +00:00
return static_cast < UInt8 > ( code ) ? code : 1 ;
2017-12-02 02:47:12 +00:00
}
2021-10-07 18:01:36 +00:00
}