mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-30 05:30:51 +00:00
241 lines
7.4 KiB
C++
241 lines
7.4 KiB
C++
#include <Poco/DirectoryIterator.h>
|
||
#include <DB/Common/ClickHouseRevision.h>
|
||
#include <ext/unlock_guard.hpp>
|
||
|
||
#include <DB/Common/SipHash.h>
|
||
#include <DB/Common/ShellCommand.h>
|
||
#include <DB/Common/StringUtils.h>
|
||
|
||
#include <DB/IO/Operators.h>
|
||
#include <DB/IO/ReadBufferFromString.h>
|
||
#include <DB/IO/ReadBufferFromFileDescriptor.h>
|
||
#include <DB/IO/WriteBufferFromFile.h>
|
||
|
||
#include <DB/Interpreters/Compiler.h>
|
||
#include <DB/Interpreters/config_compile.h>
|
||
|
||
|
||
namespace ProfileEvents
|
||
{
|
||
extern const Event CompileAttempt;
|
||
extern const Event CompileSuccess;
|
||
}
|
||
|
||
namespace DB
|
||
{
|
||
|
||
|
||
Compiler::Compiler(const std::string & path_, size_t threads)
|
||
: path(path_), pool(threads)
|
||
{
|
||
Poco::File(path).createDirectory();
|
||
|
||
Poco::DirectoryIterator dir_end;
|
||
for (Poco::DirectoryIterator dir_it(path); dir_end != dir_it; ++dir_it)
|
||
{
|
||
std::string name = dir_it.name();
|
||
if (endsWith(name, ".so"))
|
||
{
|
||
files.insert(name.substr(0, name.size() - 3));
|
||
}
|
||
}
|
||
|
||
LOG_INFO(log, "Having " << files.size() << " compiled files from previous start.");
|
||
}
|
||
|
||
Compiler::~Compiler()
|
||
{
|
||
LOG_DEBUG(log, "Waiting for threads to finish.");
|
||
pool.wait();
|
||
}
|
||
|
||
|
||
static Compiler::HashedKey getHash(const std::string & key)
|
||
{
|
||
SipHash hash;
|
||
|
||
auto revision = ClickHouseRevision::get();
|
||
hash.update(reinterpret_cast<const char *>(&revision), sizeof(revision));
|
||
hash.update(key.data(), key.size());
|
||
|
||
Compiler::HashedKey res;
|
||
hash.get128(res.first, res.second);
|
||
return res;
|
||
}
|
||
|
||
|
||
/// Без расширения .so.
|
||
static std::string hashedKeyToFileName(Compiler::HashedKey hashed_key)
|
||
{
|
||
std::string file_name;
|
||
|
||
{
|
||
WriteBufferFromString out(file_name);
|
||
out << hashed_key.first << '_' << hashed_key.second;
|
||
}
|
||
|
||
return file_name;
|
||
}
|
||
|
||
|
||
SharedLibraryPtr Compiler::getOrCount(
|
||
const std::string & key,
|
||
UInt32 min_count_to_compile,
|
||
const std::string & additional_compiler_flags,
|
||
CodeGenerator get_code,
|
||
ReadyCallback on_ready)
|
||
{
|
||
HashedKey hashed_key = getHash(key);
|
||
|
||
std::lock_guard<std::mutex> lock(mutex);
|
||
|
||
UInt32 count = ++counts[hashed_key];
|
||
|
||
/// Есть готовая открытая библиотека? Или, если библиотека в процессе компиляции, там будет nullptr.
|
||
Libraries::iterator it = libraries.find(hashed_key);
|
||
if (libraries.end() != it)
|
||
{
|
||
if (!it->second)
|
||
LOG_INFO(log, "Library " << hashedKeyToFileName(hashed_key) << " is already compiling or compilation was failed.");
|
||
|
||
/// TODO В этом случае, после окончания компиляции, не будет дёрнут колбэк.
|
||
|
||
return it->second;
|
||
}
|
||
|
||
/// Есть файл с библиотекой, оставшийся от предыдущего запуска?
|
||
std::string file_name = hashedKeyToFileName(hashed_key);
|
||
if (files.count(file_name))
|
||
{
|
||
std::string so_file_path = path + '/' + file_name + ".so";
|
||
LOG_INFO(log, "Loading existing library " << so_file_path);
|
||
|
||
SharedLibraryPtr lib(new SharedLibrary(so_file_path));
|
||
libraries[hashed_key] = lib;
|
||
return lib;
|
||
}
|
||
|
||
/// Достигнуто ли min_count_to_compile?
|
||
if (count >= min_count_to_compile)
|
||
{
|
||
/// Значение min_count_to_compile, равное нулю, обозначает необходимость синхронной компиляции.
|
||
|
||
/// Есть ли свободные потоки.
|
||
if (min_count_to_compile == 0 || pool.active() < pool.size())
|
||
{
|
||
/// Обозначает, что библиотека в процессе компиляции.
|
||
libraries[hashed_key] = nullptr;
|
||
|
||
LOG_INFO(log, "Compiling code " << file_name << ", key: " << key);
|
||
|
||
if (min_count_to_compile == 0)
|
||
{
|
||
{
|
||
ext::unlock_guard<std::mutex> unlock(mutex);
|
||
compile(hashed_key, file_name, additional_compiler_flags, get_code, on_ready);
|
||
}
|
||
|
||
return libraries[hashed_key];
|
||
}
|
||
else
|
||
{
|
||
pool.schedule([=]
|
||
{
|
||
try
|
||
{
|
||
compile(hashed_key, file_name, additional_compiler_flags, get_code, on_ready);
|
||
}
|
||
catch (...)
|
||
{
|
||
tryLogCurrentException("Compiler");
|
||
}
|
||
});
|
||
}
|
||
}
|
||
else
|
||
LOG_INFO(log, "All threads are busy.");
|
||
}
|
||
|
||
return nullptr;
|
||
}
|
||
|
||
|
||
void Compiler::compile(
|
||
HashedKey hashed_key,
|
||
std::string file_name,
|
||
const std::string & additional_compiler_flags,
|
||
CodeGenerator get_code,
|
||
ReadyCallback on_ready)
|
||
{
|
||
ProfileEvents::increment(ProfileEvents::CompileAttempt);
|
||
|
||
std::string prefix = path + "/" + file_name;
|
||
std::string cpp_file_path = prefix + ".cpp";
|
||
std::string so_file_path = prefix + ".so";
|
||
std::string so_tmp_file_path = prefix + ".so.tmp";
|
||
|
||
{
|
||
WriteBufferFromFile out(cpp_file_path);
|
||
out << get_code();
|
||
}
|
||
|
||
std::stringstream command;
|
||
|
||
/// Слегка неудобно.
|
||
command <<
|
||
"LD_LIBRARY_PATH=" PATH_SHARE "/clickhouse/bin/"
|
||
" " INTERNAL_COMPILER_EXECUTABLE
|
||
" -B " PATH_SHARE "/clickhouse/bin/"
|
||
" " INTERNAL_COMPILER_FLAGS
|
||
#if INTERNAL_COMPILER_CUSTOM_ROOT
|
||
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/include/"
|
||
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/"
|
||
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/mysql/"
|
||
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/c++/*/"
|
||
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/x86_64-linux-gnu/"
|
||
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/x86_64-linux-gnu/c++/*/"
|
||
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/lib/clang/*/include/"
|
||
#endif
|
||
" -I " INTERNAL_COMPILER_HEADERS "/dbms/include/"
|
||
" -I " INTERNAL_COMPILER_HEADERS "/contrib/libcityhash/include/"
|
||
" -I " INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR
|
||
" -I " INTERNAL_Poco_Foundation_INCLUDE_DIR
|
||
" -I " INTERNAL_Poco_Util_INCLUDE_DIR
|
||
" -I " INTERNAL_Boost_INCLUDE_DIRS
|
||
" -I " INTERNAL_COMPILER_HEADERS "/libs/libcommon/include/"
|
||
" -I " INTERNAL_COMPILER_HEADERS "/libs/libmysqlxx/include/"
|
||
" " << additional_compiler_flags <<
|
||
" -o " << so_tmp_file_path << " " << cpp_file_path
|
||
<< " 2>&1 || echo Exit code: $?";
|
||
|
||
std::string compile_result;
|
||
|
||
{
|
||
auto process = ShellCommand::execute(command.str());
|
||
readStringUntilEOF(compile_result, process->out);
|
||
process->wait();
|
||
}
|
||
|
||
if (!compile_result.empty())
|
||
throw Exception("Cannot compile code:\n\n" + command.str() + "\n\n" + compile_result);
|
||
|
||
/// Если до этого была ошибка, то файл с кодом остаётся для возможности просмотра.
|
||
Poco::File(cpp_file_path).remove();
|
||
|
||
Poco::File(so_tmp_file_path).renameTo(so_file_path);
|
||
SharedLibraryPtr lib(new SharedLibrary(so_file_path));
|
||
|
||
{
|
||
std::lock_guard<std::mutex> lock(mutex);
|
||
libraries[hashed_key] = lib;
|
||
}
|
||
|
||
LOG_INFO(log, "Compiled code " << file_name);
|
||
ProfileEvents::increment(ProfileEvents::CompileSuccess);
|
||
|
||
on_ready(lib);
|
||
}
|
||
|
||
|
||
}
|