thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 17:41:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Embedded compiler: development [#CLICKHOUSE-3396].

Browse Source
This commit is contained in:
Alexey Milovidov 2017-11-12 08:14:39 +03:00
parent e846d62410
commit 5074301f8d
14 changed files with 1553 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -236,6 +236,7 @@ include (cmake/find_zookeeper.cmake)
include (cmake/find_re2.cmake)
include (cmake/find_rdkafka.cmake)
include (cmake/find_capnp.cmake)
include (cmake/find_llvm.cmake)
include (cmake/find_contrib_lib.cmake)
find_contrib_lib(cityhash)

81
cmake/find_llvm.cmake Normal file
View File

@ -0,0 +1,81 @@
option (USE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile' option for query execution" FALSE)
if (USE_EMBEDDED_COMPILER)
# Based on source code of YT.
# Authors: Ivan Puzyrevskiy, Alexey Lukyanchikov, Ruslan Savchenko.
# Find LLVM includes and libraries.
#
# LLVM_VERSION - LLVM version.
# LLVM_INCLUDE_DIRS - Directory containing LLVM headers.
# LLVM_LIBRARY_DIRS - Directory containing LLVM libraries.
# LLVM_CXXFLAGS - C++ compiler flags for files that include LLVM headers.
# LLVM_FOUND - True if LLVM was found.
# llvm_map_components_to_libraries - Maps LLVM used components to required libraries.
# Usage: llvm_map_components_to_libraries(REQUIRED_LLVM_LIBRARIES core jit interpreter native ...)
find_program(LLVM_CONFIG_EXECUTABLE
NAMES llvm-config
PATHS $ENV{LLVM_ROOT}/bin
)
mark_as_advanced(LLVM_CONFIG_EXECUTABLE)
if(NOT LLVM_CONFIG_EXECUTABLE)
message(FATAL_ERROR "Cannot find LLVM (looking for `llvm-config`). Please, provide LLVM_ROOT environment variable.")
else()
set(LLVM_FOUND TRUE)
execute_process(
COMMAND ${LLVM_CONFIG_EXECUTABLE} --version
OUTPUT_VARIABLE LLVM_VERSION
OUTPUT_STRIP_TRAILING_WHITESPACE)
if(LLVM_VERSION VERSION_LESS "5")
message(FATAL_ERROR "LLVM 5+ is required.")
endif()
execute_process(
COMMAND ${LLVM_CONFIG_EXECUTABLE} --includedir
OUTPUT_VARIABLE LLVM_INCLUDE_DIRS
OUTPUT_STRIP_TRAILING_WHITESPACE)
execute_process(
COMMAND ${LLVM_CONFIG_EXECUTABLE} --libdir
OUTPUT_VARIABLE LLVM_LIBRARY_DIRS
OUTPUT_STRIP_TRAILING_WHITESPACE)
execute_process(
COMMAND ${LLVM_CONFIG_EXECUTABLE} --cxxflags
OUTPUT_VARIABLE LLVM_CXXFLAGS
OUTPUT_STRIP_TRAILING_WHITESPACE)
# Get the link libs we need.
function(llvm_map_components_to_libraries RESULT)
execute_process(
COMMAND ${LLVM_CONFIG_EXECUTABLE} --libs ${ARGN}
OUTPUT_VARIABLE _tmp
OUTPUT_STRIP_TRAILING_WHITESPACE)
string(REPLACE " " ";" _libs_module "${_tmp}")
message(STATUS "LLVM Libraries for '${ARGN}': ${_libs_module}")
execute_process(
COMMAND ${LLVM_CONFIG_EXECUTABLE} --system-libs ${ARGN}
OUTPUT_VARIABLE _libs_system
OUTPUT_STRIP_TRAILING_WHITESPACE)
string(REPLACE "\n" " " _libs_system "${_libs_system}")
string(REPLACE " " " " _libs_system "${_libs_system}")
string(REPLACE " " ";" _libs_system "${_libs_system}")
set(${RESULT} ${_libs_module} ${_libs_system} PARENT_SCOPE)
endfunction(llvm_map_components_to_libraries)
message(STATUS "LLVM Include Directory: ${LLVM_INCLUDE_DIRS}")
message(STATUS "LLVM Library Directory: ${LLVM_LIBRARY_DIRS}")
message(STATUS "LLVM C++ Compiler: ${LLVM_CXXFLAGS}")
endif()
endif()

1
dbms/src/Common/config.h.in
View File

@ -8,6 +8,7 @@
#cmakedefine01 USE_VECTORCLASS
#cmakedefine01 USE_RDKAFKA
#cmakedefine01 USE_CAPNP
#cmakedefine01 USE_EMBEDDED_COMPILER
#cmakedefine01 Poco_DataODBC_FOUND
#cmakedefine01 Poco_MongoDB_FOUND
#cmakedefine01 Poco_NetSSL_FOUND

7
dbms/src/Interpreters/CMakeLists.txt
View File

@ -5,10 +5,9 @@ else ()
set (PATH_SHARE "/usr/share" CACHE STRING "")
endif ()
if (INTERNAL_COMPILER_SAME)
set (INTERNAL_COMPILER_EXECUTABLE "${CMAKE_CXX_COMPILER}" CACHE STRING "")
else ()
set (INTERNAL_COMPILER_EXECUTABLE "${PATH_SHARE}/clickhouse/bin/clang" CACHE STRING "")
if (USE_EMBEDDED_COMPILER)
set (INTERNAL_COMPILER_EXECUTABLE "clickhouse-clang" CACHE STRING "")
set (INTERNAL_LINKER_EXECUTABLE "clickhouse-lld" CACHE STRING "")
endif ()
set (INTERNAL_COMPILER_NO_WARNING OFF CACHE BOOL "")

5
dbms/src/Interpreters/Compiler.cpp
View File

@ -217,9 +217,8 @@ void Compiler::compile(
/// Slightly unconvenient.
command <<
"LD_LIBRARY_PATH=" PATH_SHARE "/clickhouse/bin/"
" " INTERNAL_COMPILER_EXECUTABLE
" -B " PATH_SHARE "/clickhouse/bin/"
INTERNAL_COMPILER_EXECUTABLE
" -fuse-ld=" INTERNAL_LINKER_EXECUTABLE
" " INTERNAL_COMPILER_FLAGS
#if INTERNAL_COMPILER_CUSTOM_ROOT
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/include/"

1
dbms/src/Interpreters/config_compile.h.in
View File

@ -3,6 +3,7 @@
#cmakedefine PATH_SHARE "@PATH_SHARE@"
#cmakedefine INTERNAL_COMPILER_FLAGS "@INTERNAL_COMPILER_FLAGS@"
#cmakedefine INTERNAL_COMPILER_EXECUTABLE "@INTERNAL_COMPILER_EXECUTABLE@"
#cmakedefine INTERNAL_LINKER_EXECUTABLE "@INTERNAL_LINKER_EXECUTABLE@"
#cmakedefine INTERNAL_COMPILER_HEADERS "@INTERNAL_COMPILER_HEADERS@"
#cmakedefine INTERNAL_COMPILER_HEADERS_ROOT "@INTERNAL_COMPILER_HEADERS_ROOT@"
#cmakedefine01 INTERNAL_COMPILER_CUSTOM_ROOT

12
dbms/src/Server/CMakeLists.txt
View File

@ -43,7 +43,6 @@ target_link_libraries (clickhouse-compressor-lib dbms ${Boost_PROGRAM_OPTIONS_LI
add_library (clickhouse-format Format.cpp)
target_link_libraries (clickhouse-format dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
add_executable(clickhouse main.cpp)
target_include_directories(clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
target_link_libraries(clickhouse
@ -55,6 +54,12 @@ target_link_libraries(clickhouse
clickhouse-extract-from-config
clickhouse-compressor-lib
clickhouse-format)
if (USE_EMBEDDED_COMPILER)
add_subdirectory (Compiler)
target_link_libraries (clickhouse clickhouse-compiler)
endif ()
INSTALL(TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
# make symbolic links to concrete clickhouse applications
macro(install_symlink_to_clickhouse app)
@ -68,6 +73,11 @@ install_symlink_to_clickhouse(clickhouse-benchmark)
install_symlink_to_clickhouse(clickhouse-performance-test)
install_symlink_to_clickhouse(clickhouse-format)
if (USE_EMBEDDED_COMPILER)
install_symlink_to_clickhouse(clickhouse-clang)
install_symlink_to_clickhouse(clickhouse-lld)
endif ()
INSTALL(
FILES config.xml users.xml
DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server

37
dbms/src/Server/Compiler/CMakeLists.txt Normal file
View File

@ -0,0 +1,37 @@
add_library(clickhouse-compiler
driver.cpp
cc1_main.cpp
cc1as_main.cpp
lld.cpp)
target_compile_options(clickhouse-compiler PRIVATE -fno-rtti)
# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.
target_link_libraries(clickhouse-compiler
clangBasic clangCodeGen clangDriver clangFrontend clangFrontendTool
clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend
clangParse
clangSerialization
clangSema clangEdit clangStaticAnalyzerCheckers clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic
lldCOFF
lldDriver
lldELF
lldMinGW
lldMachO
lldReaderWriter
lldYAML
lldCommon
lldCore
LLVMAArch64CodeGen LLVMAArch64AsmPrinter LLVMAArch64AsmParser LLVMAArch64Desc LLVMAArch64Info LLVMAArch64Disassembler LLVMAMDGPUCodeGen LLVMAMDGPUAsmPrinter LLVMAMDGPUAsmParser LLVMAMDGPUDesc LLVMAMDGPUInfo LLVMAMDGPUDisassembler LLVMARMCodeGen LLVMARMAsmPrinter LLVMARMAsmParser LLVMARMDesc LLVMARMInfo LLVMARMDisassembler LLVMBPFCodeGen LLVMBPFAsmPrinter LLVMBPFDesc LLVMBPFInfo LLVMBPFDisassembler LLVMHexagonCodeGen LLVMHexagonAsmParser LLVMHexagonDesc LLVMHexagonInfo LLVMHexagonDisassembler LLVMLanaiCodeGen LLVMLanaiAsmParser LLVMLanaiDesc LLVMLanaiInfo LLVMLanaiDisassembler LLVMMipsCodeGen LLVMMipsAsmPrinter LLVMMipsAsmParser LLVMMipsDesc LLVMMipsInfo LLVMMipsDisassembler LLVMMSP430CodeGen LLVMMSP430AsmPrinter LLVMMSP430Desc LLVMMSP430Info LLVMNVPTXCodeGen LLVMNVPTXAsmPrinter LLVMNVPTXDesc LLVMNVPTXInfo LLVMPowerPCCodeGen LLVMPowerPCAsmPrinter LLVMPowerPCAsmParser LLVMPowerPCDesc LLVMPowerPCInfo LLVMPowerPCDisassembler LLVMRISCVCodeGen LLVMRISCVDesc LLVMRISCVInfo LLVMSparcCodeGen LLVMSparcAsmPrinter LLVMSparcAsmParser LLVMSparcDesc LLVMSparcInfo LLVMSparcDisassembler LLVMSystemZCodeGen LLVMSystemZAsmPrinter LLVMSystemZAsmParser LLVMSystemZDesc LLVMSystemZInfo LLVMSystemZDisassembler LLVMX86CodeGen LLVMX86AsmPrinter LLVMX86AsmParser LLVMX86Desc LLVMX86Info LLVMX86Disassembler LLVMXCoreCodeGen LLVMXCoreAsmPrinter LLVMXCoreDesc LLVMXCoreInfo LLVMXCoreDisassembler LLVMAnalysis LLVMCodeGen LLVMCore LLVMipo LLVMInstCombine LLVMInstrumentation LLVMMC LLVMMCParser LLVMObjCARCOpts LLVMOption LLVMScalarOpts LLVMSupport LLVMTransformUtils LLVMVectorize LLVMAArch64Desc LLVMAArch64AsmPrinter LLVMAArch64Info LLVMAArch64Utils LLVMAMDGPUDesc LLVMAMDGPUAsmPrinter LLVMAMDGPUInfo LLVMAMDGPUUtils LLVMARMDesc LLVMARMAsmPrinter LLVMARMInfo LLVMBPFAsmPrinter LLVMBPFInfo LLVMHexagonDesc LLVMHexagonInfo LLVMLanaiDesc LLVMLanaiInstPrinter LLVMLanaiInfo LLVMMipsAsmPrinter LLVMMipsInfo LLVMMSP430AsmPrinter LLVMNVPTXAsmPrinter LLVMPowerPCAsmPrinter LLVMPowerPCInfo LLVMSparcAsmPrinter LLVMSparcInfo LLVMSystemZDesc LLVMSystemZAsmPrinter LLVMSystemZInfo LLVMGlobalISel LLVMX86AsmPrinter LLVMX86Utils LLVMX86Info LLVMXCoreAsmPrinter LLVMAsmPrinter LLVMDebugInfoCodeView LLVMDebugInfoMSF LLVMSelectionDAG LLVMXCoreInfo LLVMMCDisassembler LLVMCoroutines LLVMCoverage LLVMLTO LLVMObjCARCOpts LLVMPasses LLVMCodeGen LLVMTarget LLVMipo LLVMBitWriter LLVMIRReader LLVMAsmParser LLVMLinker LLVMInstrumentation LLVMScalarOpts LLVMInstCombine LLVMVectorize LLVMTransformUtils LLVMAnalysis LLVMObject LLVMOption LLVMProfileData LLVMMCParser LLVMBitReader LLVMCore LLVMMC LLVMSupport LLVMDemangle
LLVMARMUtils LLVMBinaryFormat LLVMBPFAsmParser LLVMDebugInfoDWARF LLVMBinaryFormat LLVMWindowsManifest LLVMSupport LLVMDemangle
Polly
PollyISL
PollyPPCG
common
libtinfo.a
)

63
dbms/src/Server/Compiler/LICENSE.TXT Normal file
View File

@ -0,0 +1,63 @@
==============================================================================
LLVM Release License
==============================================================================
University of Illinois/NCSA
Open Source License
Copyright (c) 2007-2016 University of Illinois at Urbana-Champaign.
All rights reserved.
Developed by:
LLVM Team
University of Illinois at Urbana-Champaign
http://llvm.org
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal with
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimers.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimers in the
documentation and/or other materials provided with the distribution.
* Neither the names of the LLVM Team, University of Illinois at
Urbana-Champaign, nor the names of its contributors may be used to
endorse or promote products derived from this Software without specific
prior written permission.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
SOFTWARE.
==============================================================================
The LLVM software contains code written by third parties. Such software will
have its own individual LICENSE.TXT file in the directory in which it appears.
This file will describe the copyrights, license, and restrictions which apply
to that code.
The disclaimer of warranty in the University of Illinois Open Source License
applies to all code in the LLVM Distribution, and nothing in any of the
other licenses gives permission to use the names of the LLVM Team or the
University of Illinois to endorse or promote products derived from this
Software.
The following pieces of software have additional or alternate copyrights,
licenses, and/or restrictions:
Program Directory
------- ---------
<none yet>

239
dbms/src/Server/Compiler/cc1_main.cpp Normal file
View File

@ -0,0 +1,239 @@
//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the clang -cc1 functionality, which implements the
// core compiler functionality along with a number of additional tools for
// demonstration and testing purposes.
//
//===----------------------------------------------------------------------===//
#include "llvm/Option/Arg.h"
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
#include "clang/Config/config.h"
#include "clang/Driver/DriverDiagnostic.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "clang/Frontend/TextDiagnosticBuffer.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "clang/FrontendTool/Utils.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/LinkAllPasses.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
#include <cstdio>
#ifdef CLANG_HAVE_RLIMITS
#include <sys/resource.h>
#endif
using namespace clang;
using namespace llvm::opt;
//===----------------------------------------------------------------------===//
// Main driver
//===----------------------------------------------------------------------===//
static void LLVMErrorHandler(void *UserData, const std::string &Message,
bool GenCrashDiag) {
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
Diags.Report(diag::err_fe_error_backend) << Message;
// Run the interrupt handlers to make sure any special cleanups get done, in
// particular that we remove files registered with RemoveFileOnSignal.
llvm::sys::RunInterruptHandlers();
// We cannot recover from llvm errors. When reporting a fatal error, exit
// with status 70 to generate crash diagnostics. For BSD systems this is
// defined as an internal software error. Otherwise, exit with status 1.
exit(GenCrashDiag ? 70 : 1);
}
#ifdef LINK_POLLY_INTO_TOOLS
namespace polly {
void initializePollyPasses(llvm::PassRegistry &Registry);
}
#endif
#ifdef CLANG_HAVE_RLIMITS
// The amount of stack we think is "sufficient". If less than this much is
// available, we may be unable to reach our template instantiation depth
// limit and other similar limits.
// FIXME: Unify this with the stack we request when spawning a thread to build
// a module.
static const int kSufficientStack = 8 << 20;
#if defined(__linux__) && defined(__PIE__)
static size_t getCurrentStackAllocation() {
// If we can't compute the current stack usage, allow for 512K of command
// line arguments and environment.
size_t Usage = 512 * 1024;
if (FILE *StatFile = fopen("/proc/self/stat", "r")) {
// We assume that the stack extends from its current address to the end of
// the environment space. In reality, there is another string literal (the
// program name) after the environment, but this is close enough (we only
// need to be within 100K or so).
unsigned long StackPtr, EnvEnd;
// Disable silly GCC -Wformat warning that complains about length
// modifiers on ignored format specifiers. We want to retain these
// for documentation purposes even though they have no effect.
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wformat"
#endif
if (fscanf(StatFile,
"%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu "
"%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu "
"%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d "
"%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d",
&StackPtr, &EnvEnd) == 2) {
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic pop
#endif
Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd;
}
fclose(StatFile);
}
return Usage;
}
#include <alloca.h>
LLVM_ATTRIBUTE_NOINLINE
static void ensureStackAddressSpace(int ExtraChunks = 0) {
// Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary
// relatively close to the stack (they are only guaranteed to be 128MiB
// apart). This results in crashes if we happen to heap-allocate more than
// 128MiB before we reach our stack high-water mark.
//
// To avoid these crashes, ensure that we have sufficient virtual memory
// pages allocated before we start running.
size_t Curr = getCurrentStackAllocation();
const int kTargetStack = kSufficientStack - 256 * 1024;
if (Curr < kTargetStack) {
volatile char *volatile Alloc =
static_cast<volatile char *>(alloca(kTargetStack - Curr));
Alloc[0] = 0;
Alloc[kTargetStack - Curr - 1] = 0;
}
}
#else
static void ensureStackAddressSpace() {}
#endif
/// Attempt to ensure that we have at least 8MiB of usable stack space.
static void ensureSufficientStack() {
struct rlimit rlim;
if (getrlimit(RLIMIT_STACK, &rlim) != 0)
return;
// Increase the soft stack limit to our desired level, if necessary and
// possible.
if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) {
// Try to allocate sufficient stack.
if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack)
rlim.rlim_cur = kSufficientStack;
else if (rlim.rlim_cur == rlim.rlim_max)
return;
else
rlim.rlim_cur = rlim.rlim_max;
if (setrlimit(RLIMIT_STACK, &rlim) != 0 ||
rlim.rlim_cur != kSufficientStack)
return;
}
// We should now have a stack of size at least kSufficientStack. Ensure
// that we can actually use that much, if necessary.
ensureStackAddressSpace();
}
#else
static void ensureSufficientStack() {}
#endif
int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
ensureSufficientStack();
std::unique_ptr<CompilerInstance> Clang(new CompilerInstance());
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
// Register the support for object-file-wrapped Clang modules.
auto PCHOps = Clang->getPCHContainerOperations();
PCHOps->registerWriter(llvm::make_unique<ObjectFilePCHContainerWriter>());
PCHOps->registerReader(llvm::make_unique<ObjectFilePCHContainerReader>());
// Initialize targets first, so that --version shows registered targets.
llvm::InitializeAllTargets();
llvm::InitializeAllTargetMCs();
llvm::InitializeAllAsmPrinters();
llvm::InitializeAllAsmParsers();
#ifdef LINK_POLLY_INTO_TOOLS
llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry();
polly::initializePollyPasses(Registry);
#endif
// Buffer diagnostics from argument parsing so that we can output them using a
// well formed diagnostic object.
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer;
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer);
bool Success = CompilerInvocation::CreateFromArgs(
Clang->getInvocation(), Argv.begin(), Argv.end(), Diags);
// Infer the builtin include path if unspecified.
if (Clang->getHeaderSearchOpts().UseBuiltinIncludes &&
Clang->getHeaderSearchOpts().ResourceDir.empty())
Clang->getHeaderSearchOpts().ResourceDir =
CompilerInvocation::GetResourcesPath(Argv0, MainAddr);
// Create the actual diagnostics engine.
Clang->createDiagnostics();
if (!Clang->hasDiagnostics())
return 1;
// Set an error handler, so that any LLVM backend diagnostics go through our
// error handler.
llvm::install_fatal_error_handler(LLVMErrorHandler,
static_cast<void*>(&Clang->getDiagnostics()));
DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics());
if (!Success)
return 1;
// Execute the frontend actions.
Success = ExecuteCompilerInvocation(Clang.get());
// If any timers were active but haven't been destroyed yet, print their
// results now. This happens in -disable-free mode.
llvm::TimerGroup::printAll(llvm::errs());
// Our error handler depends on the Diagnostics object, which we're
// potentially about to delete. Uninstall the handler now so that any
// later errors use the default handling behavior instead.
llvm::remove_fatal_error_handler();
// When running with -disable-free, don't do any destruction or shutdown.
if (Clang->getFrontendOpts().DisableFree) {
BuryPointer(std::move(Clang));
return !Success;
}
return !Success;
}

541
dbms/src/Server/Compiler/cc1as_main.cpp Normal file
View File

@ -0,0 +1,541 @@
//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the clang -cc1as functionality, which implements
// the direct interface to the LLVM MC based assembler.
//
//===----------------------------------------------------------------------===//
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Driver/DriverDiagnostic.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Triple.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/MC/MCAsmBackend.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCCodeEmitter.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCInstrInfo.h"
#include "llvm/MC/MCObjectFileInfo.h"
#include "llvm/MC/MCParser/MCAsmParser.h"
#include "llvm/MC/MCParser/MCTargetAsmParser.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSubtargetInfo.h"
#include "llvm/MC/MCTargetOptions.h"
#include "llvm/Option/Arg.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/FormattedStream.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
#include <memory>
#include <system_error>
using namespace clang;
using namespace clang::driver;
using namespace clang::driver::options;
using namespace llvm;
using namespace llvm::opt;
namespace {
/// \brief Helper class for representing a single invocation of the assembler.
struct AssemblerInvocation {
/// @name Target Options
/// @{
/// The name of the target triple to assemble for.
std::string Triple;
/// If given, the name of the target CPU to determine which instructions
/// are legal.
std::string CPU;
/// The list of target specific features to enable or disable -- this should
/// be a list of strings starting with '+' or '-'.
std::vector<std::string> Features;
/// The list of symbol definitions.
std::vector<std::string> SymbolDefs;
/// @}
/// @name Language Options
/// @{
std::vector<std::string> IncludePaths;
unsigned NoInitialTextSection : 1;
unsigned SaveTemporaryLabels : 1;
unsigned GenDwarfForAssembly : 1;
unsigned RelaxELFRelocations : 1;
unsigned DwarfVersion;
std::string DwarfDebugFlags;
std::string DwarfDebugProducer;
std::string DebugCompilationDir;
llvm::DebugCompressionType CompressDebugSections =
llvm::DebugCompressionType::None;
std::string MainFileName;
/// @}
/// @name Frontend Options
/// @{
std::string InputFile;
std::vector<std::string> LLVMArgs;
std::string OutputPath;
enum FileType {
FT_Asm, ///< Assembly (.s) output, transliterate mode.
FT_Null, ///< No output, for timing purposes.
FT_Obj ///< Object file output.
};
FileType OutputType;
unsigned ShowHelp : 1;
unsigned ShowVersion : 1;
/// @}
/// @name Transliterate Options
/// @{
unsigned OutputAsmVariant;
unsigned ShowEncoding : 1;
unsigned ShowInst : 1;
/// @}
/// @name Assembler Options
/// @{
unsigned RelaxAll : 1;
unsigned NoExecStack : 1;
unsigned FatalWarnings : 1;
unsigned IncrementalLinkerCompatible : 1;
/// The name of the relocation model to use.
std::string RelocationModel;
/// @}
public:
AssemblerInvocation() {
Triple = "";
NoInitialTextSection = 0;
InputFile = "-";
OutputPath = "-";
OutputType = FT_Asm;
OutputAsmVariant = 0;
ShowInst = 0;
ShowEncoding = 0;
RelaxAll = 0;
NoExecStack = 0;
FatalWarnings = 0;
IncrementalLinkerCompatible = 0;
DwarfVersion = 0;
}
static bool CreateFromArgs(AssemblerInvocation &Res,
ArrayRef<const char *> Argv,
DiagnosticsEngine &Diags);
};
}
bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
ArrayRef<const char *> Argv,
DiagnosticsEngine &Diags) {
bool Success = true;
// Parse the arguments.
std::unique_ptr<OptTable> OptTbl(createDriverOptTable());
const unsigned IncludedFlagsBitmask = options::CC1AsOption;
unsigned MissingArgIndex, MissingArgCount;
InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount,
IncludedFlagsBitmask);
// Check for missing argument error.
if (MissingArgCount) {
Diags.Report(diag::err_drv_missing_argument)
<< Args.getArgString(MissingArgIndex) << MissingArgCount;
Success = false;
}
// Issue errors on unknown arguments.
for (const Arg *A : Args.filtered(OPT_UNKNOWN)) {
Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args);
Success = false;
}
// Construct the invocation.
// Target Options
Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple));
Opts.CPU = Args.getLastArgValue(OPT_target_cpu);
Opts.Features = Args.getAllArgValues(OPT_target_feature);
// Use the default target triple if unspecified.
if (Opts.Triple.empty())
Opts.Triple = llvm::sys::getDefaultTargetTriple();
// Language Options
Opts.IncludePaths = Args.getAllArgValues(OPT_I);
Opts.NoInitialTextSection = Args.hasArg(OPT_n);
Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels);
// Any DebugInfoKind implies GenDwarfForAssembly.
Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ);
if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections,
OPT_compress_debug_sections_EQ)) {
if (A->getOption().getID() == OPT_compress_debug_sections) {
// TODO: be more clever about the compression type auto-detection
Opts.CompressDebugSections = llvm::DebugCompressionType::GNU;
} else {
Opts.CompressDebugSections =
llvm::StringSwitch<llvm::DebugCompressionType>(A->getValue())
.Case("none", llvm::DebugCompressionType::None)
.Case("zlib", llvm::DebugCompressionType::Z)
.Case("zlib-gnu", llvm::DebugCompressionType::GNU)
.Default(llvm::DebugCompressionType::None);
}
}
Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations);
Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags);
Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags);
Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer);
Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir);
Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name);
// Frontend Options
if (Args.hasArg(OPT_INPUT)) {
bool First = true;
for (const Arg *A : Args.filtered(OPT_INPUT)) {
if (First) {
Opts.InputFile = A->getValue();
First = false;
} else {
Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args);
Success = false;
}
}
}
Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
Opts.OutputPath = Args.getLastArgValue(OPT_o);
if (Arg *A = Args.getLastArg(OPT_filetype)) {
StringRef Name = A->getValue();
unsigned OutputType = StringSwitch<unsigned>(Name)
.Case("asm", FT_Asm)
.Case("null", FT_Null)
.Case("obj", FT_Obj)
.Default(~0U);
if (OutputType == ~0U) {
Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name;
Success = false;
} else
Opts.OutputType = FileType(OutputType);
}
Opts.ShowHelp = Args.hasArg(OPT_help);
Opts.ShowVersion = Args.hasArg(OPT_version);
// Transliterate Options
Opts.OutputAsmVariant =
getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags);
Opts.ShowEncoding = Args.hasArg(OPT_show_encoding);
Opts.ShowInst = Args.hasArg(OPT_show_inst);
// Assemble Options
Opts.RelaxAll = Args.hasArg(OPT_mrelax_all);
Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack);
Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings);
Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic");
Opts.IncrementalLinkerCompatible =
Args.hasArg(OPT_mincremental_linker_compatible);
Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym);
return Success;
}
static std::unique_ptr<raw_fd_ostream>
getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags,
bool Binary) {
if (Opts.OutputPath.empty())
Opts.OutputPath = "-";
// Make sure that the Out file gets unlinked from the disk if we get a
// SIGINT.
if (Opts.OutputPath != "-")
sys::RemoveFileOnSignal(Opts.OutputPath);
std::error_code EC;
auto Out = llvm::make_unique<raw_fd_ostream>(
Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text));
if (EC) {
Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath
<< EC.message();
return nullptr;
}
return Out;
}
static bool ExecuteAssembler(AssemblerInvocation &Opts,
DiagnosticsEngine &Diags) {
// Get the target specific parser.
std::string Error;
const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error);
if (!TheTarget)
return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer =
MemoryBuffer::getFileOrSTDIN(Opts.InputFile);
if (std::error_code EC = Buffer.getError()) {
Error = EC.message();
return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile;
}
SourceMgr SrcMgr;
// Tell SrcMgr about this buffer, which is what the parser will pick up.
SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc());
// Record the location of the include directories so that the lexer can find
// it later.
SrcMgr.setIncludeDirs(Opts.IncludePaths);
std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(Opts.Triple));
assert(MRI && "Unable to create target register info!");
std::unique_ptr<MCAsmInfo> MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple));
assert(MAI && "Unable to create target asm info!");
// Ensure MCAsmInfo initialization occurs before any use, otherwise sections
// may be created with a combination of default and explicit settings.
MAI->setCompressDebugSections(Opts.CompressDebugSections);
MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations);
bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj;
std::unique_ptr<raw_fd_ostream> FDOS = getOutputStream(Opts, Diags, IsBinary);
if (!FDOS)
return true;
// FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and
// MCObjectFileInfo needs a MCContext reference in order to initialize itself.
std::unique_ptr<MCObjectFileInfo> MOFI(new MCObjectFileInfo());
MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr);
bool PIC = false;
if (Opts.RelocationModel == "static") {
PIC = false;
} else if (Opts.RelocationModel == "pic") {
PIC = true;
} else {
assert(Opts.RelocationModel == "dynamic-no-pic" &&
"Invalid PIC model!");
PIC = false;
}
MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx);
if (Opts.SaveTemporaryLabels)
Ctx.setAllowTemporaryLabels(false);
if (Opts.GenDwarfForAssembly)
Ctx.setGenDwarfForAssembly(true);
if (!Opts.DwarfDebugFlags.empty())
Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags));
if (!Opts.DwarfDebugProducer.empty())
Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer));
if (!Opts.DebugCompilationDir.empty())
Ctx.setCompilationDir(Opts.DebugCompilationDir);
if (!Opts.MainFileName.empty())
Ctx.setMainFileName(StringRef(Opts.MainFileName));
Ctx.setDwarfVersion(Opts.DwarfVersion);
// Build up the feature string from the target feature list.
std::string FS;
if (!Opts.Features.empty()) {
FS = Opts.Features[0];
for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i)
FS += "," + Opts.Features[i];
}
std::unique_ptr<MCStreamer> Str;
std::unique_ptr<MCInstrInfo> MCII(TheTarget->createMCInstrInfo());
std::unique_ptr<MCSubtargetInfo> STI(
TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS));
raw_pwrite_stream *Out = FDOS.get();
std::unique_ptr<buffer_ostream> BOS;
// FIXME: There is a bit of code duplication with addPassesToEmitFile.
if (Opts.OutputType == AssemblerInvocation::FT_Asm) {
MCInstPrinter *IP = TheTarget->createMCInstPrinter(
llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI);
MCCodeEmitter *CE = nullptr;
MCAsmBackend *MAB = nullptr;
if (Opts.ShowEncoding) {
CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx);
MCTargetOptions Options;
MAB = TheTarget->createMCAsmBackend(*MRI, Opts.Triple, Opts.CPU, Options);
}
auto FOut = llvm::make_unique<formatted_raw_ostream>(*Out);
Str.reset(TheTarget->createAsmStreamer(
Ctx, std::move(FOut), /*asmverbose*/ true,
/*useDwarfDirectory*/ true, IP, CE, MAB, Opts.ShowInst));
} else if (Opts.OutputType == AssemblerInvocation::FT_Null) {
Str.reset(createNullStreamer(Ctx));
} else {
assert(Opts.OutputType == AssemblerInvocation::FT_Obj &&
"Invalid file type!");
if (!FDOS->supportsSeeking()) {
BOS = make_unique<buffer_ostream>(*FDOS);
Out = BOS.get();
}
MCCodeEmitter *CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx);
MCTargetOptions Options;
MCAsmBackend *MAB = TheTarget->createMCAsmBackend(*MRI, Opts.Triple,
Opts.CPU, Options);
Triple T(Opts.Triple);
Str.reset(TheTarget->createMCObjectStreamer(
T, Ctx, std::unique_ptr<MCAsmBackend>(MAB), *Out, std::unique_ptr<MCCodeEmitter>(CE), *STI,
Opts.RelaxAll, Opts.IncrementalLinkerCompatible,
/*DWARFMustBeAtTheEnd*/ true));
Str.get()->InitSections(Opts.NoExecStack);
}
bool Failed = false;
std::unique_ptr<MCAsmParser> Parser(
createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI));
// FIXME: init MCTargetOptions from sanitizer flags here.
MCTargetOptions Options;
std::unique_ptr<MCTargetAsmParser> TAP(
TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options));
if (!TAP)
Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
// Set values for symbols, if any.
for (auto &S : Opts.SymbolDefs) {
auto Pair = StringRef(S).split('=');
auto Sym = Pair.first;
auto Val = Pair.second;
int64_t Value;
// We have already error checked this in the driver.
Val.getAsInteger(0, Value);
Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value);
}
if (!Failed) {
Parser->setTargetParser(*TAP.get());
Failed = Parser->Run(Opts.NoInitialTextSection);
}
// Close Streamer first.
// It might have a reference to the output stream.
Str.reset();
// Close the output stream early.
BOS.reset();
FDOS.reset();
// Delete output file if there were errors.
if (Failed && Opts.OutputPath != "-")
sys::fs::remove(Opts.OutputPath);
return Failed;
}
static void LLVMErrorHandler(void *UserData, const std::string &Message,
bool GenCrashDiag) {
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
Diags.Report(diag::err_fe_error_backend) << Message;
// We cannot recover from llvm errors.
exit(1);
}
int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
// Initialize targets and assembly printers/parsers.
InitializeAllTargetInfos();
InitializeAllTargetMCs();
InitializeAllAsmParsers();
// Construct our diagnostic client.
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
TextDiagnosticPrinter *DiagClient
= new TextDiagnosticPrinter(errs(), &*DiagOpts);
DiagClient->setPrefix("clang -cc1as");
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
// Set an error handler, so that any LLVM backend diagnostics go through our
// error handler.
ScopedFatalErrorHandler FatalErrorHandler
(LLVMErrorHandler, static_cast<void*>(&Diags));
// Parse the arguments.
AssemblerInvocation Asm;
if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags))
return 1;
if (Asm.ShowHelp) {
std::unique_ptr<OptTable> Opts(driver::createDriverOptTable());
Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler",
/*Include=*/driver::options::CC1AsOption, /*Exclude=*/0,
/*ShowAllAliases=*/false);
return 0;
}
// Honor -version.
//
// FIXME: Use a better -version message?
if (Asm.ShowVersion) {
llvm::cl::PrintVersionMessage();
return 0;
}
// Honor -mllvm.
//
// FIXME: Remove this, one day.
if (!Asm.LLVMArgs.empty()) {
unsigned NumArgs = Asm.LLVMArgs.size();
auto Args = llvm::make_unique<const char*[]>(NumArgs + 2);
Args[0] = "clang (LLVM option parsing)";
for (unsigned i = 0; i != NumArgs; ++i)
Args[i + 1] = Asm.LLVMArgs[i].c_str();
Args[NumArgs + 1] = nullptr;
llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get());
}
// Execute the invocation, unless there were parsing errors.
bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags);
// If any timers were active but haven't been destroyed yet, print their
// results now.
TimerGroup::printAll(errs());
return !!Failed;
}

520
dbms/src/Server/Compiler/driver.cpp Normal file
View File

@ -0,0 +1,520 @@
//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the clang driver; it is a thin wrapper
// for functionality in the Driver clang library.
//
//===----------------------------------------------------------------------===//
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/Driver.h"
#include "clang/Driver/DriverDiagnostic.h"
#include "clang/Driver/Options.h"
#include "clang/Driver/ToolChain.h"
#include "clang/Frontend/ChainedDiagnosticConsumer.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/SerializedDiagnosticPrinter.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Option/Option.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/PrettyStackTrace.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Program.h"
#include "llvm/Support/Regex.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/StringSaver.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
#include <memory>
#include <set>
#include <system_error>
using namespace clang;
using namespace clang::driver;
using namespace llvm::opt;
std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) {
if (!CanonicalPrefixes) {
SmallString<128> ExecutablePath(Argv0);
// Do a PATH lookup if Argv0 isn't a valid path.
if (!llvm::sys::fs::exists(ExecutablePath))
if (llvm::ErrorOr<std::string> P =
llvm::sys::findProgramByName(ExecutablePath))
ExecutablePath = *P;
return ExecutablePath.str();
}
// This just needs to be some symbol in the binary; C++ doesn't
// allow taking the address of ::main however.
void *P = (void*) (intptr_t) GetExecutablePath;
return llvm::sys::fs::getMainExecutable(Argv0, P);
}
static const char *GetStableCStr(std::set<std::string> &SavedStrings,
StringRef S) {
return SavedStrings.insert(S).first->c_str();
}
/// ApplyQAOverride - Apply a list of edits to the input argument lists.
///
/// The input string is a space separate list of edits to perform,
/// they are applied in order to the input argument lists. Edits
/// should be one of the following forms:
///
/// '#': Silence information about the changes to the command line arguments.
///
/// '^': Add FOO as a new argument at the beginning of the command line.
///
/// '+': Add FOO as a new argument at the end of the command line.
///
/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command
/// line.
///
/// 'xOPTION': Removes all instances of the literal argument OPTION.
///
/// 'XOPTION': Removes all instances of the literal argument OPTION,
/// and the following argument.
///
/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox'
/// at the end of the command line.
///
/// \param OS - The stream to write edit information to.
/// \param Args - The vector of command line arguments.
/// \param Edit - The override command to perform.
/// \param SavedStrings - Set to use for storing string representations.
static void ApplyOneQAOverride(raw_ostream &OS,
SmallVectorImpl<const char*> &Args,
StringRef Edit,
std::set<std::string> &SavedStrings) {
// This does not need to be efficient.
if (Edit[0] == '^') {
const char *Str =
GetStableCStr(SavedStrings, Edit.substr(1));
OS << "### Adding argument " << Str << " at beginning\n";
Args.insert(Args.begin() + 1, Str);
} else if (Edit[0] == '+') {
const char *Str =
GetStableCStr(SavedStrings, Edit.substr(1));
OS << "### Adding argument " << Str << " at end\n";
Args.push_back(Str);
} else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") &&
Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) {
StringRef MatchPattern = Edit.substr(2).split('/').first;
StringRef ReplPattern = Edit.substr(2).split('/').second;
ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1);
for (unsigned i = 1, e = Args.size(); i != e; ++i) {
// Ignore end-of-line response file markers
if (Args[i] == nullptr)
continue;
std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]);
if (Repl != Args[i]) {
OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n";
Args[i] = GetStableCStr(SavedStrings, Repl);
}
}
} else if (Edit[0] == 'x' || Edit[0] == 'X') {
auto Option = Edit.substr(1);
for (unsigned i = 1; i < Args.size();) {
if (Option == Args[i]) {
OS << "### Deleting argument " << Args[i] << '\n';
Args.erase(Args.begin() + i);
if (Edit[0] == 'X') {
if (i < Args.size()) {
OS << "### Deleting argument " << Args[i] << '\n';
Args.erase(Args.begin() + i);
} else
OS << "### Invalid X edit, end of command line!\n";
}
} else
++i;
}
} else if (Edit[0] == 'O') {
for (unsigned i = 1; i < Args.size();) {
const char *A = Args[i];
// Ignore end-of-line response file markers
if (A == nullptr)
continue;
if (A[0] == '-' && A[1] == 'O' &&
(A[2] == '\0' ||
(A[3] == '\0' && (A[2] == 's' || A[2] == 'z' ||
('0' <= A[2] && A[2] <= '9'))))) {
OS << "### Deleting argument " << Args[i] << '\n';
Args.erase(Args.begin() + i);
} else
++i;
}
OS << "### Adding argument " << Edit << " at end\n";
Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str()));
} else {
OS << "### Unrecognized edit: " << Edit << "\n";
}
}
/// ApplyQAOverride - Apply a comma separate list of edits to the
/// input argument lists. See ApplyOneQAOverride.
static void ApplyQAOverride(SmallVectorImpl<const char*> &Args,
const char *OverrideStr,
std::set<std::string> &SavedStrings) {
raw_ostream *OS = &llvm::errs();
if (OverrideStr[0] == '#') {
++OverrideStr;
OS = &llvm::nulls();
}
*OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n";
// This does not need to be efficient.
const char *S = OverrideStr;
while (*S) {
const char *End = ::strchr(S, ' ');
if (!End)
End = S + strlen(S);
if (End != S)
ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings);
S = End;
if (*S != '\0')
++S;
}
}
extern int cc1_main(ArrayRef<const char *> Argv, const char *Argv0,
void *MainAddr);
extern int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0,
void *MainAddr);
static void insertTargetAndModeArgs(const ParsedClangName &NameParts,
SmallVectorImpl<const char *> &ArgVector,
std::set<std::string> &SavedStrings) {
// Put target and mode arguments at the start of argument list so that
// arguments specified in command line could override them. Avoid putting
// them at index 0, as an option like '-cc1' must remain the first.
auto InsertionPoint = ArgVector.begin();
if (InsertionPoint != ArgVector.end())
++InsertionPoint;
if (NameParts.DriverMode) {
// Add the mode flag to the arguments.
ArgVector.insert(InsertionPoint,
GetStableCStr(SavedStrings, NameParts.DriverMode));
}
if (NameParts.TargetIsValid) {
const char *arr[] = {"-target", GetStableCStr(SavedStrings,
NameParts.TargetPrefix)};
ArgVector.insert(InsertionPoint, std::begin(arr), std::end(arr));
}
}
static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver,
SmallVectorImpl<const char *> &Opts) {
llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts);
// The first instance of '#' should be replaced with '=' in each option.
for (const char *Opt : Opts)
if (char *NumberSignPtr = const_cast<char *>(::strchr(Opt, '#')))
*NumberSignPtr = '=';
}
static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) {
// Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE.
TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS");
if (TheDriver.CCPrintOptions)
TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE");
// Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE.
TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS");
if (TheDriver.CCPrintHeaders)
TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE");
// Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE.
TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS");
if (TheDriver.CCLogDiagnostics)
TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE");
}
static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient,
const std::string &Path) {
// If the clang binary happens to be named cl.exe for compatibility reasons,
// use clang-cl.exe as the prefix to avoid confusion between clang and MSVC.
StringRef ExeBasename(llvm::sys::path::filename(Path));
if (ExeBasename.equals_lower("cl.exe"))
ExeBasename = "clang-cl.exe";
DiagClient->setPrefix(ExeBasename);
}
// This lets us create the DiagnosticsEngine with a properly-filled-out
// DiagnosticOptions instance.
static DiagnosticOptions *
CreateAndPopulateDiagOpts(ArrayRef<const char *> argv) {
auto *DiagOpts = new DiagnosticOptions;
std::unique_ptr<OptTable> Opts(createDriverOptTable());
unsigned MissingArgIndex, MissingArgCount;
InputArgList Args =
Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount);
// We ignore MissingArgCount and the return value of ParseDiagnosticArgs.
// Any errors that would be diagnosed here will also be diagnosed later,
// when the DiagnosticsEngine actually exists.
(void)ParseDiagnosticArgs(*DiagOpts, Args);
return DiagOpts;
}
static void SetInstallDir(SmallVectorImpl<const char *> &argv,
Driver &TheDriver, bool CanonicalPrefixes) {
// Attempt to find the original path used to invoke the driver, to determine
// the installed path. We do this manually, because we want to support that
// path being a symlink.
SmallString<128> InstalledPath(argv[0]);
// Do a PATH lookup, if there are no directory components.
if (llvm::sys::path::filename(InstalledPath) == InstalledPath)
if (llvm::ErrorOr<std::string> Tmp = llvm::sys::findProgramByName(
llvm::sys::path::filename(InstalledPath.str())))
InstalledPath = *Tmp;
// FIXME: We don't actually canonicalize this, we just make it absolute.
if (CanonicalPrefixes)
llvm::sys::fs::make_absolute(InstalledPath);
StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath));
if (llvm::sys::fs::exists(InstalledPathParent))
TheDriver.setInstalledDir(InstalledPathParent);
}
static int ExecuteCC1Tool(ArrayRef<const char *> argv, StringRef Tool) {
void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath;
if (Tool == "")
return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP);
if (Tool == "as")
return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP);
// Reject unknown tools.
llvm::errs() << "error: unknown integrated tool '" << Tool << "'\n";
return 1;
}
int mainEntryClickHouseClang(int argc_, char **argv_) {
llvm::sys::PrintStackTraceOnErrorSignal(argv_[0]);
llvm::PrettyStackTraceProgram X(argc_, argv_);
llvm::llvm_shutdown_obj Y; // Call llvm_shutdown() on exit.
if (llvm::sys::Process::FixupStandardFileDescriptors())
return 1;
SmallVector<const char *, 256> argv;
llvm::SpecificBumpPtrAllocator<char> ArgAllocator;
std::error_code EC = llvm::sys::Process::GetArgumentVector(
argv, llvm::makeArrayRef(argv_, argc_), ArgAllocator);
if (EC) {
llvm::errs() << "error: couldn't get arguments: " << EC.message() << '\n';
return 1;
}
llvm::InitializeAllTargets();
auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]);
llvm::BumpPtrAllocator A;
llvm::StringSaver Saver(A);
// Parse response files using the GNU syntax, unless we're in CL mode. There
// are two ways to put clang in CL compatibility mode: argv[0] is either
// clang-cl or cl, or --driver-mode=cl is on the command line. The normal
// command line parsing can't happen until after response file parsing, so we
// have to manually search for a --driver-mode=cl argument the hard way.
// Finally, our -cc1 tools don't care which tokenization mode we use because
// response files written by clang will tokenize the same way in either mode.
bool ClangCLMode = false;
if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") ||
std::find_if(argv.begin(), argv.end(), [](const char *F) {
return F && strcmp(F, "--driver-mode=cl") == 0;
}) != argv.end()) {
ClangCLMode = true;
}
enum { Default, POSIX, Windows } RSPQuoting = Default;
for (const char *F : argv) {
if (strcmp(F, "--rsp-quoting=posix") == 0)
RSPQuoting = POSIX;
else if (strcmp(F, "--rsp-quoting=windows") == 0)
RSPQuoting = Windows;
}
// Determines whether we want nullptr markers in argv to indicate response
// files end-of-lines. We only use this for the /LINK driver argument with
// clang-cl.exe on Windows.
bool MarkEOLs = ClangCLMode;
llvm::cl::TokenizerCallback Tokenizer;
if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode))
Tokenizer = &llvm::cl::TokenizeWindowsCommandLine;
else
Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1"))
MarkEOLs = false;
llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs);
// Handle -cc1 integrated tools, even if -cc1 was expanded from a response
// file.
auto FirstArg = std::find_if(argv.begin() + 1, argv.end(),
[](const char *A) { return A != nullptr; });
if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) {
// If -cc1 came from a response file, remove the EOL sentinels.
if (MarkEOLs) {
auto newEnd = std::remove(argv.begin(), argv.end(), nullptr);
argv.resize(newEnd - argv.begin());
}
return ExecuteCC1Tool(argv, argv[1] + 4);
}
bool CanonicalPrefixes = true;
for (int i = 1, size = argv.size(); i < size; ++i) {
// Skip end-of-line response file markers
if (argv[i] == nullptr)
continue;
if (StringRef(argv[i]) == "-no-canonical-prefixes") {
CanonicalPrefixes = false;
break;
}
}
// Handle CL and _CL_ which permits additional command line options to be
// prepended or appended.
if (ClangCLMode) {
// Arguments in "CL" are prepended.
llvm::Optional<std::string> OptCL = llvm::sys::Process::GetEnv("CL");
if (OptCL.hasValue()) {
SmallVector<const char *, 8> PrependedOpts;
getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts);
// Insert right after the program name to prepend to the argument list.
argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end());
}
// Arguments in "_CL_" are appended.
llvm::Optional<std::string> Opt_CL_ = llvm::sys::Process::GetEnv("_CL_");
if (Opt_CL_.hasValue()) {
SmallVector<const char *, 8> AppendedOpts;
getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts);
// Insert at the end of the argument list to append.
argv.append(AppendedOpts.begin(), AppendedOpts.end());
}
}
std::set<std::string> SavedStrings;
// Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the
// scenes.
if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) {
// FIXME: Driver shouldn't take extra initial argument.
ApplyQAOverride(argv, OverrideStr, SavedStrings);
}
std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes);
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts =
CreateAndPopulateDiagOpts(argv);
TextDiagnosticPrinter *DiagClient
= new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
FixupDiagPrefixExeName(DiagClient, Path);
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
if (!DiagOpts->DiagnosticSerializationFile.empty()) {
auto SerializedConsumer =
clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile,
&*DiagOpts, /*MergeChildRecords=*/true);
Diags.setClient(new ChainedDiagnosticConsumer(
Diags.takeClient(), std::move(SerializedConsumer)));
}
ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false);
Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags);
SetInstallDir(argv, TheDriver, CanonicalPrefixes);
TheDriver.setTargetAndMode(TargetAndMode);
insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings);
SetBackdoorDriverOutputsFromEnvVars(TheDriver);
std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(argv));
int Res = 1;
if (C && !C->containsError()) {
SmallVector<std::pair<int, const Command *>, 4> FailingCommands;
Res = TheDriver.ExecuteCompilation(*C, FailingCommands);
// Force a crash to test the diagnostics.
if (TheDriver.GenReproducer) {
Diags.Report(diag::err_drv_force_crash)
<< !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH");
// Pretend that every command failed.
FailingCommands.clear();
for (const auto &J : C->getJobs())
if (const Command *C = dyn_cast<Command>(&J))
FailingCommands.push_back(std::make_pair(-1, C));
}
for (const auto &P : FailingCommands) {
int CommandRes = P.first;
const Command *FailingCommand = P.second;
if (!Res)
Res = CommandRes;
// If result status is < 0, then the driver command signalled an error.
// If result status is 70, then the driver command reported a fatal error.
// On Windows, abort will return an exit code of 3. In these cases,
// generate additional diagnostic information if possible.
bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70;
#ifdef LLVM_ON_WIN32
DiagnoseCrash |= CommandRes == 3;
#endif
if (DiagnoseCrash) {
TheDriver.generateCompilationDiagnostics(*C, *FailingCommand);
break;
}
}
}
Diags.getClient()->finish();
// If any timers were active but haven't been destroyed yet, print their
// results now. This happens in -disable-free mode.
llvm::TimerGroup::printAll(llvm::errs());
#ifdef LLVM_ON_WIN32
// Exit status should not be negative on Win32, unless abnormal termination.
// Once abnormal termiation was caught, negative status should not be
// propagated.
if (Res < 0)
Res = 1;
#endif
// If we have multiple failing commands, we return the result of the first
// failing command.
return Res;
}

40
dbms/src/Server/Compiler/lld.cpp Normal file
View File

@ -0,0 +1,40 @@
//===- tools/lld/lld.cpp - Linker Driver Dispatcher -----------------------===//
//
// The LLVM Linker
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the lld driver. This is a thin wrapper which
// dispatches to the given platform specific driver.
//
// If there is -flavor option, it is dispatched according to the arguments.
// If the flavor parameter is not present, then it is dispatched according
// to argv[0].
//
//===----------------------------------------------------------------------===//
#include "lld/Common/Driver.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/PrettyStackTrace.h"
#include "llvm/Support/Signals.h"
using namespace lld;
using namespace llvm;
using namespace llvm::sys;
int mainEntryClickHouseLLD(int Argc, char **Argv) {
// Standard set up, so program fails gracefully.
sys::PrintStackTraceOnErrorSignal(Argv[0]);
PrettyStackTraceProgram StackPrinter(Argc, Argv);
llvm_shutdown_obj Shutdown;
std::vector<const char *> Args(Argv, Argv + Argc);
return !elf::link(Args, true);
}

14
dbms/src/Server/main.cpp
View File

@ -1,7 +1,10 @@
#include <common/config_common.h>
#include <Common/config.h>
#if USE_TCMALLOC
#include <gperftools/malloc_extension.h>
#endif
#include "Server.h"
#include "LocalServer.h"
#include <Common/StringUtils.h>
@ -16,6 +19,11 @@ int mainEntryClickHouseExtractFromConfig(int argc, char ** argv);
int mainEntryClickHouseCompressor(int argc, char ** argv);
int mainEntryClickHouseFormat(int argc, char ** argv);
#if USE_EMBEDDED_COMPILER
int mainEntryClickHouseClang(int argc, char ** argv);
int mainEntryClickHouseLLD(int argc, char ** argv);
#endif
namespace
{
@ -32,7 +40,11 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
{"performance-test", mainEntryClickHousePerformanceTest},
{"extract-from-config", mainEntryClickHouseExtractFromConfig},
{"compressor", mainEntryClickHouseCompressor},
{"format", mainEntryClickHouseFormat}
{"format", mainEntryClickHouseFormat},
#if USE_EMBEDDED_COMPILER
{"clang", mainEntryClickHouseClang},
{"lld", mainEntryClickHouseLLD},
#endif
};