mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-10-10 10:30:51 +00:00
Merge remote-tracking branch 'upstream/master' into fix25
This commit is contained in:
proller
commit
8288a62844
@ -264,11 +264,10 @@ if (USE_STATIC_LIBRARIES AND HAVE_NO_PIE)
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAG_NO_PIE}")
|
||||
endif ()
|
||||
|
||||
# TODO: only make this extra-checks in CI builds, since a lot of contrib libs won't link -
|
||||
# CI works around this problem by explicitly adding GLIBC_COMPATIBILITY flag.
|
||||
if (NOT SANITIZE AND YANDEX_OFFICIAL_BUILD)
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined")
|
||||
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined")
|
||||
# Make this extra-checks for correct library dependencies.
|
||||
if (NOT SANITIZE)
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined")
|
||||
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined")
|
||||
endif ()
|
||||
|
||||
include (cmake/find_unwind.cmake)
|
||||
|
||||
@ -219,7 +219,7 @@ endif()
|
||||
add_library(${ARROW_LIBRARY} ${ARROW_SRCS})
|
||||
add_dependencies(${ARROW_LIBRARY} protoc)
|
||||
target_include_directories(${ARROW_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src ${Boost_INCLUDE_DIRS})
|
||||
target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${PROTOBUF_LIBRARIES} Threads::Threads)
|
||||
target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY} Threads::Threads)
|
||||
if (ARROW_WITH_LZ4)
|
||||
target_link_libraries(${ARROW_LIBRARY} PRIVATE ${LZ4_LIBRARY})
|
||||
endif()
|
||||
|
||||
@ -81,7 +81,6 @@ add_subdirectory (extract-from-config)
|
||||
add_subdirectory (compressor)
|
||||
add_subdirectory (copier)
|
||||
add_subdirectory (format)
|
||||
add_subdirectory (clang)
|
||||
add_subdirectory (obfuscator)
|
||||
|
||||
if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
|
||||
@ -89,9 +88,9 @@ if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
|
||||
endif ()
|
||||
|
||||
if (CLICKHOUSE_ONE_SHARED)
|
||||
add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_PERFORMANCE_TEST_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_COMPILER_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES})
|
||||
target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_PERFORMANCE_TEST_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_COMPILER_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK})
|
||||
target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_PERFORMANCE_TEST_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_COMPILER_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE})
|
||||
add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_PERFORMANCE_TEST_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES})
|
||||
target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_PERFORMANCE_TEST_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK})
|
||||
target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_PERFORMANCE_TEST_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE})
|
||||
set_target_properties(clickhouse-lib PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse DEBUG_POSTFIX "")
|
||||
install (TARGETS clickhouse-lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse)
|
||||
endif()
|
||||
@ -104,10 +103,6 @@ if (CLICKHOUSE_SPLIT_BINARY)
|
||||
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-odbc-bridge)
|
||||
endif ()
|
||||
|
||||
if (USE_EMBEDDED_COMPILER)
|
||||
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-clang clickhouse-lld)
|
||||
endif ()
|
||||
|
||||
set_target_properties(${CLICKHOUSE_ALL_TARGETS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
|
||||
|
||||
add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS})
|
||||
@ -115,10 +110,6 @@ if (CLICKHOUSE_SPLIT_BINARY)
|
||||
|
||||
install(PROGRAMS clickhouse-split-helper DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME clickhouse COMPONENT clickhouse)
|
||||
else ()
|
||||
if (USE_EMBEDDED_COMPILER)
|
||||
# before add_executable !
|
||||
link_directories (${LLVM_LIBRARY_DIRS})
|
||||
endif ()
|
||||
add_executable (clickhouse main.cpp)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils)
|
||||
target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
|
||||
@ -154,9 +145,6 @@ else ()
|
||||
if (ENABLE_CLICKHOUSE_OBFUSCATOR)
|
||||
clickhouse_target_link_split_lib(clickhouse obfuscator)
|
||||
endif ()
|
||||
if (USE_EMBEDDED_COMPILER)
|
||||
target_link_libraries(clickhouse PRIVATE clickhouse-compiler-lib)
|
||||
endif ()
|
||||
|
||||
set (CLICKHOUSE_BUNDLE)
|
||||
if (ENABLE_CLICKHOUSE_SERVER)
|
||||
@ -213,18 +201,8 @@ else ()
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-odbc-bridge)
|
||||
endif()
|
||||
|
||||
# install always because depian package want this files:
|
||||
add_custom_target (clickhouse-clang ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-clang DEPENDS clickhouse)
|
||||
add_custom_target (clickhouse-lld ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-lld DEPENDS clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-clang clickhouse-lld)
|
||||
|
||||
install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
|
||||
install (FILES
|
||||
${CMAKE_CURRENT_BINARY_DIR}/clickhouse-clang
|
||||
${CMAKE_CURRENT_BINARY_DIR}/clickhouse-lld
|
||||
DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
|
||||
add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_BUNDLE})
|
||||
|
||||
endif ()
|
||||
|
||||
@ -1,38 +0,0 @@
|
||||
if (USE_EMBEDDED_COMPILER)
|
||||
add_subdirectory ("Compiler-${LLVM_VERSION}")
|
||||
endif ()
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
if (USE_EMBEDDED_COMPILER)
|
||||
link_directories (${LLVM_LIBRARY_DIRS})
|
||||
add_executable (clickhouse-clang clickhouse-clang.cpp)
|
||||
target_link_libraries (clickhouse-clang PRIVATE clickhouse-compiler-lib)
|
||||
add_executable (clickhouse-lld clickhouse-lld.cpp)
|
||||
target_link_libraries (clickhouse-lld PRIVATE clickhouse-compiler-lib)
|
||||
install (TARGETS clickhouse-clang clickhouse-lld RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
set (TMP_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/${INTERNAL_COMPILER_HEADERS_RELATIVE}")
|
||||
# Make and install empty dir for debian package if compiler disabled
|
||||
add_custom_target (make-headers-directory ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${TMP_HEADERS_DIR})
|
||||
install (DIRECTORY ${TMP_HEADERS_DIR} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse/${INTERNAL_COMPILER_HEADERS_DIR} COMPONENT clickhouse)
|
||||
# TODO: fix on macos copy_headers.sh: sed --posix
|
||||
|
||||
if (USE_EMBEDDED_COMPILER)
|
||||
set (COPY_HEADERS_COMPILER "${CMAKE_CURRENT_BINARY_DIR}/../${INTERNAL_COMPILER_EXECUTABLE}")
|
||||
set (COPY_HEADERS_DEPENDS clickhouse-clang)
|
||||
elseif (EXISTS ${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE})
|
||||
set (COPY_HEADERS_COMPILER "${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE}")
|
||||
endif ()
|
||||
|
||||
if (COPY_HEADERS_COMPILER)
|
||||
add_custom_target (copy-headers [ -f ${TMP_HEADERS_DIR}/dbms/src/Interpreters/SpecializedAggregator.h ] || env CLANG=${COPY_HEADERS_COMPILER} BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} CMAKE_CXX_COMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} ${CMAKE_CURRENT_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${TMP_HEADERS_DIR} DEPENDS ${COPY_HEADERS_DEPENDS} WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES copy_headers.sh)
|
||||
|
||||
if (USE_INTERNAL_LLVM_LIBRARY)
|
||||
set (CLANG_HEADERS_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/clang/lib/Headers")
|
||||
set (CLANG_HEADERS_DEST "${TMP_HEADERS_DIR}/usr/local/lib/clang/${LLVM_VERSION}/include") # original: ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION}/include
|
||||
add_custom_target (copy-headers-clang ${CMAKE_COMMAND} -E make_directory ${CLANG_HEADERS_DEST} && ${CMAKE_COMMAND} -E copy_if_different ${CLANG_HEADERS_DIR}/* ${CLANG_HEADERS_DEST} )
|
||||
add_dependencies (copy-headers copy-headers-clang)
|
||||
endif ()
|
||||
endif ()
|
||||
@ -1,53 +0,0 @@
|
||||
add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG)
|
||||
|
||||
link_directories(${LLVM_LIBRARY_DIRS})
|
||||
|
||||
add_library(clickhouse-compiler-lib
|
||||
driver.cpp
|
||||
cc1_main.cpp
|
||||
cc1as_main.cpp
|
||||
lld.cpp)
|
||||
|
||||
target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0)
|
||||
|
||||
string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti
|
||||
|
||||
llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
|
||||
|
||||
message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
|
||||
|
||||
target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
|
||||
|
||||
# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.
|
||||
|
||||
target_link_libraries(clickhouse-compiler-lib PRIVATE
|
||||
|
||||
clangBasic clangCodeGen clangDriver clangFrontend clangFrontendTool
|
||||
clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend
|
||||
clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers
|
||||
clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic
|
||||
|
||||
lldCOFF
|
||||
lldDriver
|
||||
lldELF
|
||||
#lldMinGW
|
||||
lldMachO
|
||||
lldReaderWriter
|
||||
lldYAML
|
||||
#lldCommon
|
||||
lldCore
|
||||
lldConfig
|
||||
|
||||
${REQUIRED_LLVM_LIBRARIES}
|
||||
|
||||
LLVMSupport
|
||||
|
||||
#Polly
|
||||
#PollyISL
|
||||
#PollyPPCG
|
||||
|
||||
PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARIES} Threads::Threads
|
||||
${MALLOC_LIBRARIES}
|
||||
${GLIBC_COMPATIBILITY_LIBRARIES}
|
||||
${MEMCPY_LIBRARIES}
|
||||
)
|
||||
@ -1,63 +0,0 @@
|
||||
==============================================================================
|
||||
LLVM Release License
|
||||
==============================================================================
|
||||
University of Illinois/NCSA
|
||||
Open Source License
|
||||
|
||||
Copyright (c) 2007-2016 University of Illinois at Urbana-Champaign.
|
||||
All rights reserved.
|
||||
|
||||
Developed by:
|
||||
|
||||
LLVM Team
|
||||
|
||||
University of Illinois at Urbana-Champaign
|
||||
|
||||
http://llvm.org
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal with
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
of the Software, and to permit persons to whom the Software is furnished to do
|
||||
so, subject to the following conditions:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimers.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimers in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the names of the LLVM Team, University of Illinois at
|
||||
Urbana-Champaign, nor the names of its contributors may be used to
|
||||
endorse or promote products derived from this Software without specific
|
||||
prior written permission.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
|
||||
SOFTWARE.
|
||||
|
||||
==============================================================================
|
||||
The LLVM software contains code written by third parties. Such software will
|
||||
have its own individual LICENSE.TXT file in the directory in which it appears.
|
||||
This file will describe the copyrights, license, and restrictions which apply
|
||||
to that code.
|
||||
|
||||
The disclaimer of warranty in the University of Illinois Open Source License
|
||||
applies to all code in the LLVM Distribution, and nothing in any of the
|
||||
other licenses gives permission to use the names of the LLVM Team or the
|
||||
University of Illinois to endorse or promote products derived from this
|
||||
Software.
|
||||
|
||||
The following pieces of software have additional or alternate copyrights,
|
||||
licenses, and/or restrictions:
|
||||
|
||||
Program Directory
|
||||
------- ---------
|
||||
<none yet>
|
||||
|
||||
@ -1,242 +0,0 @@
|
||||
//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This is the entry point to the clang -cc1 functionality, which implements the
|
||||
// core compiler functionality along with a number of additional tools for
|
||||
// demonstration and testing purposes.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "llvm/Option/Arg.h"
|
||||
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
|
||||
#include "clang/Config/config.h"
|
||||
#include "clang/Driver/DriverDiagnostic.h"
|
||||
#include "clang/Driver/Options.h"
|
||||
#include "clang/Frontend/CompilerInstance.h"
|
||||
#include "clang/Frontend/CompilerInvocation.h"
|
||||
#include "clang/Frontend/FrontendDiagnostic.h"
|
||||
#include "clang/Frontend/TextDiagnosticBuffer.h"
|
||||
#include "clang/Frontend/TextDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/Utils.h"
|
||||
#include "clang/FrontendTool/Utils.h"
|
||||
#include "llvm/ADT/Statistic.h"
|
||||
#include "llvm/LinkAllPasses.h"
|
||||
#include "llvm/Option/ArgList.h"
|
||||
#include "llvm/Option/OptTable.h"
|
||||
#include "llvm/Support/Compiler.h"
|
||||
#include "llvm/Support/ErrorHandling.h"
|
||||
#include "llvm/Support/ManagedStatic.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
#include "llvm/Support/TargetSelect.h"
|
||||
#include "llvm/Support/Timer.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
#include <cstdio>
|
||||
|
||||
#ifdef CLANG_HAVE_RLIMITS
|
||||
#include <sys/resource.h>
|
||||
#endif
|
||||
|
||||
// have no .a version in packages
|
||||
#undef LINK_POLLY_INTO_TOOLS
|
||||
|
||||
using namespace clang;
|
||||
using namespace llvm::opt;
|
||||
|
||||
//===----------------------------------------------------------------------===//
|
||||
// Main driver
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
static void LLVMErrorHandler(void *UserData, const std::string &Message,
|
||||
bool GenCrashDiag) {
|
||||
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
|
||||
|
||||
Diags.Report(diag::err_fe_error_backend) << Message;
|
||||
|
||||
// Run the interrupt handlers to make sure any special cleanups get done, in
|
||||
// particular that we remove files registered with RemoveFileOnSignal.
|
||||
llvm::sys::RunInterruptHandlers();
|
||||
|
||||
// We cannot recover from llvm errors. When reporting a fatal error, exit
|
||||
// with status 70 to generate crash diagnostics. For BSD systems this is
|
||||
// defined as an internal software error. Otherwise, exit with status 1.
|
||||
exit(GenCrashDiag ? 70 : 1);
|
||||
}
|
||||
|
||||
#ifdef LINK_POLLY_INTO_TOOLS
|
||||
namespace polly {
|
||||
void initializePollyPasses(llvm::PassRegistry &Registry);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef CLANG_HAVE_RLIMITS
|
||||
// The amount of stack we think is "sufficient". If less than this much is
|
||||
// available, we may be unable to reach our template instantiation depth
|
||||
// limit and other similar limits.
|
||||
// FIXME: Unify this with the stack we request when spawning a thread to build
|
||||
// a module.
|
||||
static const int kSufficientStack = 8 << 20;
|
||||
|
||||
#if defined(__linux__) && defined(__PIE__)
|
||||
static size_t getCurrentStackAllocation() {
|
||||
// If we can't compute the current stack usage, allow for 512K of command
|
||||
// line arguments and environment.
|
||||
size_t Usage = 512 * 1024;
|
||||
if (FILE *StatFile = fopen("/proc/self/stat", "r")) {
|
||||
// We assume that the stack extends from its current address to the end of
|
||||
// the environment space. In reality, there is another string literal (the
|
||||
// program name) after the environment, but this is close enough (we only
|
||||
// need to be within 100K or so).
|
||||
unsigned long StackPtr, EnvEnd;
|
||||
// Disable silly GCC -Wformat warning that complains about length
|
||||
// modifiers on ignored format specifiers. We want to retain these
|
||||
// for documentation purposes even though they have no effect.
|
||||
#if defined(__GNUC__) && !defined(__clang__)
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wformat"
|
||||
#endif
|
||||
if (fscanf(StatFile,
|
||||
"%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu "
|
||||
"%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu "
|
||||
"%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d "
|
||||
"%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d",
|
||||
&StackPtr, &EnvEnd) == 2) {
|
||||
#if defined(__GNUC__) && !defined(__clang__)
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd;
|
||||
}
|
||||
fclose(StatFile);
|
||||
}
|
||||
return Usage;
|
||||
}
|
||||
|
||||
#include <alloca.h>
|
||||
|
||||
LLVM_ATTRIBUTE_NOINLINE
|
||||
static void ensureStackAddressSpace(int ExtraChunks = 0) {
|
||||
// Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary
|
||||
// relatively close to the stack (they are only guaranteed to be 128MiB
|
||||
// apart). This results in crashes if we happen to heap-allocate more than
|
||||
// 128MiB before we reach our stack high-water mark.
|
||||
//
|
||||
// To avoid these crashes, ensure that we have sufficient virtual memory
|
||||
// pages allocated before we start running.
|
||||
size_t Curr = getCurrentStackAllocation();
|
||||
const int kTargetStack = kSufficientStack - 256 * 1024;
|
||||
if (Curr < kTargetStack) {
|
||||
volatile char *volatile Alloc =
|
||||
static_cast<volatile char *>(alloca(kTargetStack - Curr));
|
||||
Alloc[0] = 0;
|
||||
Alloc[kTargetStack - Curr - 1] = 0;
|
||||
}
|
||||
}
|
||||
#else
|
||||
static void ensureStackAddressSpace() {}
|
||||
#endif
|
||||
|
||||
/// Attempt to ensure that we have at least 8MiB of usable stack space.
|
||||
static void ensureSufficientStack() {
|
||||
struct rlimit rlim;
|
||||
if (getrlimit(RLIMIT_STACK, &rlim) != 0)
|
||||
return;
|
||||
|
||||
// Increase the soft stack limit to our desired level, if necessary and
|
||||
// possible.
|
||||
if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) {
|
||||
// Try to allocate sufficient stack.
|
||||
if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack)
|
||||
rlim.rlim_cur = kSufficientStack;
|
||||
else if (rlim.rlim_cur == rlim.rlim_max)
|
||||
return;
|
||||
else
|
||||
rlim.rlim_cur = rlim.rlim_max;
|
||||
|
||||
if (setrlimit(RLIMIT_STACK, &rlim) != 0 ||
|
||||
rlim.rlim_cur != kSufficientStack)
|
||||
return;
|
||||
}
|
||||
|
||||
// We should now have a stack of size at least kSufficientStack. Ensure
|
||||
// that we can actually use that much, if necessary.
|
||||
ensureStackAddressSpace();
|
||||
}
|
||||
#else
|
||||
static void ensureSufficientStack() {}
|
||||
#endif
|
||||
|
||||
int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
|
||||
ensureSufficientStack();
|
||||
|
||||
std::unique_ptr<CompilerInstance> Clang(new CompilerInstance());
|
||||
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
|
||||
|
||||
// Register the support for object-file-wrapped Clang modules.
|
||||
auto PCHOps = Clang->getPCHContainerOperations();
|
||||
PCHOps->registerWriter(llvm::make_unique<ObjectFilePCHContainerWriter>());
|
||||
PCHOps->registerReader(llvm::make_unique<ObjectFilePCHContainerReader>());
|
||||
|
||||
// Initialize targets first, so that --version shows registered targets.
|
||||
llvm::InitializeAllTargets();
|
||||
llvm::InitializeAllTargetMCs();
|
||||
llvm::InitializeAllAsmPrinters();
|
||||
llvm::InitializeAllAsmParsers();
|
||||
|
||||
#ifdef LINK_POLLY_INTO_TOOLS
|
||||
llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry();
|
||||
polly::initializePollyPasses(Registry);
|
||||
#endif
|
||||
|
||||
// Buffer diagnostics from argument parsing so that we can output them using a
|
||||
// well formed diagnostic object.
|
||||
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
|
||||
TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer;
|
||||
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer);
|
||||
bool Success = CompilerInvocation::CreateFromArgs(
|
||||
Clang->getInvocation(), Argv.begin(), Argv.end(), Diags);
|
||||
|
||||
// Infer the builtin include path if unspecified.
|
||||
if (Clang->getHeaderSearchOpts().UseBuiltinIncludes &&
|
||||
Clang->getHeaderSearchOpts().ResourceDir.empty())
|
||||
Clang->getHeaderSearchOpts().ResourceDir =
|
||||
CompilerInvocation::GetResourcesPath(Argv0, MainAddr);
|
||||
|
||||
// Create the actual diagnostics engine.
|
||||
Clang->createDiagnostics();
|
||||
if (!Clang->hasDiagnostics())
|
||||
return 1;
|
||||
|
||||
// Set an error handler, so that any LLVM backend diagnostics go through our
|
||||
// error handler.
|
||||
llvm::install_fatal_error_handler(LLVMErrorHandler,
|
||||
static_cast<void*>(&Clang->getDiagnostics()));
|
||||
|
||||
DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics());
|
||||
if (!Success)
|
||||
return 1;
|
||||
|
||||
// Execute the frontend actions.
|
||||
Success = ExecuteCompilerInvocation(Clang.get());
|
||||
|
||||
// If any timers were active but haven't been destroyed yet, print their
|
||||
// results now. This happens in -disable-free mode.
|
||||
llvm::TimerGroup::printAll(llvm::errs());
|
||||
|
||||
// Our error handler depends on the Diagnostics object, which we're
|
||||
// potentially about to delete. Uninstall the handler now so that any
|
||||
// later errors use the default handling behavior instead.
|
||||
llvm::remove_fatal_error_handler();
|
||||
|
||||
// When running with -disable-free, don't do any destruction or shutdown.
|
||||
if (Clang->getFrontendOpts().DisableFree) {
|
||||
BuryPointer(std::move(Clang));
|
||||
return !Success;
|
||||
}
|
||||
|
||||
return !Success;
|
||||
}
|
||||
@ -1,540 +0,0 @@
|
||||
//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This is the entry point to the clang -cc1as functionality, which implements
|
||||
// the direct interface to the LLVM MC based assembler.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "clang/Basic/Diagnostic.h"
|
||||
#include "clang/Basic/DiagnosticOptions.h"
|
||||
#include "clang/Driver/DriverDiagnostic.h"
|
||||
#include "clang/Driver/Options.h"
|
||||
#include "clang/Frontend/FrontendDiagnostic.h"
|
||||
#include "clang/Frontend/TextDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/Utils.h"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/ADT/StringSwitch.h"
|
||||
#include "llvm/ADT/Triple.h"
|
||||
#include "llvm/IR/DataLayout.h"
|
||||
#include "llvm/MC/MCAsmBackend.h"
|
||||
#include "llvm/MC/MCAsmInfo.h"
|
||||
#include "llvm/MC/MCCodeEmitter.h"
|
||||
#include "llvm/MC/MCContext.h"
|
||||
#include "llvm/MC/MCInstrInfo.h"
|
||||
#include "llvm/MC/MCObjectFileInfo.h"
|
||||
#include "llvm/MC/MCParser/MCAsmParser.h"
|
||||
#include "llvm/MC/MCParser/MCTargetAsmParser.h"
|
||||
#include "llvm/MC/MCRegisterInfo.h"
|
||||
#include "llvm/MC/MCStreamer.h"
|
||||
#include "llvm/MC/MCSubtargetInfo.h"
|
||||
#include "llvm/MC/MCTargetOptions.h"
|
||||
#include "llvm/Option/Arg.h"
|
||||
#include "llvm/Option/ArgList.h"
|
||||
#include "llvm/Option/OptTable.h"
|
||||
#include "llvm/Support/CommandLine.h"
|
||||
#include "llvm/Support/ErrorHandling.h"
|
||||
#include "llvm/Support/FileSystem.h"
|
||||
#include "llvm/Support/FormattedStream.h"
|
||||
#include "llvm/Support/Host.h"
|
||||
#include "llvm/Support/MemoryBuffer.h"
|
||||
#include "llvm/Support/Path.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
#include "llvm/Support/SourceMgr.h"
|
||||
#include "llvm/Support/TargetRegistry.h"
|
||||
#include "llvm/Support/TargetSelect.h"
|
||||
#include "llvm/Support/Timer.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
#include <memory>
|
||||
#include <system_error>
|
||||
using namespace clang;
|
||||
using namespace clang::driver;
|
||||
using namespace clang::driver::options;
|
||||
using namespace llvm;
|
||||
using namespace llvm::opt;
|
||||
|
||||
namespace {
|
||||
|
||||
/// \brief Helper class for representing a single invocation of the assembler.
|
||||
struct AssemblerInvocation {
|
||||
/// @name Target Options
|
||||
/// @{
|
||||
|
||||
/// The name of the target triple to assemble for.
|
||||
std::string Triple;
|
||||
|
||||
/// If given, the name of the target CPU to determine which instructions
|
||||
/// are legal.
|
||||
std::string CPU;
|
||||
|
||||
/// The list of target specific features to enable or disable -- this should
|
||||
/// be a list of strings starting with '+' or '-'.
|
||||
std::vector<std::string> Features;
|
||||
|
||||
/// The list of symbol definitions.
|
||||
std::vector<std::string> SymbolDefs;
|
||||
|
||||
/// @}
|
||||
/// @name Language Options
|
||||
/// @{
|
||||
|
||||
std::vector<std::string> IncludePaths;
|
||||
unsigned NoInitialTextSection : 1;
|
||||
unsigned SaveTemporaryLabels : 1;
|
||||
unsigned GenDwarfForAssembly : 1;
|
||||
unsigned RelaxELFRelocations : 1;
|
||||
unsigned DwarfVersion;
|
||||
std::string DwarfDebugFlags;
|
||||
std::string DwarfDebugProducer;
|
||||
std::string DebugCompilationDir;
|
||||
llvm::DebugCompressionType CompressDebugSections =
|
||||
llvm::DebugCompressionType::None;
|
||||
std::string MainFileName;
|
||||
|
||||
/// @}
|
||||
/// @name Frontend Options
|
||||
/// @{
|
||||
|
||||
std::string InputFile;
|
||||
std::vector<std::string> LLVMArgs;
|
||||
std::string OutputPath;
|
||||
enum FileType {
|
||||
FT_Asm, ///< Assembly (.s) output, transliterate mode.
|
||||
FT_Null, ///< No output, for timing purposes.
|
||||
FT_Obj ///< Object file output.
|
||||
};
|
||||
FileType OutputType;
|
||||
unsigned ShowHelp : 1;
|
||||
unsigned ShowVersion : 1;
|
||||
|
||||
/// @}
|
||||
/// @name Transliterate Options
|
||||
/// @{
|
||||
|
||||
unsigned OutputAsmVariant;
|
||||
unsigned ShowEncoding : 1;
|
||||
unsigned ShowInst : 1;
|
||||
|
||||
/// @}
|
||||
/// @name Assembler Options
|
||||
/// @{
|
||||
|
||||
unsigned RelaxAll : 1;
|
||||
unsigned NoExecStack : 1;
|
||||
unsigned FatalWarnings : 1;
|
||||
unsigned IncrementalLinkerCompatible : 1;
|
||||
|
||||
/// The name of the relocation model to use.
|
||||
std::string RelocationModel;
|
||||
|
||||
/// @}
|
||||
|
||||
public:
|
||||
AssemblerInvocation() {
|
||||
Triple = "";
|
||||
NoInitialTextSection = 0;
|
||||
InputFile = "-";
|
||||
OutputPath = "-";
|
||||
OutputType = FT_Asm;
|
||||
OutputAsmVariant = 0;
|
||||
ShowInst = 0;
|
||||
ShowEncoding = 0;
|
||||
RelaxAll = 0;
|
||||
NoExecStack = 0;
|
||||
FatalWarnings = 0;
|
||||
IncrementalLinkerCompatible = 0;
|
||||
DwarfVersion = 0;
|
||||
}
|
||||
|
||||
static bool CreateFromArgs(AssemblerInvocation &Res,
|
||||
ArrayRef<const char *> Argv,
|
||||
DiagnosticsEngine &Diags);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
|
||||
ArrayRef<const char *> Argv,
|
||||
DiagnosticsEngine &Diags) {
|
||||
bool Success = true;
|
||||
|
||||
// Parse the arguments.
|
||||
std::unique_ptr<OptTable> OptTbl(createDriverOptTable());
|
||||
|
||||
const unsigned IncludedFlagsBitmask = options::CC1AsOption;
|
||||
unsigned MissingArgIndex, MissingArgCount;
|
||||
InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount,
|
||||
IncludedFlagsBitmask);
|
||||
|
||||
// Check for missing argument error.
|
||||
if (MissingArgCount) {
|
||||
Diags.Report(diag::err_drv_missing_argument)
|
||||
<< Args.getArgString(MissingArgIndex) << MissingArgCount;
|
||||
Success = false;
|
||||
}
|
||||
|
||||
// Issue errors on unknown arguments.
|
||||
for (const Arg *A : Args.filtered(OPT_UNKNOWN)) {
|
||||
Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args);
|
||||
Success = false;
|
||||
}
|
||||
|
||||
// Construct the invocation.
|
||||
|
||||
// Target Options
|
||||
Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple));
|
||||
Opts.CPU = Args.getLastArgValue(OPT_target_cpu);
|
||||
Opts.Features = Args.getAllArgValues(OPT_target_feature);
|
||||
|
||||
// Use the default target triple if unspecified.
|
||||
if (Opts.Triple.empty())
|
||||
Opts.Triple = llvm::sys::getDefaultTargetTriple();
|
||||
|
||||
// Language Options
|
||||
Opts.IncludePaths = Args.getAllArgValues(OPT_I);
|
||||
Opts.NoInitialTextSection = Args.hasArg(OPT_n);
|
||||
Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels);
|
||||
// Any DebugInfoKind implies GenDwarfForAssembly.
|
||||
Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ);
|
||||
|
||||
if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections,
|
||||
OPT_compress_debug_sections_EQ)) {
|
||||
if (A->getOption().getID() == OPT_compress_debug_sections) {
|
||||
// TODO: be more clever about the compression type auto-detection
|
||||
Opts.CompressDebugSections = llvm::DebugCompressionType::GNU;
|
||||
} else {
|
||||
Opts.CompressDebugSections =
|
||||
llvm::StringSwitch<llvm::DebugCompressionType>(A->getValue())
|
||||
.Case("none", llvm::DebugCompressionType::None)
|
||||
.Case("zlib", llvm::DebugCompressionType::Z)
|
||||
.Case("zlib-gnu", llvm::DebugCompressionType::GNU)
|
||||
.Default(llvm::DebugCompressionType::None);
|
||||
}
|
||||
}
|
||||
|
||||
Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations);
|
||||
Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags);
|
||||
Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags);
|
||||
Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer);
|
||||
Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir);
|
||||
Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name);
|
||||
|
||||
// Frontend Options
|
||||
if (Args.hasArg(OPT_INPUT)) {
|
||||
bool First = true;
|
||||
for (const Arg *A : Args.filtered(OPT_INPUT)) {
|
||||
if (First) {
|
||||
Opts.InputFile = A->getValue();
|
||||
First = false;
|
||||
} else {
|
||||
Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args);
|
||||
Success = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
|
||||
Opts.OutputPath = Args.getLastArgValue(OPT_o);
|
||||
if (Arg *A = Args.getLastArg(OPT_filetype)) {
|
||||
StringRef Name = A->getValue();
|
||||
unsigned OutputType = StringSwitch<unsigned>(Name)
|
||||
.Case("asm", FT_Asm)
|
||||
.Case("null", FT_Null)
|
||||
.Case("obj", FT_Obj)
|
||||
.Default(~0U);
|
||||
if (OutputType == ~0U) {
|
||||
Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name;
|
||||
Success = false;
|
||||
} else
|
||||
Opts.OutputType = FileType(OutputType);
|
||||
}
|
||||
Opts.ShowHelp = Args.hasArg(OPT_help);
|
||||
Opts.ShowVersion = Args.hasArg(OPT_version);
|
||||
|
||||
// Transliterate Options
|
||||
Opts.OutputAsmVariant =
|
||||
getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags);
|
||||
Opts.ShowEncoding = Args.hasArg(OPT_show_encoding);
|
||||
Opts.ShowInst = Args.hasArg(OPT_show_inst);
|
||||
|
||||
// Assemble Options
|
||||
Opts.RelaxAll = Args.hasArg(OPT_mrelax_all);
|
||||
Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack);
|
||||
Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings);
|
||||
Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic");
|
||||
Opts.IncrementalLinkerCompatible =
|
||||
Args.hasArg(OPT_mincremental_linker_compatible);
|
||||
Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym);
|
||||
|
||||
return Success;
|
||||
}
|
||||
|
||||
static std::unique_ptr<raw_fd_ostream>
|
||||
getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags,
|
||||
bool Binary) {
|
||||
if (Opts.OutputPath.empty())
|
||||
Opts.OutputPath = "-";
|
||||
|
||||
// Make sure that the Out file gets unlinked from the disk if we get a
|
||||
// SIGINT.
|
||||
if (Opts.OutputPath != "-")
|
||||
sys::RemoveFileOnSignal(Opts.OutputPath);
|
||||
|
||||
std::error_code EC;
|
||||
auto Out = llvm::make_unique<raw_fd_ostream>(
|
||||
Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text));
|
||||
if (EC) {
|
||||
Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath
|
||||
<< EC.message();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return Out;
|
||||
}
|
||||
|
||||
static bool ExecuteAssembler(AssemblerInvocation &Opts,
|
||||
DiagnosticsEngine &Diags) {
|
||||
// Get the target specific parser.
|
||||
std::string Error;
|
||||
const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error);
|
||||
if (!TheTarget)
|
||||
return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
|
||||
|
||||
ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer =
|
||||
MemoryBuffer::getFileOrSTDIN(Opts.InputFile);
|
||||
|
||||
if (std::error_code EC = Buffer.getError()) {
|
||||
Error = EC.message();
|
||||
return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile;
|
||||
}
|
||||
|
||||
SourceMgr SrcMgr;
|
||||
|
||||
// Tell SrcMgr about this buffer, which is what the parser will pick up.
|
||||
SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc());
|
||||
|
||||
// Record the location of the include directories so that the lexer can find
|
||||
// it later.
|
||||
SrcMgr.setIncludeDirs(Opts.IncludePaths);
|
||||
|
||||
std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(Opts.Triple));
|
||||
assert(MRI && "Unable to create target register info!");
|
||||
|
||||
std::unique_ptr<MCAsmInfo> MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple));
|
||||
assert(MAI && "Unable to create target asm info!");
|
||||
|
||||
// Ensure MCAsmInfo initialization occurs before any use, otherwise sections
|
||||
// may be created with a combination of default and explicit settings.
|
||||
MAI->setCompressDebugSections(Opts.CompressDebugSections);
|
||||
|
||||
MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations);
|
||||
|
||||
bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj;
|
||||
std::unique_ptr<raw_fd_ostream> FDOS = getOutputStream(Opts, Diags, IsBinary);
|
||||
if (!FDOS)
|
||||
return true;
|
||||
|
||||
// FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and
|
||||
// MCObjectFileInfo needs a MCContext reference in order to initialize itself.
|
||||
std::unique_ptr<MCObjectFileInfo> MOFI(new MCObjectFileInfo());
|
||||
|
||||
MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr);
|
||||
|
||||
bool PIC = false;
|
||||
if (Opts.RelocationModel == "static") {
|
||||
PIC = false;
|
||||
} else if (Opts.RelocationModel == "pic") {
|
||||
PIC = true;
|
||||
} else {
|
||||
assert(Opts.RelocationModel == "dynamic-no-pic" &&
|
||||
"Invalid PIC model!");
|
||||
PIC = false;
|
||||
}
|
||||
|
||||
MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, CodeModel::Default, Ctx);
|
||||
if (Opts.SaveTemporaryLabels)
|
||||
Ctx.setAllowTemporaryLabels(false);
|
||||
if (Opts.GenDwarfForAssembly)
|
||||
Ctx.setGenDwarfForAssembly(true);
|
||||
if (!Opts.DwarfDebugFlags.empty())
|
||||
Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags));
|
||||
if (!Opts.DwarfDebugProducer.empty())
|
||||
Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer));
|
||||
if (!Opts.DebugCompilationDir.empty())
|
||||
Ctx.setCompilationDir(Opts.DebugCompilationDir);
|
||||
if (!Opts.MainFileName.empty())
|
||||
Ctx.setMainFileName(StringRef(Opts.MainFileName));
|
||||
Ctx.setDwarfVersion(Opts.DwarfVersion);
|
||||
|
||||
// Build up the feature string from the target feature list.
|
||||
std::string FS;
|
||||
if (!Opts.Features.empty()) {
|
||||
FS = Opts.Features[0];
|
||||
for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i)
|
||||
FS += "," + Opts.Features[i];
|
||||
}
|
||||
|
||||
std::unique_ptr<MCStreamer> Str;
|
||||
|
||||
std::unique_ptr<MCInstrInfo> MCII(TheTarget->createMCInstrInfo());
|
||||
std::unique_ptr<MCSubtargetInfo> STI(
|
||||
TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS));
|
||||
|
||||
raw_pwrite_stream *Out = FDOS.get();
|
||||
std::unique_ptr<buffer_ostream> BOS;
|
||||
|
||||
// FIXME: There is a bit of code duplication with addPassesToEmitFile.
|
||||
if (Opts.OutputType == AssemblerInvocation::FT_Asm) {
|
||||
MCInstPrinter *IP = TheTarget->createMCInstPrinter(
|
||||
llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI);
|
||||
MCCodeEmitter *CE = nullptr;
|
||||
MCAsmBackend *MAB = nullptr;
|
||||
if (Opts.ShowEncoding) {
|
||||
CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx);
|
||||
MCTargetOptions Options;
|
||||
MAB = TheTarget->createMCAsmBackend(*MRI, Opts.Triple, Opts.CPU, Options);
|
||||
}
|
||||
auto FOut = llvm::make_unique<formatted_raw_ostream>(*Out);
|
||||
Str.reset(TheTarget->createAsmStreamer(
|
||||
Ctx, std::move(FOut), /*asmverbose*/ true,
|
||||
/*useDwarfDirectory*/ true, IP, CE, MAB, Opts.ShowInst));
|
||||
} else if (Opts.OutputType == AssemblerInvocation::FT_Null) {
|
||||
Str.reset(createNullStreamer(Ctx));
|
||||
} else {
|
||||
assert(Opts.OutputType == AssemblerInvocation::FT_Obj &&
|
||||
"Invalid file type!");
|
||||
if (!FDOS->supportsSeeking()) {
|
||||
BOS = make_unique<buffer_ostream>(*FDOS);
|
||||
Out = BOS.get();
|
||||
}
|
||||
|
||||
MCCodeEmitter *CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx);
|
||||
MCTargetOptions Options;
|
||||
MCAsmBackend *MAB = TheTarget->createMCAsmBackend(*MRI, Opts.Triple,
|
||||
Opts.CPU, Options);
|
||||
Triple T(Opts.Triple);
|
||||
Str.reset(TheTarget->createMCObjectStreamer(
|
||||
T, Ctx, *MAB, *Out, CE, *STI, Opts.RelaxAll,
|
||||
Opts.IncrementalLinkerCompatible,
|
||||
/*DWARFMustBeAtTheEnd*/ true));
|
||||
Str.get()->InitSections(Opts.NoExecStack);
|
||||
}
|
||||
|
||||
bool Failed = false;
|
||||
|
||||
std::unique_ptr<MCAsmParser> Parser(
|
||||
createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI));
|
||||
|
||||
// FIXME: init MCTargetOptions from sanitizer flags here.
|
||||
MCTargetOptions Options;
|
||||
std::unique_ptr<MCTargetAsmParser> TAP(
|
||||
TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options));
|
||||
if (!TAP)
|
||||
Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
|
||||
|
||||
// Set values for symbols, if any.
|
||||
for (auto &S : Opts.SymbolDefs) {
|
||||
auto Pair = StringRef(S).split('=');
|
||||
auto Sym = Pair.first;
|
||||
auto Val = Pair.second;
|
||||
int64_t Value;
|
||||
// We have already error checked this in the driver.
|
||||
Val.getAsInteger(0, Value);
|
||||
Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value);
|
||||
}
|
||||
|
||||
if (!Failed) {
|
||||
Parser->setTargetParser(*TAP.get());
|
||||
Failed = Parser->Run(Opts.NoInitialTextSection);
|
||||
}
|
||||
|
||||
// Close Streamer first.
|
||||
// It might have a reference to the output stream.
|
||||
Str.reset();
|
||||
// Close the output stream early.
|
||||
BOS.reset();
|
||||
FDOS.reset();
|
||||
|
||||
// Delete output file if there were errors.
|
||||
if (Failed && Opts.OutputPath != "-")
|
||||
sys::fs::remove(Opts.OutputPath);
|
||||
|
||||
return Failed;
|
||||
}
|
||||
|
||||
static void LLVMErrorHandler(void *UserData, const std::string &Message,
|
||||
bool GenCrashDiag) {
|
||||
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
|
||||
|
||||
Diags.Report(diag::err_fe_error_backend) << Message;
|
||||
|
||||
// We cannot recover from llvm errors.
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
|
||||
// Initialize targets and assembly printers/parsers.
|
||||
InitializeAllTargetInfos();
|
||||
InitializeAllTargetMCs();
|
||||
InitializeAllAsmParsers();
|
||||
|
||||
// Construct our diagnostic client.
|
||||
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
|
||||
TextDiagnosticPrinter *DiagClient
|
||||
= new TextDiagnosticPrinter(errs(), &*DiagOpts);
|
||||
DiagClient->setPrefix("clang -cc1as");
|
||||
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
|
||||
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
|
||||
|
||||
// Set an error handler, so that any LLVM backend diagnostics go through our
|
||||
// error handler.
|
||||
ScopedFatalErrorHandler FatalErrorHandler
|
||||
(LLVMErrorHandler, static_cast<void*>(&Diags));
|
||||
|
||||
// Parse the arguments.
|
||||
AssemblerInvocation Asm;
|
||||
if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags))
|
||||
return 1;
|
||||
|
||||
if (Asm.ShowHelp) {
|
||||
std::unique_ptr<OptTable> Opts(driver::createDriverOptTable());
|
||||
Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler",
|
||||
/*Include=*/driver::options::CC1AsOption, /*Exclude=*/0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Honor -version.
|
||||
//
|
||||
// FIXME: Use a better -version message?
|
||||
if (Asm.ShowVersion) {
|
||||
llvm::cl::PrintVersionMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Honor -mllvm.
|
||||
//
|
||||
// FIXME: Remove this, one day.
|
||||
if (!Asm.LLVMArgs.empty()) {
|
||||
unsigned NumArgs = Asm.LLVMArgs.size();
|
||||
auto Args = llvm::make_unique<const char*[]>(NumArgs + 2);
|
||||
Args[0] = "clang (LLVM option parsing)";
|
||||
for (unsigned i = 0; i != NumArgs; ++i)
|
||||
Args[i + 1] = Asm.LLVMArgs[i].c_str();
|
||||
Args[NumArgs + 1] = nullptr;
|
||||
llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get());
|
||||
}
|
||||
|
||||
// Execute the invocation, unless there were parsing errors.
|
||||
bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags);
|
||||
|
||||
// If any timers were active but haven't been destroyed yet, print their
|
||||
// results now.
|
||||
TimerGroup::printAll(errs());
|
||||
|
||||
return !!Failed;
|
||||
}
|
||||
@ -1,519 +0,0 @@
|
||||
//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This is the entry point to the clang driver; it is a thin wrapper
|
||||
// for functionality in the Driver clang library.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "clang/Basic/DiagnosticOptions.h"
|
||||
#include "clang/Driver/Compilation.h"
|
||||
#include "clang/Driver/Driver.h"
|
||||
#include "clang/Driver/DriverDiagnostic.h"
|
||||
#include "clang/Driver/Options.h"
|
||||
#include "clang/Driver/ToolChain.h"
|
||||
#include "clang/Frontend/ChainedDiagnosticConsumer.h"
|
||||
#include "clang/Frontend/CompilerInvocation.h"
|
||||
#include "clang/Frontend/SerializedDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/TextDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/Utils.h"
|
||||
#include "llvm/ADT/ArrayRef.h"
|
||||
#include "llvm/ADT/SmallString.h"
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
#include "llvm/Config/llvm-config.h"
|
||||
#include "llvm/Option/ArgList.h"
|
||||
#include "llvm/Option/OptTable.h"
|
||||
#include "llvm/Option/Option.h"
|
||||
#include "llvm/Support/CommandLine.h"
|
||||
#include "llvm/Support/ErrorHandling.h"
|
||||
#include "llvm/Support/FileSystem.h"
|
||||
#include "llvm/Support/Host.h"
|
||||
#include "llvm/Support/ManagedStatic.h"
|
||||
#include "llvm/Support/Path.h"
|
||||
#include "llvm/Support/PrettyStackTrace.h"
|
||||
#include "llvm/Support/Process.h"
|
||||
#include "llvm/Support/Program.h"
|
||||
#include "llvm/Support/Regex.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
#include "llvm/Support/StringSaver.h"
|
||||
#include "llvm/Support/TargetSelect.h"
|
||||
#include "llvm/Support/Timer.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <system_error>
|
||||
using namespace clang;
|
||||
using namespace clang::driver;
|
||||
using namespace llvm::opt;
|
||||
|
||||
std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) {
|
||||
if (!CanonicalPrefixes) {
|
||||
SmallString<128> ExecutablePath(Argv0);
|
||||
// Do a PATH lookup if Argv0 isn't a valid path.
|
||||
if (!llvm::sys::fs::exists(ExecutablePath))
|
||||
if (llvm::ErrorOr<std::string> P =
|
||||
llvm::sys::findProgramByName(ExecutablePath))
|
||||
ExecutablePath = *P;
|
||||
return ExecutablePath.str();
|
||||
}
|
||||
|
||||
// This just needs to be some symbol in the binary; C++ doesn't
|
||||
// allow taking the address of ::main however.
|
||||
void *P = (void*) (intptr_t) GetExecutablePath;
|
||||
return llvm::sys::fs::getMainExecutable(Argv0, P);
|
||||
}
|
||||
|
||||
static const char *GetStableCStr(std::set<std::string> &SavedStrings,
|
||||
StringRef S) {
|
||||
return SavedStrings.insert(S).first->c_str();
|
||||
}
|
||||
|
||||
/// ApplyQAOverride - Apply a list of edits to the input argument lists.
|
||||
///
|
||||
/// The input string is a space separate list of edits to perform,
|
||||
/// they are applied in order to the input argument lists. Edits
|
||||
/// should be one of the following forms:
|
||||
///
|
||||
/// '#': Silence information about the changes to the command line arguments.
|
||||
///
|
||||
/// '^': Add FOO as a new argument at the beginning of the command line.
|
||||
///
|
||||
/// '+': Add FOO as a new argument at the end of the command line.
|
||||
///
|
||||
/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command
|
||||
/// line.
|
||||
///
|
||||
/// 'xOPTION': Removes all instances of the literal argument OPTION.
|
||||
///
|
||||
/// 'XOPTION': Removes all instances of the literal argument OPTION,
|
||||
/// and the following argument.
|
||||
///
|
||||
/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox'
|
||||
/// at the end of the command line.
|
||||
///
|
||||
/// \param OS - The stream to write edit information to.
|
||||
/// \param Args - The vector of command line arguments.
|
||||
/// \param Edit - The override command to perform.
|
||||
/// \param SavedStrings - Set to use for storing string representations.
|
||||
static void ApplyOneQAOverride(raw_ostream &OS,
|
||||
SmallVectorImpl<const char*> &Args,
|
||||
StringRef Edit,
|
||||
std::set<std::string> &SavedStrings) {
|
||||
// This does not need to be efficient.
|
||||
|
||||
if (Edit[0] == '^') {
|
||||
const char *Str =
|
||||
GetStableCStr(SavedStrings, Edit.substr(1));
|
||||
OS << "### Adding argument " << Str << " at beginning\n";
|
||||
Args.insert(Args.begin() + 1, Str);
|
||||
} else if (Edit[0] == '+') {
|
||||
const char *Str =
|
||||
GetStableCStr(SavedStrings, Edit.substr(1));
|
||||
OS << "### Adding argument " << Str << " at end\n";
|
||||
Args.push_back(Str);
|
||||
} else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") &&
|
||||
Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) {
|
||||
StringRef MatchPattern = Edit.substr(2).split('/').first;
|
||||
StringRef ReplPattern = Edit.substr(2).split('/').second;
|
||||
ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1);
|
||||
|
||||
for (unsigned i = 1, e = Args.size(); i != e; ++i) {
|
||||
// Ignore end-of-line response file markers
|
||||
if (Args[i] == nullptr)
|
||||
continue;
|
||||
std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]);
|
||||
|
||||
if (Repl != Args[i]) {
|
||||
OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n";
|
||||
Args[i] = GetStableCStr(SavedStrings, Repl);
|
||||
}
|
||||
}
|
||||
} else if (Edit[0] == 'x' || Edit[0] == 'X') {
|
||||
auto Option = Edit.substr(1);
|
||||
for (unsigned i = 1; i < Args.size();) {
|
||||
if (Option == Args[i]) {
|
||||
OS << "### Deleting argument " << Args[i] << '\n';
|
||||
Args.erase(Args.begin() + i);
|
||||
if (Edit[0] == 'X') {
|
||||
if (i < Args.size()) {
|
||||
OS << "### Deleting argument " << Args[i] << '\n';
|
||||
Args.erase(Args.begin() + i);
|
||||
} else
|
||||
OS << "### Invalid X edit, end of command line!\n";
|
||||
}
|
||||
} else
|
||||
++i;
|
||||
}
|
||||
} else if (Edit[0] == 'O') {
|
||||
for (unsigned i = 1; i < Args.size();) {
|
||||
const char *A = Args[i];
|
||||
// Ignore end-of-line response file markers
|
||||
if (A == nullptr)
|
||||
continue;
|
||||
if (A[0] == '-' && A[1] == 'O' &&
|
||||
(A[2] == '\0' ||
|
||||
(A[3] == '\0' && (A[2] == 's' || A[2] == 'z' ||
|
||||
('0' <= A[2] && A[2] <= '9'))))) {
|
||||
OS << "### Deleting argument " << Args[i] << '\n';
|
||||
Args.erase(Args.begin() + i);
|
||||
} else
|
||||
++i;
|
||||
}
|
||||
OS << "### Adding argument " << Edit << " at end\n";
|
||||
Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str()));
|
||||
} else {
|
||||
OS << "### Unrecognized edit: " << Edit << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
/// ApplyQAOverride - Apply a comma separate list of edits to the
|
||||
/// input argument lists. See ApplyOneQAOverride.
|
||||
static void ApplyQAOverride(SmallVectorImpl<const char*> &Args,
|
||||
const char *OverrideStr,
|
||||
std::set<std::string> &SavedStrings) {
|
||||
raw_ostream *OS = &llvm::errs();
|
||||
|
||||
if (OverrideStr[0] == '#') {
|
||||
++OverrideStr;
|
||||
OS = &llvm::nulls();
|
||||
}
|
||||
|
||||
*OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n";
|
||||
|
||||
// This does not need to be efficient.
|
||||
|
||||
const char *S = OverrideStr;
|
||||
while (*S) {
|
||||
const char *End = ::strchr(S, ' ');
|
||||
if (!End)
|
||||
End = S + strlen(S);
|
||||
if (End != S)
|
||||
ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings);
|
||||
S = End;
|
||||
if (*S != '\0')
|
||||
++S;
|
||||
}
|
||||
}
|
||||
|
||||
extern int cc1_main(ArrayRef<const char *> Argv, const char *Argv0,
|
||||
void *MainAddr);
|
||||
extern int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0,
|
||||
void *MainAddr);
|
||||
|
||||
static void insertTargetAndModeArgs(StringRef Target, StringRef Mode,
|
||||
SmallVectorImpl<const char *> &ArgVector,
|
||||
std::set<std::string> &SavedStrings) {
|
||||
if (!Mode.empty()) {
|
||||
// Add the mode flag to the arguments.
|
||||
auto it = ArgVector.begin();
|
||||
if (it != ArgVector.end())
|
||||
++it;
|
||||
ArgVector.insert(it, GetStableCStr(SavedStrings, Mode));
|
||||
}
|
||||
|
||||
if (!Target.empty()) {
|
||||
auto it = ArgVector.begin();
|
||||
if (it != ArgVector.end())
|
||||
++it;
|
||||
const char *arr[] = {"-target", GetStableCStr(SavedStrings, Target)};
|
||||
ArgVector.insert(it, std::begin(arr), std::end(arr));
|
||||
}
|
||||
}
|
||||
|
||||
static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver,
|
||||
SmallVectorImpl<const char *> &Opts) {
|
||||
llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts);
|
||||
// The first instance of '#' should be replaced with '=' in each option.
|
||||
for (const char *Opt : Opts)
|
||||
if (char *NumberSignPtr = const_cast<char *>(::strchr(Opt, '#')))
|
||||
*NumberSignPtr = '=';
|
||||
}
|
||||
|
||||
static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) {
|
||||
// Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE.
|
||||
TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS");
|
||||
if (TheDriver.CCPrintOptions)
|
||||
TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE");
|
||||
|
||||
// Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE.
|
||||
TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS");
|
||||
if (TheDriver.CCPrintHeaders)
|
||||
TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE");
|
||||
|
||||
// Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE.
|
||||
TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS");
|
||||
if (TheDriver.CCLogDiagnostics)
|
||||
TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE");
|
||||
}
|
||||
|
||||
static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient,
|
||||
const std::string &Path) {
|
||||
// If the clang binary happens to be named cl.exe for compatibility reasons,
|
||||
// use clang-cl.exe as the prefix to avoid confusion between clang and MSVC.
|
||||
StringRef ExeBasename(llvm::sys::path::filename(Path));
|
||||
if (ExeBasename.equals_lower("cl.exe"))
|
||||
ExeBasename = "clang-cl.exe";
|
||||
DiagClient->setPrefix(ExeBasename);
|
||||
}
|
||||
|
||||
// This lets us create the DiagnosticsEngine with a properly-filled-out
|
||||
// DiagnosticOptions instance.
|
||||
static DiagnosticOptions *
|
||||
CreateAndPopulateDiagOpts(ArrayRef<const char *> argv) {
|
||||
auto *DiagOpts = new DiagnosticOptions;
|
||||
std::unique_ptr<OptTable> Opts(createDriverOptTable());
|
||||
unsigned MissingArgIndex, MissingArgCount;
|
||||
InputArgList Args =
|
||||
Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount);
|
||||
// We ignore MissingArgCount and the return value of ParseDiagnosticArgs.
|
||||
// Any errors that would be diagnosed here will also be diagnosed later,
|
||||
// when the DiagnosticsEngine actually exists.
|
||||
(void)ParseDiagnosticArgs(*DiagOpts, Args);
|
||||
return DiagOpts;
|
||||
}
|
||||
|
||||
static void SetInstallDir(SmallVectorImpl<const char *> &argv,
|
||||
Driver &TheDriver, bool CanonicalPrefixes) {
|
||||
// Attempt to find the original path used to invoke the driver, to determine
|
||||
// the installed path. We do this manually, because we want to support that
|
||||
// path being a symlink.
|
||||
SmallString<128> InstalledPath(argv[0]);
|
||||
|
||||
// Do a PATH lookup, if there are no directory components.
|
||||
if (llvm::sys::path::filename(InstalledPath) == InstalledPath)
|
||||
if (llvm::ErrorOr<std::string> Tmp = llvm::sys::findProgramByName(
|
||||
llvm::sys::path::filename(InstalledPath.str())))
|
||||
InstalledPath = *Tmp;
|
||||
|
||||
// FIXME: We don't actually canonicalize this, we just make it absolute.
|
||||
if (CanonicalPrefixes)
|
||||
llvm::sys::fs::make_absolute(InstalledPath);
|
||||
|
||||
StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath));
|
||||
if (llvm::sys::fs::exists(InstalledPathParent))
|
||||
TheDriver.setInstalledDir(InstalledPathParent);
|
||||
}
|
||||
|
||||
static int ExecuteCC1Tool(ArrayRef<const char *> argv, StringRef Tool) {
|
||||
void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath;
|
||||
if (Tool == "")
|
||||
return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP);
|
||||
if (Tool == "as")
|
||||
return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP);
|
||||
|
||||
// Reject unknown tools.
|
||||
llvm::errs() << "error: unknown integrated tool '" << Tool << "'\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
int mainEntryClickHouseClang(int argc_, char **argv_) {
|
||||
llvm::sys::PrintStackTraceOnErrorSignal(argv_[0]);
|
||||
llvm::PrettyStackTraceProgram X(argc_, argv_);
|
||||
llvm::llvm_shutdown_obj Y; // Call llvm_shutdown() on exit.
|
||||
|
||||
if (llvm::sys::Process::FixupStandardFileDescriptors())
|
||||
return 1;
|
||||
|
||||
SmallVector<const char *, 256> argv;
|
||||
llvm::SpecificBumpPtrAllocator<char> ArgAllocator;
|
||||
std::error_code EC = llvm::sys::Process::GetArgumentVector(
|
||||
argv, llvm::makeArrayRef(argv_, argc_), ArgAllocator);
|
||||
if (EC) {
|
||||
llvm::errs() << "error: couldn't get arguments: " << EC.message() << '\n';
|
||||
return 1;
|
||||
}
|
||||
|
||||
llvm::InitializeAllTargets();
|
||||
std::string ProgName = argv[0];
|
||||
std::pair<std::string, std::string> TargetAndMode =
|
||||
ToolChain::getTargetAndModeFromProgramName(ProgName);
|
||||
|
||||
llvm::BumpPtrAllocator A;
|
||||
llvm::StringSaver Saver(A);
|
||||
|
||||
// Parse response files using the GNU syntax, unless we're in CL mode. There
|
||||
// are two ways to put clang in CL compatibility mode: argv[0] is either
|
||||
// clang-cl or cl, or --driver-mode=cl is on the command line. The normal
|
||||
// command line parsing can't happen until after response file parsing, so we
|
||||
// have to manually search for a --driver-mode=cl argument the hard way.
|
||||
// Finally, our -cc1 tools don't care which tokenization mode we use because
|
||||
// response files written by clang will tokenize the same way in either mode.
|
||||
bool ClangCLMode = false;
|
||||
if (TargetAndMode.second == "--driver-mode=cl" ||
|
||||
std::find_if(argv.begin(), argv.end(), [](const char *F) {
|
||||
return F && strcmp(F, "--driver-mode=cl") == 0;
|
||||
}) != argv.end()) {
|
||||
ClangCLMode = true;
|
||||
}
|
||||
enum { Default, POSIX, Windows } RSPQuoting = Default;
|
||||
for (const char *F : argv) {
|
||||
if (strcmp(F, "--rsp-quoting=posix") == 0)
|
||||
RSPQuoting = POSIX;
|
||||
else if (strcmp(F, "--rsp-quoting=windows") == 0)
|
||||
RSPQuoting = Windows;
|
||||
}
|
||||
|
||||
// Determines whether we want nullptr markers in argv to indicate response
|
||||
// files end-of-lines. We only use this for the /LINK driver argument with
|
||||
// clang-cl.exe on Windows.
|
||||
bool MarkEOLs = ClangCLMode;
|
||||
|
||||
llvm::cl::TokenizerCallback Tokenizer;
|
||||
if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode))
|
||||
Tokenizer = &llvm::cl::TokenizeWindowsCommandLine;
|
||||
else
|
||||
Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
|
||||
|
||||
if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1"))
|
||||
MarkEOLs = false;
|
||||
llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs);
|
||||
|
||||
// Handle -cc1 integrated tools, even if -cc1 was expanded from a response
|
||||
// file.
|
||||
auto FirstArg = std::find_if(argv.begin() + 1, argv.end(),
|
||||
[](const char *A) { return A != nullptr; });
|
||||
if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) {
|
||||
// If -cc1 came from a response file, remove the EOL sentinels.
|
||||
if (MarkEOLs) {
|
||||
auto newEnd = std::remove(argv.begin(), argv.end(), nullptr);
|
||||
argv.resize(newEnd - argv.begin());
|
||||
}
|
||||
return ExecuteCC1Tool(argv, argv[1] + 4);
|
||||
}
|
||||
|
||||
bool CanonicalPrefixes = true;
|
||||
for (int i = 1, size = argv.size(); i < size; ++i) {
|
||||
// Skip end-of-line response file markers
|
||||
if (argv[i] == nullptr)
|
||||
continue;
|
||||
if (StringRef(argv[i]) == "-no-canonical-prefixes") {
|
||||
CanonicalPrefixes = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle CL and _CL_ which permits additional command line options to be
|
||||
// prepended or appended.
|
||||
if (ClangCLMode) {
|
||||
// Arguments in "CL" are prepended.
|
||||
llvm::Optional<std::string> OptCL = llvm::sys::Process::GetEnv("CL");
|
||||
if (OptCL.hasValue()) {
|
||||
SmallVector<const char *, 8> PrependedOpts;
|
||||
getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts);
|
||||
|
||||
// Insert right after the program name to prepend to the argument list.
|
||||
argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end());
|
||||
}
|
||||
// Arguments in "_CL_" are appended.
|
||||
llvm::Optional<std::string> Opt_CL_ = llvm::sys::Process::GetEnv("_CL_");
|
||||
if (Opt_CL_.hasValue()) {
|
||||
SmallVector<const char *, 8> AppendedOpts;
|
||||
getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts);
|
||||
|
||||
// Insert at the end of the argument list to append.
|
||||
argv.append(AppendedOpts.begin(), AppendedOpts.end());
|
||||
}
|
||||
}
|
||||
|
||||
std::set<std::string> SavedStrings;
|
||||
// Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the
|
||||
// scenes.
|
||||
if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) {
|
||||
// FIXME: Driver shouldn't take extra initial argument.
|
||||
ApplyQAOverride(argv, OverrideStr, SavedStrings);
|
||||
}
|
||||
|
||||
std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes);
|
||||
|
||||
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts =
|
||||
CreateAndPopulateDiagOpts(argv);
|
||||
|
||||
TextDiagnosticPrinter *DiagClient
|
||||
= new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
|
||||
FixupDiagPrefixExeName(DiagClient, Path);
|
||||
|
||||
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
|
||||
|
||||
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
|
||||
|
||||
if (!DiagOpts->DiagnosticSerializationFile.empty()) {
|
||||
auto SerializedConsumer =
|
||||
clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile,
|
||||
&*DiagOpts, /*MergeChildRecords=*/true);
|
||||
Diags.setClient(new ChainedDiagnosticConsumer(
|
||||
Diags.takeClient(), std::move(SerializedConsumer)));
|
||||
}
|
||||
|
||||
ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false);
|
||||
|
||||
Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags);
|
||||
SetInstallDir(argv, TheDriver, CanonicalPrefixes);
|
||||
|
||||
insertTargetAndModeArgs(TargetAndMode.first, TargetAndMode.second, argv,
|
||||
SavedStrings);
|
||||
|
||||
SetBackdoorDriverOutputsFromEnvVars(TheDriver);
|
||||
|
||||
std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(argv));
|
||||
int Res = 1;
|
||||
if (C && !C->containsError()) {
|
||||
SmallVector<std::pair<int, const Command *>, 4> FailingCommands;
|
||||
Res = TheDriver.ExecuteCompilation(*C, FailingCommands);
|
||||
|
||||
// Force a crash to test the diagnostics.
|
||||
if (TheDriver.GenReproducer) {
|
||||
Diags.Report(diag::err_drv_force_crash)
|
||||
<< !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH");
|
||||
|
||||
// Pretend that every command failed.
|
||||
FailingCommands.clear();
|
||||
for (const auto &J : C->getJobs())
|
||||
if (const Command *C = dyn_cast<Command>(&J))
|
||||
FailingCommands.push_back(std::make_pair(-1, C));
|
||||
}
|
||||
|
||||
for (const auto &P : FailingCommands) {
|
||||
int CommandRes = P.first;
|
||||
const Command *FailingCommand = P.second;
|
||||
if (!Res)
|
||||
Res = CommandRes;
|
||||
|
||||
// If result status is < 0, then the driver command signalled an error.
|
||||
// If result status is 70, then the driver command reported a fatal error.
|
||||
// On Windows, abort will return an exit code of 3. In these cases,
|
||||
// generate additional diagnostic information if possible.
|
||||
bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70;
|
||||
#ifdef LLVM_ON_WIN32
|
||||
DiagnoseCrash |= CommandRes == 3;
|
||||
#endif
|
||||
if (DiagnoseCrash) {
|
||||
TheDriver.generateCompilationDiagnostics(*C, *FailingCommand);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Diags.getClient()->finish();
|
||||
|
||||
// If any timers were active but haven't been destroyed yet, print their
|
||||
// results now. This happens in -disable-free mode.
|
||||
llvm::TimerGroup::printAll(llvm::errs());
|
||||
|
||||
#ifdef LLVM_ON_WIN32
|
||||
// Exit status should not be negative on Win32, unless abnormal termination.
|
||||
// Once abnormal termiation was caught, negative status should not be
|
||||
// propagated.
|
||||
if (Res < 0)
|
||||
Res = 1;
|
||||
#endif
|
||||
|
||||
// If we have multiple failing commands, we return the result of the first
|
||||
// failing command.
|
||||
return Res;
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
#include "lld/Driver/Driver.h"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/ADT/StringSwitch.h"
|
||||
#include "llvm/ADT/Twine.h"
|
||||
#include "llvm/Support/ManagedStatic.h"
|
||||
#include "llvm/Support/Path.h"
|
||||
#include "llvm/Support/PrettyStackTrace.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
|
||||
using namespace lld;
|
||||
using namespace llvm;
|
||||
using namespace llvm::sys;
|
||||
|
||||
int mainEntryClickHouseLLD(int Argc, char **Argv)
|
||||
{
|
||||
// Standard set up, so program fails gracefully.
|
||||
sys::PrintStackTraceOnErrorSignal(Argv[0]);
|
||||
PrettyStackTraceProgram StackPrinter(Argc, Argv);
|
||||
llvm_shutdown_obj Shutdown;
|
||||
|
||||
std::vector<const char *> Args(Argv, Argv + Argc);
|
||||
return !elf::link(Args, true);
|
||||
}
|
||||
@ -1 +0,0 @@
|
||||
Compiler-5.0.0
|
||||
@ -1 +0,0 @@
|
||||
Compiler-5.0.0
|
||||
@ -1,54 +0,0 @@
|
||||
|
||||
add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG)
|
||||
|
||||
link_directories(${LLVM_LIBRARY_DIRS})
|
||||
|
||||
add_library(clickhouse-compiler-lib
|
||||
driver.cpp
|
||||
cc1_main.cpp
|
||||
cc1as_main.cpp
|
||||
lld.cpp)
|
||||
|
||||
target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0)
|
||||
|
||||
string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti
|
||||
|
||||
llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
|
||||
|
||||
message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
|
||||
|
||||
target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
|
||||
|
||||
# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.
|
||||
|
||||
target_link_libraries(clickhouse-compiler-lib PRIVATE
|
||||
|
||||
clangBasic clangCodeGen clangDriver
|
||||
clangFrontend
|
||||
clangFrontendTool
|
||||
clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend
|
||||
clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers
|
||||
clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic
|
||||
|
||||
lldCOFF
|
||||
lldDriver
|
||||
lldELF
|
||||
lldMinGW
|
||||
lldMachO
|
||||
lldReaderWriter
|
||||
lldYAML
|
||||
lldCommon
|
||||
lldCore
|
||||
#lldWasm
|
||||
|
||||
${REQUIRED_LLVM_LIBRARIES}
|
||||
|
||||
#Polly
|
||||
#PollyISL
|
||||
#PollyPPCG
|
||||
|
||||
PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARIES} Threads::Threads
|
||||
${MALLOC_LIBRARIES}
|
||||
${GLIBC_COMPATIBILITY_LIBRARIES}
|
||||
${MEMCPY_LIBRARIES}
|
||||
)
|
||||
@ -1,63 +0,0 @@
|
||||
==============================================================================
|
||||
LLVM Release License
|
||||
==============================================================================
|
||||
University of Illinois/NCSA
|
||||
Open Source License
|
||||
|
||||
Copyright (c) 2007-2016 University of Illinois at Urbana-Champaign.
|
||||
All rights reserved.
|
||||
|
||||
Developed by:
|
||||
|
||||
LLVM Team
|
||||
|
||||
University of Illinois at Urbana-Champaign
|
||||
|
||||
http://llvm.org
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal with
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
of the Software, and to permit persons to whom the Software is furnished to do
|
||||
so, subject to the following conditions:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimers.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimers in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the names of the LLVM Team, University of Illinois at
|
||||
Urbana-Champaign, nor the names of its contributors may be used to
|
||||
endorse or promote products derived from this Software without specific
|
||||
prior written permission.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
|
||||
SOFTWARE.
|
||||
|
||||
==============================================================================
|
||||
The LLVM software contains code written by third parties. Such software will
|
||||
have its own individual LICENSE.TXT file in the directory in which it appears.
|
||||
This file will describe the copyrights, license, and restrictions which apply
|
||||
to that code.
|
||||
|
||||
The disclaimer of warranty in the University of Illinois Open Source License
|
||||
applies to all code in the LLVM Distribution, and nothing in any of the
|
||||
other licenses gives permission to use the names of the LLVM Team or the
|
||||
University of Illinois to endorse or promote products derived from this
|
||||
Software.
|
||||
|
||||
The following pieces of software have additional or alternate copyrights,
|
||||
licenses, and/or restrictions:
|
||||
|
||||
Program Directory
|
||||
------- ---------
|
||||
<none yet>
|
||||
|
||||
@ -1,242 +0,0 @@
|
||||
//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This is the entry point to the clang -cc1 functionality, which implements the
|
||||
// core compiler functionality along with a number of additional tools for
|
||||
// demonstration and testing purposes.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "llvm/Option/Arg.h"
|
||||
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
|
||||
#include "clang/Config/config.h"
|
||||
#include "clang/Driver/DriverDiagnostic.h"
|
||||
#include "clang/Driver/Options.h"
|
||||
#include "clang/Frontend/CompilerInstance.h"
|
||||
#include "clang/Frontend/CompilerInvocation.h"
|
||||
#include "clang/Frontend/FrontendDiagnostic.h"
|
||||
#include "clang/Frontend/TextDiagnosticBuffer.h"
|
||||
#include "clang/Frontend/TextDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/Utils.h"
|
||||
#include "clang/FrontendTool/Utils.h"
|
||||
#include "llvm/ADT/Statistic.h"
|
||||
#include "llvm/LinkAllPasses.h"
|
||||
#include "llvm/Option/ArgList.h"
|
||||
#include "llvm/Option/OptTable.h"
|
||||
#include "llvm/Support/Compiler.h"
|
||||
#include "llvm/Support/ErrorHandling.h"
|
||||
#include "llvm/Support/ManagedStatic.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
#include "llvm/Support/TargetSelect.h"
|
||||
#include "llvm/Support/Timer.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
#include <cstdio>
|
||||
|
||||
#ifdef CLANG_HAVE_RLIMITS
|
||||
#include <sys/resource.h>
|
||||
#endif
|
||||
|
||||
// have no .a version in packages
|
||||
#undef LINK_POLLY_INTO_TOOLS
|
||||
|
||||
using namespace clang;
|
||||
using namespace llvm::opt;
|
||||
|
||||
//===----------------------------------------------------------------------===//
|
||||
// Main driver
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
static void LLVMErrorHandler(void *UserData, const std::string &Message,
|
||||
bool GenCrashDiag) {
|
||||
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
|
||||
|
||||
Diags.Report(diag::err_fe_error_backend) << Message;
|
||||
|
||||
// Run the interrupt handlers to make sure any special cleanups get done, in
|
||||
// particular that we remove files registered with RemoveFileOnSignal.
|
||||
llvm::sys::RunInterruptHandlers();
|
||||
|
||||
// We cannot recover from llvm errors. When reporting a fatal error, exit
|
||||
// with status 70 to generate crash diagnostics. For BSD systems this is
|
||||
// defined as an internal software error. Otherwise, exit with status 1.
|
||||
exit(GenCrashDiag ? 70 : 1);
|
||||
}
|
||||
|
||||
#ifdef LINK_POLLY_INTO_TOOLS
|
||||
namespace polly {
|
||||
void initializePollyPasses(llvm::PassRegistry &Registry);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef CLANG_HAVE_RLIMITS
|
||||
// The amount of stack we think is "sufficient". If less than this much is
|
||||
// available, we may be unable to reach our template instantiation depth
|
||||
// limit and other similar limits.
|
||||
// FIXME: Unify this with the stack we request when spawning a thread to build
|
||||
// a module.
|
||||
static const int kSufficientStack = 8 << 20;
|
||||
|
||||
#if defined(__linux__) && defined(__PIE__)
|
||||
static size_t getCurrentStackAllocation() {
|
||||
// If we can't compute the current stack usage, allow for 512K of command
|
||||
// line arguments and environment.
|
||||
size_t Usage = 512 * 1024;
|
||||
if (FILE *StatFile = fopen("/proc/self/stat", "r")) {
|
||||
// We assume that the stack extends from its current address to the end of
|
||||
// the environment space. In reality, there is another string literal (the
|
||||
// program name) after the environment, but this is close enough (we only
|
||||
// need to be within 100K or so).
|
||||
unsigned long StackPtr, EnvEnd;
|
||||
// Disable silly GCC -Wformat warning that complains about length
|
||||
// modifiers on ignored format specifiers. We want to retain these
|
||||
// for documentation purposes even though they have no effect.
|
||||
#if defined(__GNUC__) && !defined(__clang__)
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wformat"
|
||||
#endif
|
||||
if (fscanf(StatFile,
|
||||
"%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu "
|
||||
"%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu "
|
||||
"%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d "
|
||||
"%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d",
|
||||
&StackPtr, &EnvEnd) == 2) {
|
||||
#if defined(__GNUC__) && !defined(__clang__)
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd;
|
||||
}
|
||||
fclose(StatFile);
|
||||
}
|
||||
return Usage;
|
||||
}
|
||||
|
||||
#include <alloca.h>
|
||||
|
||||
LLVM_ATTRIBUTE_NOINLINE
|
||||
static void ensureStackAddressSpace(int ExtraChunks = 0) {
|
||||
// Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary
|
||||
// relatively close to the stack (they are only guaranteed to be 128MiB
|
||||
// apart). This results in crashes if we happen to heap-allocate more than
|
||||
// 128MiB before we reach our stack high-water mark.
|
||||
//
|
||||
// To avoid these crashes, ensure that we have sufficient virtual memory
|
||||
// pages allocated before we start running.
|
||||
size_t Curr = getCurrentStackAllocation();
|
||||
const int kTargetStack = kSufficientStack - 256 * 1024;
|
||||
if (Curr < kTargetStack) {
|
||||
volatile char *volatile Alloc =
|
||||
static_cast<volatile char *>(alloca(kTargetStack - Curr));
|
||||
Alloc[0] = 0;
|
||||
Alloc[kTargetStack - Curr - 1] = 0;
|
||||
}
|
||||
}
|
||||
#else
|
||||
static void ensureStackAddressSpace() {}
|
||||
#endif
|
||||
|
||||
/// Attempt to ensure that we have at least 8MiB of usable stack space.
|
||||
static void ensureSufficientStack() {
|
||||
struct rlimit rlim;
|
||||
if (getrlimit(RLIMIT_STACK, &rlim) != 0)
|
||||
return;
|
||||
|
||||
// Increase the soft stack limit to our desired level, if necessary and
|
||||
// possible.
|
||||
if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) {
|
||||
// Try to allocate sufficient stack.
|
||||
if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack)
|
||||
rlim.rlim_cur = kSufficientStack;
|
||||
else if (rlim.rlim_cur == rlim.rlim_max)
|
||||
return;
|
||||
else
|
||||
rlim.rlim_cur = rlim.rlim_max;
|
||||
|
||||
if (setrlimit(RLIMIT_STACK, &rlim) != 0 ||
|
||||
rlim.rlim_cur != kSufficientStack)
|
||||
return;
|
||||
}
|
||||
|
||||
// We should now have a stack of size at least kSufficientStack. Ensure
|
||||
// that we can actually use that much, if necessary.
|
||||
ensureStackAddressSpace();
|
||||
}
|
||||
#else
|
||||
static void ensureSufficientStack() {}
|
||||
#endif
|
||||
|
||||
int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
|
||||
ensureSufficientStack();
|
||||
|
||||
std::unique_ptr<CompilerInstance> Clang(new CompilerInstance());
|
||||
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
|
||||
|
||||
// Register the support for object-file-wrapped Clang modules.
|
||||
auto PCHOps = Clang->getPCHContainerOperations();
|
||||
PCHOps->registerWriter(llvm::make_unique<ObjectFilePCHContainerWriter>());
|
||||
PCHOps->registerReader(llvm::make_unique<ObjectFilePCHContainerReader>());
|
||||
|
||||
// Initialize targets first, so that --version shows registered targets.
|
||||
llvm::InitializeAllTargets();
|
||||
llvm::InitializeAllTargetMCs();
|
||||
llvm::InitializeAllAsmPrinters();
|
||||
llvm::InitializeAllAsmParsers();
|
||||
|
||||
#ifdef LINK_POLLY_INTO_TOOLS
|
||||
llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry();
|
||||
polly::initializePollyPasses(Registry);
|
||||
#endif
|
||||
|
||||
// Buffer diagnostics from argument parsing so that we can output them using a
|
||||
// well formed diagnostic object.
|
||||
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
|
||||
TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer;
|
||||
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer);
|
||||
bool Success = CompilerInvocation::CreateFromArgs(
|
||||
Clang->getInvocation(), Argv.begin(), Argv.end(), Diags);
|
||||
|
||||
// Infer the builtin include path if unspecified.
|
||||
if (Clang->getHeaderSearchOpts().UseBuiltinIncludes &&
|
||||
Clang->getHeaderSearchOpts().ResourceDir.empty())
|
||||
Clang->getHeaderSearchOpts().ResourceDir =
|
||||
CompilerInvocation::GetResourcesPath(Argv0, MainAddr);
|
||||
|
||||
// Create the actual diagnostics engine.
|
||||
Clang->createDiagnostics();
|
||||
if (!Clang->hasDiagnostics())
|
||||
return 1;
|
||||
|
||||
// Set an error handler, so that any LLVM backend diagnostics go through our
|
||||
// error handler.
|
||||
llvm::install_fatal_error_handler(LLVMErrorHandler,
|
||||
static_cast<void*>(&Clang->getDiagnostics()));
|
||||
|
||||
DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics());
|
||||
if (!Success)
|
||||
return 1;
|
||||
|
||||
// Execute the frontend actions.
|
||||
Success = ExecuteCompilerInvocation(Clang.get());
|
||||
|
||||
// If any timers were active but haven't been destroyed yet, print their
|
||||
// results now. This happens in -disable-free mode.
|
||||
llvm::TimerGroup::printAll(llvm::errs());
|
||||
|
||||
// Our error handler depends on the Diagnostics object, which we're
|
||||
// potentially about to delete. Uninstall the handler now so that any
|
||||
// later errors use the default handling behavior instead.
|
||||
llvm::remove_fatal_error_handler();
|
||||
|
||||
// When running with -disable-free, don't do any destruction or shutdown.
|
||||
if (Clang->getFrontendOpts().DisableFree) {
|
||||
BuryPointer(std::move(Clang));
|
||||
return !Success;
|
||||
}
|
||||
|
||||
return !Success;
|
||||
}
|
||||
@ -1,540 +0,0 @@
|
||||
//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This is the entry point to the clang -cc1as functionality, which implements
|
||||
// the direct interface to the LLVM MC based assembler.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "clang/Basic/Diagnostic.h"
|
||||
#include "clang/Basic/DiagnosticOptions.h"
|
||||
#include "clang/Driver/DriverDiagnostic.h"
|
||||
#include "clang/Driver/Options.h"
|
||||
#include "clang/Frontend/FrontendDiagnostic.h"
|
||||
#include "clang/Frontend/TextDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/Utils.h"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/ADT/StringSwitch.h"
|
||||
#include "llvm/ADT/Triple.h"
|
||||
#include "llvm/IR/DataLayout.h"
|
||||
#include "llvm/MC/MCAsmBackend.h"
|
||||
#include "llvm/MC/MCAsmInfo.h"
|
||||
#include "llvm/MC/MCCodeEmitter.h"
|
||||
#include "llvm/MC/MCContext.h"
|
||||
#include "llvm/MC/MCInstrInfo.h"
|
||||
#include "llvm/MC/MCObjectFileInfo.h"
|
||||
#include "llvm/MC/MCParser/MCAsmParser.h"
|
||||
#include "llvm/MC/MCParser/MCTargetAsmParser.h"
|
||||
#include "llvm/MC/MCRegisterInfo.h"
|
||||
#include "llvm/MC/MCStreamer.h"
|
||||
#include "llvm/MC/MCSubtargetInfo.h"
|
||||
#include "llvm/MC/MCTargetOptions.h"
|
||||
#include "llvm/Option/Arg.h"
|
||||
#include "llvm/Option/ArgList.h"
|
||||
#include "llvm/Option/OptTable.h"
|
||||
#include "llvm/Support/CommandLine.h"
|
||||
#include "llvm/Support/ErrorHandling.h"
|
||||
#include "llvm/Support/FileSystem.h"
|
||||
#include "llvm/Support/FormattedStream.h"
|
||||
#include "llvm/Support/Host.h"
|
||||
#include "llvm/Support/MemoryBuffer.h"
|
||||
#include "llvm/Support/Path.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
#include "llvm/Support/SourceMgr.h"
|
||||
#include "llvm/Support/TargetRegistry.h"
|
||||
#include "llvm/Support/TargetSelect.h"
|
||||
#include "llvm/Support/Timer.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
#include <memory>
|
||||
#include <system_error>
|
||||
using namespace clang;
|
||||
using namespace clang::driver;
|
||||
using namespace clang::driver::options;
|
||||
using namespace llvm;
|
||||
using namespace llvm::opt;
|
||||
|
||||
namespace {
|
||||
|
||||
/// \brief Helper class for representing a single invocation of the assembler.
|
||||
struct AssemblerInvocation {
|
||||
/// @name Target Options
|
||||
/// @{
|
||||
|
||||
/// The name of the target triple to assemble for.
|
||||
std::string Triple;
|
||||
|
||||
/// If given, the name of the target CPU to determine which instructions
|
||||
/// are legal.
|
||||
std::string CPU;
|
||||
|
||||
/// The list of target specific features to enable or disable -- this should
|
||||
/// be a list of strings starting with '+' or '-'.
|
||||
std::vector<std::string> Features;
|
||||
|
||||
/// The list of symbol definitions.
|
||||
std::vector<std::string> SymbolDefs;
|
||||
|
||||
/// @}
|
||||
/// @name Language Options
|
||||
/// @{
|
||||
|
||||
std::vector<std::string> IncludePaths;
|
||||
unsigned NoInitialTextSection : 1;
|
||||
unsigned SaveTemporaryLabels : 1;
|
||||
unsigned GenDwarfForAssembly : 1;
|
||||
unsigned RelaxELFRelocations : 1;
|
||||
unsigned DwarfVersion;
|
||||
std::string DwarfDebugFlags;
|
||||
std::string DwarfDebugProducer;
|
||||
std::string DebugCompilationDir;
|
||||
llvm::DebugCompressionType CompressDebugSections =
|
||||
llvm::DebugCompressionType::None;
|
||||
std::string MainFileName;
|
||||
|
||||
/// @}
|
||||
/// @name Frontend Options
|
||||
/// @{
|
||||
|
||||
std::string InputFile;
|
||||
std::vector<std::string> LLVMArgs;
|
||||
std::string OutputPath;
|
||||
enum FileType {
|
||||
FT_Asm, ///< Assembly (.s) output, transliterate mode.
|
||||
FT_Null, ///< No output, for timing purposes.
|
||||
FT_Obj ///< Object file output.
|
||||
};
|
||||
FileType OutputType;
|
||||
unsigned ShowHelp : 1;
|
||||
unsigned ShowVersion : 1;
|
||||
|
||||
/// @}
|
||||
/// @name Transliterate Options
|
||||
/// @{
|
||||
|
||||
unsigned OutputAsmVariant;
|
||||
unsigned ShowEncoding : 1;
|
||||
unsigned ShowInst : 1;
|
||||
|
||||
/// @}
|
||||
/// @name Assembler Options
|
||||
/// @{
|
||||
|
||||
unsigned RelaxAll : 1;
|
||||
unsigned NoExecStack : 1;
|
||||
unsigned FatalWarnings : 1;
|
||||
unsigned IncrementalLinkerCompatible : 1;
|
||||
|
||||
/// The name of the relocation model to use.
|
||||
std::string RelocationModel;
|
||||
|
||||
/// @}
|
||||
|
||||
public:
|
||||
AssemblerInvocation() {
|
||||
Triple = "";
|
||||
NoInitialTextSection = 0;
|
||||
InputFile = "-";
|
||||
OutputPath = "-";
|
||||
OutputType = FT_Asm;
|
||||
OutputAsmVariant = 0;
|
||||
ShowInst = 0;
|
||||
ShowEncoding = 0;
|
||||
RelaxAll = 0;
|
||||
NoExecStack = 0;
|
||||
FatalWarnings = 0;
|
||||
IncrementalLinkerCompatible = 0;
|
||||
DwarfVersion = 0;
|
||||
}
|
||||
|
||||
static bool CreateFromArgs(AssemblerInvocation &Res,
|
||||
ArrayRef<const char *> Argv,
|
||||
DiagnosticsEngine &Diags);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
|
||||
ArrayRef<const char *> Argv,
|
||||
DiagnosticsEngine &Diags) {
|
||||
bool Success = true;
|
||||
|
||||
// Parse the arguments.
|
||||
std::unique_ptr<OptTable> OptTbl(createDriverOptTable());
|
||||
|
||||
const unsigned IncludedFlagsBitmask = options::CC1AsOption;
|
||||
unsigned MissingArgIndex, MissingArgCount;
|
||||
InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount,
|
||||
IncludedFlagsBitmask);
|
||||
|
||||
// Check for missing argument error.
|
||||
if (MissingArgCount) {
|
||||
Diags.Report(diag::err_drv_missing_argument)
|
||||
<< Args.getArgString(MissingArgIndex) << MissingArgCount;
|
||||
Success = false;
|
||||
}
|
||||
|
||||
// Issue errors on unknown arguments.
|
||||
for (const Arg *A : Args.filtered(OPT_UNKNOWN)) {
|
||||
Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args);
|
||||
Success = false;
|
||||
}
|
||||
|
||||
// Construct the invocation.
|
||||
|
||||
// Target Options
|
||||
Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple));
|
||||
Opts.CPU = Args.getLastArgValue(OPT_target_cpu);
|
||||
Opts.Features = Args.getAllArgValues(OPT_target_feature);
|
||||
|
||||
// Use the default target triple if unspecified.
|
||||
if (Opts.Triple.empty())
|
||||
Opts.Triple = llvm::sys::getDefaultTargetTriple();
|
||||
|
||||
// Language Options
|
||||
Opts.IncludePaths = Args.getAllArgValues(OPT_I);
|
||||
Opts.NoInitialTextSection = Args.hasArg(OPT_n);
|
||||
Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels);
|
||||
// Any DebugInfoKind implies GenDwarfForAssembly.
|
||||
Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ);
|
||||
|
||||
if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections,
|
||||
OPT_compress_debug_sections_EQ)) {
|
||||
if (A->getOption().getID() == OPT_compress_debug_sections) {
|
||||
// TODO: be more clever about the compression type auto-detection
|
||||
Opts.CompressDebugSections = llvm::DebugCompressionType::GNU;
|
||||
} else {
|
||||
Opts.CompressDebugSections =
|
||||
llvm::StringSwitch<llvm::DebugCompressionType>(A->getValue())
|
||||
.Case("none", llvm::DebugCompressionType::None)
|
||||
.Case("zlib", llvm::DebugCompressionType::Z)
|
||||
.Case("zlib-gnu", llvm::DebugCompressionType::GNU)
|
||||
.Default(llvm::DebugCompressionType::None);
|
||||
}
|
||||
}
|
||||
|
||||
Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations);
|
||||
Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags);
|
||||
Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags);
|
||||
Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer);
|
||||
Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir);
|
||||
Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name);
|
||||
|
||||
// Frontend Options
|
||||
if (Args.hasArg(OPT_INPUT)) {
|
||||
bool First = true;
|
||||
for (const Arg *A : Args.filtered(OPT_INPUT)) {
|
||||
if (First) {
|
||||
Opts.InputFile = A->getValue();
|
||||
First = false;
|
||||
} else {
|
||||
Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args);
|
||||
Success = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
|
||||
Opts.OutputPath = Args.getLastArgValue(OPT_o);
|
||||
if (Arg *A = Args.getLastArg(OPT_filetype)) {
|
||||
StringRef Name = A->getValue();
|
||||
unsigned OutputType = StringSwitch<unsigned>(Name)
|
||||
.Case("asm", FT_Asm)
|
||||
.Case("null", FT_Null)
|
||||
.Case("obj", FT_Obj)
|
||||
.Default(~0U);
|
||||
if (OutputType == ~0U) {
|
||||
Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name;
|
||||
Success = false;
|
||||
} else
|
||||
Opts.OutputType = FileType(OutputType);
|
||||
}
|
||||
Opts.ShowHelp = Args.hasArg(OPT_help);
|
||||
Opts.ShowVersion = Args.hasArg(OPT_version);
|
||||
|
||||
// Transliterate Options
|
||||
Opts.OutputAsmVariant =
|
||||
getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags);
|
||||
Opts.ShowEncoding = Args.hasArg(OPT_show_encoding);
|
||||
Opts.ShowInst = Args.hasArg(OPT_show_inst);
|
||||
|
||||
// Assemble Options
|
||||
Opts.RelaxAll = Args.hasArg(OPT_mrelax_all);
|
||||
Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack);
|
||||
Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings);
|
||||
Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic");
|
||||
Opts.IncrementalLinkerCompatible =
|
||||
Args.hasArg(OPT_mincremental_linker_compatible);
|
||||
Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym);
|
||||
|
||||
return Success;
|
||||
}
|
||||
|
||||
static std::unique_ptr<raw_fd_ostream>
|
||||
getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags,
|
||||
bool Binary) {
|
||||
if (Opts.OutputPath.empty())
|
||||
Opts.OutputPath = "-";
|
||||
|
||||
// Make sure that the Out file gets unlinked from the disk if we get a
|
||||
// SIGINT.
|
||||
if (Opts.OutputPath != "-")
|
||||
sys::RemoveFileOnSignal(Opts.OutputPath);
|
||||
|
||||
std::error_code EC;
|
||||
auto Out = llvm::make_unique<raw_fd_ostream>(
|
||||
Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text));
|
||||
if (EC) {
|
||||
Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath
|
||||
<< EC.message();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return Out;
|
||||
}
|
||||
|
||||
static bool ExecuteAssembler(AssemblerInvocation &Opts,
|
||||
DiagnosticsEngine &Diags) {
|
||||
// Get the target specific parser.
|
||||
std::string Error;
|
||||
const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error);
|
||||
if (!TheTarget)
|
||||
return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
|
||||
|
||||
ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer =
|
||||
MemoryBuffer::getFileOrSTDIN(Opts.InputFile);
|
||||
|
||||
if (std::error_code EC = Buffer.getError()) {
|
||||
Error = EC.message();
|
||||
return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile;
|
||||
}
|
||||
|
||||
SourceMgr SrcMgr;
|
||||
|
||||
// Tell SrcMgr about this buffer, which is what the parser will pick up.
|
||||
SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc());
|
||||
|
||||
// Record the location of the include directories so that the lexer can find
|
||||
// it later.
|
||||
SrcMgr.setIncludeDirs(Opts.IncludePaths);
|
||||
|
||||
std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(Opts.Triple));
|
||||
assert(MRI && "Unable to create target register info!");
|
||||
|
||||
std::unique_ptr<MCAsmInfo> MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple));
|
||||
assert(MAI && "Unable to create target asm info!");
|
||||
|
||||
// Ensure MCAsmInfo initialization occurs before any use, otherwise sections
|
||||
// may be created with a combination of default and explicit settings.
|
||||
MAI->setCompressDebugSections(Opts.CompressDebugSections);
|
||||
|
||||
MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations);
|
||||
|
||||
bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj;
|
||||
std::unique_ptr<raw_fd_ostream> FDOS = getOutputStream(Opts, Diags, IsBinary);
|
||||
if (!FDOS)
|
||||
return true;
|
||||
|
||||
// FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and
|
||||
// MCObjectFileInfo needs a MCContext reference in order to initialize itself.
|
||||
std::unique_ptr<MCObjectFileInfo> MOFI(new MCObjectFileInfo());
|
||||
|
||||
MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr);
|
||||
|
||||
bool PIC = false;
|
||||
if (Opts.RelocationModel == "static") {
|
||||
PIC = false;
|
||||
} else if (Opts.RelocationModel == "pic") {
|
||||
PIC = true;
|
||||
} else {
|
||||
assert(Opts.RelocationModel == "dynamic-no-pic" &&
|
||||
"Invalid PIC model!");
|
||||
PIC = false;
|
||||
}
|
||||
|
||||
MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx);
|
||||
if (Opts.SaveTemporaryLabels)
|
||||
Ctx.setAllowTemporaryLabels(false);
|
||||
if (Opts.GenDwarfForAssembly)
|
||||
Ctx.setGenDwarfForAssembly(true);
|
||||
if (!Opts.DwarfDebugFlags.empty())
|
||||
Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags));
|
||||
if (!Opts.DwarfDebugProducer.empty())
|
||||
Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer));
|
||||
if (!Opts.DebugCompilationDir.empty())
|
||||
Ctx.setCompilationDir(Opts.DebugCompilationDir);
|
||||
if (!Opts.MainFileName.empty())
|
||||
Ctx.setMainFileName(StringRef(Opts.MainFileName));
|
||||
Ctx.setDwarfVersion(Opts.DwarfVersion);
|
||||
|
||||
// Build up the feature string from the target feature list.
|
||||
std::string FS;
|
||||
if (!Opts.Features.empty()) {
|
||||
FS = Opts.Features[0];
|
||||
for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i)
|
||||
FS += "," + Opts.Features[i];
|
||||
}
|
||||
|
||||
std::unique_ptr<MCStreamer> Str;
|
||||
|
||||
std::unique_ptr<MCInstrInfo> MCII(TheTarget->createMCInstrInfo());
|
||||
std::unique_ptr<MCSubtargetInfo> STI(
|
||||
TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS));
|
||||
|
||||
raw_pwrite_stream *Out = FDOS.get();
|
||||
std::unique_ptr<buffer_ostream> BOS;
|
||||
|
||||
// FIXME: There is a bit of code duplication with addPassesToEmitFile.
|
||||
if (Opts.OutputType == AssemblerInvocation::FT_Asm) {
|
||||
MCInstPrinter *IP = TheTarget->createMCInstPrinter(
|
||||
llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI);
|
||||
MCCodeEmitter *CE = nullptr;
|
||||
MCAsmBackend *MAB = nullptr;
|
||||
if (Opts.ShowEncoding) {
|
||||
CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx);
|
||||
MCTargetOptions Options;
|
||||
MAB = TheTarget->createMCAsmBackend(*STI, *MRI, Options);
|
||||
}
|
||||
auto FOut = llvm::make_unique<formatted_raw_ostream>(*Out);
|
||||
Str.reset(TheTarget->createAsmStreamer(
|
||||
Ctx, std::move(FOut), /*asmverbose*/ true,
|
||||
/*useDwarfDirectory*/ true, IP, CE, MAB, Opts.ShowInst));
|
||||
} else if (Opts.OutputType == AssemblerInvocation::FT_Null) {
|
||||
Str.reset(createNullStreamer(Ctx));
|
||||
} else {
|
||||
assert(Opts.OutputType == AssemblerInvocation::FT_Obj &&
|
||||
"Invalid file type!");
|
||||
if (!FDOS->supportsSeeking()) {
|
||||
BOS = make_unique<buffer_ostream>(*FDOS);
|
||||
Out = BOS.get();
|
||||
}
|
||||
|
||||
MCCodeEmitter *CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx);
|
||||
MCTargetOptions Options;
|
||||
MCAsmBackend *MAB = TheTarget->createMCAsmBackend(*STI, *MRI, Options);
|
||||
Triple T(Opts.Triple);
|
||||
Str.reset(TheTarget->createMCObjectStreamer(
|
||||
T, Ctx, std::unique_ptr<MCAsmBackend>(MAB), *Out, std::unique_ptr<MCCodeEmitter>(CE), *STI,
|
||||
Opts.RelaxAll, Opts.IncrementalLinkerCompatible,
|
||||
/*DWARFMustBeAtTheEnd*/ true));
|
||||
Str.get()->InitSections(Opts.NoExecStack);
|
||||
}
|
||||
|
||||
bool Failed = false;
|
||||
|
||||
std::unique_ptr<MCAsmParser> Parser(
|
||||
createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI));
|
||||
|
||||
// FIXME: init MCTargetOptions from sanitizer flags here.
|
||||
MCTargetOptions Options;
|
||||
std::unique_ptr<MCTargetAsmParser> TAP(
|
||||
TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options));
|
||||
if (!TAP)
|
||||
Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
|
||||
|
||||
// Set values for symbols, if any.
|
||||
for (auto &S : Opts.SymbolDefs) {
|
||||
auto Pair = StringRef(S).split('=');
|
||||
auto Sym = Pair.first;
|
||||
auto Val = Pair.second;
|
||||
int64_t Value = 0;
|
||||
// We have already error checked this in the driver.
|
||||
Val.getAsInteger(0, Value);
|
||||
Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value);
|
||||
}
|
||||
|
||||
if (!Failed) {
|
||||
Parser->setTargetParser(*TAP.get());
|
||||
Failed = Parser->Run(Opts.NoInitialTextSection);
|
||||
}
|
||||
|
||||
// Close Streamer first.
|
||||
// It might have a reference to the output stream.
|
||||
Str.reset();
|
||||
// Close the output stream early.
|
||||
BOS.reset();
|
||||
FDOS.reset();
|
||||
|
||||
// Delete output file if there were errors.
|
||||
if (Failed && Opts.OutputPath != "-")
|
||||
sys::fs::remove(Opts.OutputPath);
|
||||
|
||||
return Failed;
|
||||
}
|
||||
|
||||
static void LLVMErrorHandler(void *UserData, const std::string &Message,
|
||||
bool GenCrashDiag) {
|
||||
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
|
||||
|
||||
Diags.Report(diag::err_fe_error_backend) << Message;
|
||||
|
||||
// We cannot recover from llvm errors.
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
|
||||
// Initialize targets and assembly printers/parsers.
|
||||
InitializeAllTargetInfos();
|
||||
InitializeAllTargetMCs();
|
||||
InitializeAllAsmParsers();
|
||||
|
||||
// Construct our diagnostic client.
|
||||
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
|
||||
TextDiagnosticPrinter *DiagClient
|
||||
= new TextDiagnosticPrinter(errs(), &*DiagOpts);
|
||||
DiagClient->setPrefix("clang -cc1as");
|
||||
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
|
||||
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
|
||||
|
||||
// Set an error handler, so that any LLVM backend diagnostics go through our
|
||||
// error handler.
|
||||
ScopedFatalErrorHandler FatalErrorHandler
|
||||
(LLVMErrorHandler, static_cast<void*>(&Diags));
|
||||
|
||||
// Parse the arguments.
|
||||
AssemblerInvocation Asm;
|
||||
if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags))
|
||||
return 1;
|
||||
|
||||
if (Asm.ShowHelp) {
|
||||
std::unique_ptr<OptTable> Opts(driver::createDriverOptTable());
|
||||
Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler",
|
||||
/*Include=*/driver::options::CC1AsOption, /*Exclude=*/0,
|
||||
/*ShowAllAliases=*/false);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Honor -version.
|
||||
//
|
||||
// FIXME: Use a better -version message?
|
||||
if (Asm.ShowVersion) {
|
||||
llvm::cl::PrintVersionMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Honor -mllvm.
|
||||
//
|
||||
// FIXME: Remove this, one day.
|
||||
if (!Asm.LLVMArgs.empty()) {
|
||||
unsigned NumArgs = Asm.LLVMArgs.size();
|
||||
auto Args = llvm::make_unique<const char*[]>(NumArgs + 2);
|
||||
Args[0] = "clang (LLVM option parsing)";
|
||||
for (unsigned i = 0; i != NumArgs; ++i)
|
||||
Args[i + 1] = Asm.LLVMArgs[i].c_str();
|
||||
Args[NumArgs + 1] = nullptr;
|
||||
llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get());
|
||||
}
|
||||
|
||||
// Execute the invocation, unless there were parsing errors.
|
||||
bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags);
|
||||
|
||||
// If any timers were active but haven't been destroyed yet, print their
|
||||
// results now.
|
||||
TimerGroup::printAll(errs());
|
||||
|
||||
return !!Failed;
|
||||
}
|
||||
@ -1,520 +0,0 @@
|
||||
//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This is the entry point to the clang driver; it is a thin wrapper
|
||||
// for functionality in the Driver clang library.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "clang/Basic/DiagnosticOptions.h"
|
||||
#include "clang/Driver/Compilation.h"
|
||||
#include "clang/Driver/Driver.h"
|
||||
#include "clang/Driver/DriverDiagnostic.h"
|
||||
#include "clang/Driver/Options.h"
|
||||
#include "clang/Driver/ToolChain.h"
|
||||
#include "clang/Frontend/ChainedDiagnosticConsumer.h"
|
||||
#include "clang/Frontend/CompilerInvocation.h"
|
||||
#include "clang/Frontend/SerializedDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/TextDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/Utils.h"
|
||||
#include "llvm/ADT/ArrayRef.h"
|
||||
#include "llvm/ADT/SmallString.h"
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
#include "llvm/Config/llvm-config.h"
|
||||
#include "llvm/Option/ArgList.h"
|
||||
#include "llvm/Option/OptTable.h"
|
||||
#include "llvm/Option/Option.h"
|
||||
#include "llvm/Support/CommandLine.h"
|
||||
#include "llvm/Support/ErrorHandling.h"
|
||||
#include "llvm/Support/FileSystem.h"
|
||||
#include "llvm/Support/Host.h"
|
||||
#include "llvm/Support/ManagedStatic.h"
|
||||
#include "llvm/Support/Path.h"
|
||||
#include "llvm/Support/PrettyStackTrace.h"
|
||||
#include "llvm/Support/Process.h"
|
||||
#include "llvm/Support/Program.h"
|
||||
#include "llvm/Support/Regex.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
#include "llvm/Support/StringSaver.h"
|
||||
#include "llvm/Support/TargetSelect.h"
|
||||
#include "llvm/Support/Timer.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <system_error>
|
||||
using namespace clang;
|
||||
using namespace clang::driver;
|
||||
using namespace llvm::opt;
|
||||
|
||||
std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) {
|
||||
if (!CanonicalPrefixes) {
|
||||
SmallString<128> ExecutablePath(Argv0);
|
||||
// Do a PATH lookup if Argv0 isn't a valid path.
|
||||
if (!llvm::sys::fs::exists(ExecutablePath))
|
||||
if (llvm::ErrorOr<std::string> P =
|
||||
llvm::sys::findProgramByName(ExecutablePath))
|
||||
ExecutablePath = *P;
|
||||
return ExecutablePath.str();
|
||||
}
|
||||
|
||||
// This just needs to be some symbol in the binary; C++ doesn't
|
||||
// allow taking the address of ::main however.
|
||||
void *P = (void*) (intptr_t) GetExecutablePath;
|
||||
return llvm::sys::fs::getMainExecutable(Argv0, P);
|
||||
}
|
||||
|
||||
static const char *GetStableCStr(std::set<std::string> &SavedStrings,
|
||||
StringRef S) {
|
||||
return SavedStrings.insert(S).first->c_str();
|
||||
}
|
||||
|
||||
/// ApplyQAOverride - Apply a list of edits to the input argument lists.
|
||||
///
|
||||
/// The input string is a space separate list of edits to perform,
|
||||
/// they are applied in order to the input argument lists. Edits
|
||||
/// should be one of the following forms:
|
||||
///
|
||||
/// '#': Silence information about the changes to the command line arguments.
|
||||
///
|
||||
/// '^': Add FOO as a new argument at the beginning of the command line.
|
||||
///
|
||||
/// '+': Add FOO as a new argument at the end of the command line.
|
||||
///
|
||||
/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command
|
||||
/// line.
|
||||
///
|
||||
/// 'xOPTION': Removes all instances of the literal argument OPTION.
|
||||
///
|
||||
/// 'XOPTION': Removes all instances of the literal argument OPTION,
|
||||
/// and the following argument.
|
||||
///
|
||||
/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox'
|
||||
/// at the end of the command line.
|
||||
///
|
||||
/// \param OS - The stream to write edit information to.
|
||||
/// \param Args - The vector of command line arguments.
|
||||
/// \param Edit - The override command to perform.
|
||||
/// \param SavedStrings - Set to use for storing string representations.
|
||||
static void ApplyOneQAOverride(raw_ostream &OS,
|
||||
SmallVectorImpl<const char*> &Args,
|
||||
StringRef Edit,
|
||||
std::set<std::string> &SavedStrings) {
|
||||
// This does not need to be efficient.
|
||||
|
||||
if (Edit[0] == '^') {
|
||||
const char *Str =
|
||||
GetStableCStr(SavedStrings, Edit.substr(1));
|
||||
OS << "### Adding argument " << Str << " at beginning\n";
|
||||
Args.insert(Args.begin() + 1, Str);
|
||||
} else if (Edit[0] == '+') {
|
||||
const char *Str =
|
||||
GetStableCStr(SavedStrings, Edit.substr(1));
|
||||
OS << "### Adding argument " << Str << " at end\n";
|
||||
Args.push_back(Str);
|
||||
} else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") &&
|
||||
Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) {
|
||||
StringRef MatchPattern = Edit.substr(2).split('/').first;
|
||||
StringRef ReplPattern = Edit.substr(2).split('/').second;
|
||||
ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1);
|
||||
|
||||
for (unsigned i = 1, e = Args.size(); i != e; ++i) {
|
||||
// Ignore end-of-line response file markers
|
||||
if (Args[i] == nullptr)
|
||||
continue;
|
||||
std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]);
|
||||
|
||||
if (Repl != Args[i]) {
|
||||
OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n";
|
||||
Args[i] = GetStableCStr(SavedStrings, Repl);
|
||||
}
|
||||
}
|
||||
} else if (Edit[0] == 'x' || Edit[0] == 'X') {
|
||||
auto Option = Edit.substr(1);
|
||||
for (unsigned i = 1; i < Args.size();) {
|
||||
if (Option == Args[i]) {
|
||||
OS << "### Deleting argument " << Args[i] << '\n';
|
||||
Args.erase(Args.begin() + i);
|
||||
if (Edit[0] == 'X') {
|
||||
if (i < Args.size()) {
|
||||
OS << "### Deleting argument " << Args[i] << '\n';
|
||||
Args.erase(Args.begin() + i);
|
||||
} else
|
||||
OS << "### Invalid X edit, end of command line!\n";
|
||||
}
|
||||
} else
|
||||
++i;
|
||||
}
|
||||
} else if (Edit[0] == 'O') {
|
||||
for (unsigned i = 1; i < Args.size();) {
|
||||
const char *A = Args[i];
|
||||
// Ignore end-of-line response file markers
|
||||
if (A == nullptr)
|
||||
continue;
|
||||
if (A[0] == '-' && A[1] == 'O' &&
|
||||
(A[2] == '\0' ||
|
||||
(A[3] == '\0' && (A[2] == 's' || A[2] == 'z' ||
|
||||
('0' <= A[2] && A[2] <= '9'))))) {
|
||||
OS << "### Deleting argument " << Args[i] << '\n';
|
||||
Args.erase(Args.begin() + i);
|
||||
} else
|
||||
++i;
|
||||
}
|
||||
OS << "### Adding argument " << Edit << " at end\n";
|
||||
Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str()));
|
||||
} else {
|
||||
OS << "### Unrecognized edit: " << Edit << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
/// ApplyQAOverride - Apply a comma separate list of edits to the
|
||||
/// input argument lists. See ApplyOneQAOverride.
|
||||
static void ApplyQAOverride(SmallVectorImpl<const char*> &Args,
|
||||
const char *OverrideStr,
|
||||
std::set<std::string> &SavedStrings) {
|
||||
raw_ostream *OS = &llvm::errs();
|
||||
|
||||
if (OverrideStr[0] == '#') {
|
||||
++OverrideStr;
|
||||
OS = &llvm::nulls();
|
||||
}
|
||||
|
||||
*OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n";
|
||||
|
||||
// This does not need to be efficient.
|
||||
|
||||
const char *S = OverrideStr;
|
||||
while (*S) {
|
||||
const char *End = ::strchr(S, ' ');
|
||||
if (!End)
|
||||
End = S + strlen(S);
|
||||
if (End != S)
|
||||
ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings);
|
||||
S = End;
|
||||
if (*S != '\0')
|
||||
++S;
|
||||
}
|
||||
}
|
||||
|
||||
extern int cc1_main(ArrayRef<const char *> Argv, const char *Argv0,
|
||||
void *MainAddr);
|
||||
extern int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0,
|
||||
void *MainAddr);
|
||||
|
||||
static void insertTargetAndModeArgs(const ParsedClangName &NameParts,
|
||||
SmallVectorImpl<const char *> &ArgVector,
|
||||
std::set<std::string> &SavedStrings) {
|
||||
// Put target and mode arguments at the start of argument list so that
|
||||
// arguments specified in command line could override them. Avoid putting
|
||||
// them at index 0, as an option like '-cc1' must remain the first.
|
||||
auto InsertionPoint = ArgVector.begin();
|
||||
if (InsertionPoint != ArgVector.end())
|
||||
++InsertionPoint;
|
||||
|
||||
if (NameParts.DriverMode) {
|
||||
// Add the mode flag to the arguments.
|
||||
ArgVector.insert(InsertionPoint,
|
||||
GetStableCStr(SavedStrings, NameParts.DriverMode));
|
||||
}
|
||||
|
||||
if (NameParts.TargetIsValid) {
|
||||
const char *arr[] = {"-target", GetStableCStr(SavedStrings,
|
||||
NameParts.TargetPrefix)};
|
||||
ArgVector.insert(InsertionPoint, std::begin(arr), std::end(arr));
|
||||
}
|
||||
}
|
||||
|
||||
static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver,
|
||||
SmallVectorImpl<const char *> &Opts) {
|
||||
llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts);
|
||||
// The first instance of '#' should be replaced with '=' in each option.
|
||||
for (const char *Opt : Opts)
|
||||
if (char *NumberSignPtr = const_cast<char *>(::strchr(Opt, '#')))
|
||||
*NumberSignPtr = '=';
|
||||
}
|
||||
|
||||
static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) {
|
||||
// Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE.
|
||||
TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS");
|
||||
if (TheDriver.CCPrintOptions)
|
||||
TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE");
|
||||
|
||||
// Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE.
|
||||
TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS");
|
||||
if (TheDriver.CCPrintHeaders)
|
||||
TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE");
|
||||
|
||||
// Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE.
|
||||
TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS");
|
||||
if (TheDriver.CCLogDiagnostics)
|
||||
TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE");
|
||||
}
|
||||
|
||||
static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient,
|
||||
const std::string &Path) {
|
||||
// If the clang binary happens to be named cl.exe for compatibility reasons,
|
||||
// use clang-cl.exe as the prefix to avoid confusion between clang and MSVC.
|
||||
StringRef ExeBasename(llvm::sys::path::filename(Path));
|
||||
if (ExeBasename.equals_lower("cl.exe"))
|
||||
ExeBasename = "clang-cl.exe";
|
||||
DiagClient->setPrefix(ExeBasename);
|
||||
}
|
||||
|
||||
// This lets us create the DiagnosticsEngine with a properly-filled-out
|
||||
// DiagnosticOptions instance.
|
||||
static DiagnosticOptions *
|
||||
CreateAndPopulateDiagOpts(ArrayRef<const char *> argv) {
|
||||
auto *DiagOpts = new DiagnosticOptions;
|
||||
std::unique_ptr<OptTable> Opts(createDriverOptTable());
|
||||
unsigned MissingArgIndex, MissingArgCount;
|
||||
InputArgList Args =
|
||||
Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount);
|
||||
// We ignore MissingArgCount and the return value of ParseDiagnosticArgs.
|
||||
// Any errors that would be diagnosed here will also be diagnosed later,
|
||||
// when the DiagnosticsEngine actually exists.
|
||||
(void)ParseDiagnosticArgs(*DiagOpts, Args);
|
||||
return DiagOpts;
|
||||
}
|
||||
|
||||
static void SetInstallDir(SmallVectorImpl<const char *> &argv,
|
||||
Driver &TheDriver, bool CanonicalPrefixes) {
|
||||
// Attempt to find the original path used to invoke the driver, to determine
|
||||
// the installed path. We do this manually, because we want to support that
|
||||
// path being a symlink.
|
||||
SmallString<128> InstalledPath(argv[0]);
|
||||
|
||||
// Do a PATH lookup, if there are no directory components.
|
||||
if (llvm::sys::path::filename(InstalledPath) == InstalledPath)
|
||||
if (llvm::ErrorOr<std::string> Tmp = llvm::sys::findProgramByName(
|
||||
llvm::sys::path::filename(InstalledPath.str())))
|
||||
InstalledPath = *Tmp;
|
||||
|
||||
// FIXME: We don't actually canonicalize this, we just make it absolute.
|
||||
if (CanonicalPrefixes)
|
||||
llvm::sys::fs::make_absolute(InstalledPath);
|
||||
|
||||
StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath));
|
||||
if (llvm::sys::fs::exists(InstalledPathParent))
|
||||
TheDriver.setInstalledDir(InstalledPathParent);
|
||||
}
|
||||
|
||||
static int ExecuteCC1Tool(ArrayRef<const char *> argv, StringRef Tool) {
|
||||
void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath;
|
||||
if (Tool == "")
|
||||
return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP);
|
||||
if (Tool == "as")
|
||||
return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP);
|
||||
|
||||
// Reject unknown tools.
|
||||
llvm::errs() << "error: unknown integrated tool '" << Tool << "'\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
int mainEntryClickHouseClang(int argc_, char **argv_) {
|
||||
llvm::sys::PrintStackTraceOnErrorSignal(argv_[0]);
|
||||
llvm::PrettyStackTraceProgram X(argc_, argv_);
|
||||
llvm::llvm_shutdown_obj Y; // Call llvm_shutdown() on exit.
|
||||
|
||||
if (llvm::sys::Process::FixupStandardFileDescriptors())
|
||||
return 1;
|
||||
|
||||
SmallVector<const char *, 256> argv;
|
||||
llvm::SpecificBumpPtrAllocator<char> ArgAllocator;
|
||||
std::error_code EC = llvm::sys::Process::GetArgumentVector(
|
||||
argv, llvm::makeArrayRef(argv_, argc_), ArgAllocator);
|
||||
if (EC) {
|
||||
llvm::errs() << "error: couldn't get arguments: " << EC.message() << '\n';
|
||||
return 1;
|
||||
}
|
||||
|
||||
llvm::InitializeAllTargets();
|
||||
auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]);
|
||||
|
||||
llvm::BumpPtrAllocator A;
|
||||
llvm::StringSaver Saver(A);
|
||||
|
||||
// Parse response files using the GNU syntax, unless we're in CL mode. There
|
||||
// are two ways to put clang in CL compatibility mode: argv[0] is either
|
||||
// clang-cl or cl, or --driver-mode=cl is on the command line. The normal
|
||||
// command line parsing can't happen until after response file parsing, so we
|
||||
// have to manually search for a --driver-mode=cl argument the hard way.
|
||||
// Finally, our -cc1 tools don't care which tokenization mode we use because
|
||||
// response files written by clang will tokenize the same way in either mode.
|
||||
bool ClangCLMode = false;
|
||||
if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") ||
|
||||
std::find_if(argv.begin(), argv.end(), [](const char *F) {
|
||||
return F && strcmp(F, "--driver-mode=cl") == 0;
|
||||
}) != argv.end()) {
|
||||
ClangCLMode = true;
|
||||
}
|
||||
enum { Default, POSIX, Windows } RSPQuoting = Default;
|
||||
for (const char *F : argv) {
|
||||
if (strcmp(F, "--rsp-quoting=posix") == 0)
|
||||
RSPQuoting = POSIX;
|
||||
else if (strcmp(F, "--rsp-quoting=windows") == 0)
|
||||
RSPQuoting = Windows;
|
||||
}
|
||||
|
||||
// Determines whether we want nullptr markers in argv to indicate response
|
||||
// files end-of-lines. We only use this for the /LINK driver argument with
|
||||
// clang-cl.exe on Windows.
|
||||
bool MarkEOLs = ClangCLMode;
|
||||
|
||||
llvm::cl::TokenizerCallback Tokenizer;
|
||||
if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode))
|
||||
Tokenizer = &llvm::cl::TokenizeWindowsCommandLine;
|
||||
else
|
||||
Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
|
||||
|
||||
if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1"))
|
||||
MarkEOLs = false;
|
||||
llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs);
|
||||
|
||||
// Handle -cc1 integrated tools, even if -cc1 was expanded from a response
|
||||
// file.
|
||||
auto FirstArg = std::find_if(argv.begin() + 1, argv.end(),
|
||||
[](const char *A) { return A != nullptr; });
|
||||
if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) {
|
||||
// If -cc1 came from a response file, remove the EOL sentinels.
|
||||
if (MarkEOLs) {
|
||||
auto newEnd = std::remove(argv.begin(), argv.end(), nullptr);
|
||||
argv.resize(newEnd - argv.begin());
|
||||
}
|
||||
return ExecuteCC1Tool(argv, argv[1] + 4);
|
||||
}
|
||||
|
||||
bool CanonicalPrefixes = true;
|
||||
for (int i = 1, size = argv.size(); i < size; ++i) {
|
||||
// Skip end-of-line response file markers
|
||||
if (argv[i] == nullptr)
|
||||
continue;
|
||||
if (StringRef(argv[i]) == "-no-canonical-prefixes") {
|
||||
CanonicalPrefixes = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle CL and _CL_ which permits additional command line options to be
|
||||
// prepended or appended.
|
||||
if (ClangCLMode) {
|
||||
// Arguments in "CL" are prepended.
|
||||
llvm::Optional<std::string> OptCL = llvm::sys::Process::GetEnv("CL");
|
||||
if (OptCL.hasValue()) {
|
||||
SmallVector<const char *, 8> PrependedOpts;
|
||||
getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts);
|
||||
|
||||
// Insert right after the program name to prepend to the argument list.
|
||||
argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end());
|
||||
}
|
||||
// Arguments in "_CL_" are appended.
|
||||
llvm::Optional<std::string> Opt_CL_ = llvm::sys::Process::GetEnv("_CL_");
|
||||
if (Opt_CL_.hasValue()) {
|
||||
SmallVector<const char *, 8> AppendedOpts;
|
||||
getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts);
|
||||
|
||||
// Insert at the end of the argument list to append.
|
||||
argv.append(AppendedOpts.begin(), AppendedOpts.end());
|
||||
}
|
||||
}
|
||||
|
||||
std::set<std::string> SavedStrings;
|
||||
// Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the
|
||||
// scenes.
|
||||
if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) {
|
||||
// FIXME: Driver shouldn't take extra initial argument.
|
||||
ApplyQAOverride(argv, OverrideStr, SavedStrings);
|
||||
}
|
||||
|
||||
std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes);
|
||||
|
||||
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts =
|
||||
CreateAndPopulateDiagOpts(argv);
|
||||
|
||||
TextDiagnosticPrinter *DiagClient
|
||||
= new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
|
||||
FixupDiagPrefixExeName(DiagClient, Path);
|
||||
|
||||
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
|
||||
|
||||
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
|
||||
|
||||
if (!DiagOpts->DiagnosticSerializationFile.empty()) {
|
||||
auto SerializedConsumer =
|
||||
clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile,
|
||||
&*DiagOpts, /*MergeChildRecords=*/true);
|
||||
Diags.setClient(new ChainedDiagnosticConsumer(
|
||||
Diags.takeClient(), std::move(SerializedConsumer)));
|
||||
}
|
||||
|
||||
ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false);
|
||||
|
||||
Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags);
|
||||
SetInstallDir(argv, TheDriver, CanonicalPrefixes);
|
||||
TheDriver.setTargetAndMode(TargetAndMode);
|
||||
|
||||
insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings);
|
||||
|
||||
SetBackdoorDriverOutputsFromEnvVars(TheDriver);
|
||||
|
||||
std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(argv));
|
||||
int Res = 1;
|
||||
if (C && !C->containsError()) {
|
||||
SmallVector<std::pair<int, const Command *>, 4> FailingCommands;
|
||||
Res = TheDriver.ExecuteCompilation(*C, FailingCommands);
|
||||
|
||||
// Force a crash to test the diagnostics.
|
||||
if (TheDriver.GenReproducer) {
|
||||
Diags.Report(diag::err_drv_force_crash)
|
||||
<< !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH");
|
||||
|
||||
// Pretend that every command failed.
|
||||
FailingCommands.clear();
|
||||
for (const auto &J : C->getJobs())
|
||||
if (const Command *C = dyn_cast<Command>(&J))
|
||||
FailingCommands.push_back(std::make_pair(-1, C));
|
||||
}
|
||||
|
||||
for (const auto &P : FailingCommands) {
|
||||
int CommandRes = P.first;
|
||||
const Command *FailingCommand = P.second;
|
||||
if (!Res)
|
||||
Res = CommandRes;
|
||||
|
||||
// If result status is < 0, then the driver command signalled an error.
|
||||
// If result status is 70, then the driver command reported a fatal error.
|
||||
// On Windows, abort will return an exit code of 3. In these cases,
|
||||
// generate additional diagnostic information if possible.
|
||||
bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70;
|
||||
#ifdef LLVM_ON_WIN32
|
||||
DiagnoseCrash |= CommandRes == 3;
|
||||
#endif
|
||||
if (DiagnoseCrash) {
|
||||
TheDriver.generateCompilationDiagnostics(*C, *FailingCommand);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Diags.getClient()->finish();
|
||||
|
||||
// If any timers were active but haven't been destroyed yet, print their
|
||||
// results now. This happens in -disable-free mode.
|
||||
llvm::TimerGroup::printAll(llvm::errs());
|
||||
|
||||
#ifdef LLVM_ON_WIN32
|
||||
// Exit status should not be negative on Win32, unless abnormal termination.
|
||||
// Once abnormal termiation was caught, negative status should not be
|
||||
// propagated.
|
||||
if (Res < 0)
|
||||
Res = 1;
|
||||
#endif
|
||||
|
||||
// If we have multiple failing commands, we return the result of the first
|
||||
// failing command.
|
||||
return Res;
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
#include "lld/Common/Driver.h"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/ADT/StringSwitch.h"
|
||||
#include "llvm/ADT/Twine.h"
|
||||
#include "llvm/Support/ManagedStatic.h"
|
||||
#include "llvm/Support/Path.h"
|
||||
#include "llvm/Support/PrettyStackTrace.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
|
||||
using namespace lld;
|
||||
using namespace llvm;
|
||||
using namespace llvm::sys;
|
||||
|
||||
int mainEntryClickHouseLLD(int Argc, char **Argv)
|
||||
{
|
||||
// Standard set up, so program fails gracefully.
|
||||
sys::PrintStackTraceOnErrorSignal(Argv[0]);
|
||||
PrettyStackTraceProgram StackPrinter(Argc, Argv);
|
||||
llvm_shutdown_obj Shutdown;
|
||||
|
||||
std::vector<const char *> Args(Argv, Argv + Argc);
|
||||
return !elf::link(Args, true);
|
||||
}
|
||||
@ -1 +0,0 @@
|
||||
Compiler-6.0.0
|
||||
@ -1 +0,0 @@
|
||||
Compiler-6.0.0
|
||||
@ -1,49 +0,0 @@
|
||||
add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG)
|
||||
|
||||
link_directories(${LLVM_LIBRARY_DIRS})
|
||||
|
||||
add_library(clickhouse-compiler-lib
|
||||
driver.cpp
|
||||
cc1_main.cpp
|
||||
cc1gen_reproducer_main.cpp
|
||||
cc1as_main.cpp
|
||||
lld.cpp)
|
||||
|
||||
target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0)
|
||||
|
||||
string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti
|
||||
|
||||
llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
|
||||
|
||||
message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
|
||||
|
||||
target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
|
||||
|
||||
# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.
|
||||
|
||||
target_link_libraries(clickhouse-compiler-lib PRIVATE
|
||||
clangBasic clangCodeGen clangDriver
|
||||
clangFrontend
|
||||
clangFrontendTool
|
||||
clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend
|
||||
clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers
|
||||
clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic
|
||||
clangCrossTU clangIndex
|
||||
|
||||
lldCOFF
|
||||
lldDriver
|
||||
lldELF
|
||||
lldMinGW
|
||||
lldMachO
|
||||
lldReaderWriter
|
||||
lldYAML
|
||||
lldCommon
|
||||
lldCore
|
||||
|
||||
${REQUIRED_LLVM_LIBRARIES}
|
||||
|
||||
PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARIES} Threads::Threads
|
||||
${MALLOC_LIBRARIES}
|
||||
${GLIBC_COMPATIBILITY_LIBRARIES}
|
||||
${MEMCPY_LIBRARIES}
|
||||
)
|
||||
@ -1,239 +0,0 @@
|
||||
//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This is the entry point to the clang -cc1 functionality, which implements the
|
||||
// core compiler functionality along with a number of additional tools for
|
||||
// demonstration and testing purposes.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "llvm/Option/Arg.h"
|
||||
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
|
||||
#include "clang/Config/config.h"
|
||||
#include "clang/Basic/Stack.h"
|
||||
#include "clang/Driver/DriverDiagnostic.h"
|
||||
#include "clang/Driver/Options.h"
|
||||
#include "clang/Frontend/CompilerInstance.h"
|
||||
#include "clang/Frontend/CompilerInvocation.h"
|
||||
#include "clang/Frontend/FrontendDiagnostic.h"
|
||||
#include "clang/Frontend/TextDiagnosticBuffer.h"
|
||||
#include "clang/Frontend/TextDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/Utils.h"
|
||||
#include "clang/FrontendTool/Utils.h"
|
||||
#include "llvm/ADT/Statistic.h"
|
||||
#include "llvm/Config/llvm-config.h"
|
||||
#include "llvm/LinkAllPasses.h"
|
||||
#include "llvm/Option/ArgList.h"
|
||||
#include "llvm/Option/OptTable.h"
|
||||
#include "llvm/Support/Compiler.h"
|
||||
#include "llvm/Support/ErrorHandling.h"
|
||||
#include "llvm/Support/ManagedStatic.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
#include "llvm/Support/TargetSelect.h"
|
||||
#include "llvm/Support/Timer.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
#include <cstdio>
|
||||
|
||||
#ifdef CLANG_HAVE_RLIMITS
|
||||
#include <sys/resource.h>
|
||||
#endif
|
||||
|
||||
// have no .a version in packages
|
||||
#undef LINK_POLLY_INTO_TOOLS
|
||||
|
||||
using namespace clang;
|
||||
using namespace llvm::opt;
|
||||
|
||||
//===----------------------------------------------------------------------===//
|
||||
// Main driver
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
static void LLVMErrorHandler(void *UserData, const std::string &Message,
|
||||
bool GenCrashDiag) {
|
||||
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
|
||||
|
||||
Diags.Report(diag::err_fe_error_backend) << Message;
|
||||
|
||||
// Run the interrupt handlers to make sure any special cleanups get done, in
|
||||
// particular that we remove files registered with RemoveFileOnSignal.
|
||||
llvm::sys::RunInterruptHandlers();
|
||||
|
||||
// We cannot recover from llvm errors. When reporting a fatal error, exit
|
||||
// with status 70 to generate crash diagnostics. For BSD systems this is
|
||||
// defined as an internal software error. Otherwise, exit with status 1.
|
||||
exit(GenCrashDiag ? 70 : 1);
|
||||
}
|
||||
|
||||
#ifdef LINK_POLLY_INTO_TOOLS
|
||||
namespace polly {
|
||||
void initializePollyPasses(llvm::PassRegistry &Registry);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef CLANG_HAVE_RLIMITS
|
||||
#if defined(__linux__) && defined(__PIE__)
|
||||
static size_t getCurrentStackAllocation() {
|
||||
// If we can't compute the current stack usage, allow for 512K of command
|
||||
// line arguments and environment.
|
||||
size_t Usage = 512 * 1024;
|
||||
if (FILE *StatFile = fopen("/proc/self/stat", "r")) {
|
||||
// We assume that the stack extends from its current address to the end of
|
||||
// the environment space. In reality, there is another string literal (the
|
||||
// program name) after the environment, but this is close enough (we only
|
||||
// need to be within 100K or so).
|
||||
unsigned long StackPtr, EnvEnd;
|
||||
// Disable silly GCC -Wformat warning that complains about length
|
||||
// modifiers on ignored format specifiers. We want to retain these
|
||||
// for documentation purposes even though they have no effect.
|
||||
#if defined(__GNUC__) && !defined(__clang__)
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wformat"
|
||||
#endif
|
||||
if (fscanf(StatFile,
|
||||
"%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu "
|
||||
"%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu "
|
||||
"%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d "
|
||||
"%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d",
|
||||
&StackPtr, &EnvEnd) == 2) {
|
||||
#if defined(__GNUC__) && !defined(__clang__)
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd;
|
||||
}
|
||||
fclose(StatFile);
|
||||
}
|
||||
return Usage;
|
||||
}
|
||||
|
||||
#include <alloca.h>
|
||||
|
||||
LLVM_ATTRIBUTE_NOINLINE
|
||||
static void ensureStackAddressSpace() {
|
||||
// Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary
|
||||
// relatively close to the stack (they are only guaranteed to be 128MiB
|
||||
// apart). This results in crashes if we happen to heap-allocate more than
|
||||
// 128MiB before we reach our stack high-water mark.
|
||||
//
|
||||
// To avoid these crashes, ensure that we have sufficient virtual memory
|
||||
// pages allocated before we start running.
|
||||
size_t Curr = getCurrentStackAllocation();
|
||||
const int kTargetStack = DesiredStackSize - 256 * 1024;
|
||||
if (Curr < kTargetStack) {
|
||||
volatile char *volatile Alloc =
|
||||
static_cast<volatile char *>(alloca(kTargetStack - Curr));
|
||||
Alloc[0] = 0;
|
||||
Alloc[kTargetStack - Curr - 1] = 0;
|
||||
}
|
||||
}
|
||||
#else
|
||||
static void ensureStackAddressSpace() {}
|
||||
#endif
|
||||
|
||||
/// Attempt to ensure that we have at least 8MiB of usable stack space.
|
||||
static void ensureSufficientStack() {
|
||||
struct rlimit rlim;
|
||||
if (getrlimit(RLIMIT_STACK, &rlim) != 0)
|
||||
return;
|
||||
|
||||
// Increase the soft stack limit to our desired level, if necessary and
|
||||
// possible.
|
||||
if (rlim.rlim_cur != RLIM_INFINITY &&
|
||||
rlim.rlim_cur < rlim_t(DesiredStackSize)) {
|
||||
// Try to allocate sufficient stack.
|
||||
if (rlim.rlim_max == RLIM_INFINITY ||
|
||||
rlim.rlim_max >= rlim_t(DesiredStackSize))
|
||||
rlim.rlim_cur = DesiredStackSize;
|
||||
else if (rlim.rlim_cur == rlim.rlim_max)
|
||||
return;
|
||||
else
|
||||
rlim.rlim_cur = rlim.rlim_max;
|
||||
|
||||
if (setrlimit(RLIMIT_STACK, &rlim) != 0 ||
|
||||
rlim.rlim_cur != DesiredStackSize)
|
||||
return;
|
||||
}
|
||||
|
||||
// We should now have a stack of size at least DesiredStackSize. Ensure
|
||||
// that we can actually use that much, if necessary.
|
||||
ensureStackAddressSpace();
|
||||
}
|
||||
#else
|
||||
static void ensureSufficientStack() {}
|
||||
#endif
|
||||
|
||||
int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
|
||||
ensureSufficientStack();
|
||||
|
||||
std::unique_ptr<CompilerInstance> Clang(new CompilerInstance());
|
||||
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
|
||||
|
||||
// Register the support for object-file-wrapped Clang modules.
|
||||
auto PCHOps = Clang->getPCHContainerOperations();
|
||||
PCHOps->registerWriter(llvm::make_unique<ObjectFilePCHContainerWriter>());
|
||||
PCHOps->registerReader(llvm::make_unique<ObjectFilePCHContainerReader>());
|
||||
|
||||
// Initialize targets first, so that --version shows registered targets.
|
||||
llvm::InitializeAllTargets();
|
||||
llvm::InitializeAllTargetMCs();
|
||||
llvm::InitializeAllAsmPrinters();
|
||||
llvm::InitializeAllAsmParsers();
|
||||
|
||||
#ifdef LINK_POLLY_INTO_TOOLS
|
||||
llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry();
|
||||
polly::initializePollyPasses(Registry);
|
||||
#endif
|
||||
|
||||
// Buffer diagnostics from argument parsing so that we can output them using a
|
||||
// well formed diagnostic object.
|
||||
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
|
||||
TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer;
|
||||
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer);
|
||||
bool Success = CompilerInvocation::CreateFromArgs(
|
||||
Clang->getInvocation(), Argv.begin(), Argv.end(), Diags);
|
||||
|
||||
// Infer the builtin include path if unspecified.
|
||||
if (Clang->getHeaderSearchOpts().UseBuiltinIncludes &&
|
||||
Clang->getHeaderSearchOpts().ResourceDir.empty())
|
||||
Clang->getHeaderSearchOpts().ResourceDir =
|
||||
CompilerInvocation::GetResourcesPath(Argv0, MainAddr);
|
||||
|
||||
// Create the actual diagnostics engine.
|
||||
Clang->createDiagnostics();
|
||||
if (!Clang->hasDiagnostics())
|
||||
return 1;
|
||||
|
||||
// Set an error handler, so that any LLVM backend diagnostics go through our
|
||||
// error handler.
|
||||
llvm::install_fatal_error_handler(LLVMErrorHandler,
|
||||
static_cast<void*>(&Clang->getDiagnostics()));
|
||||
|
||||
DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics());
|
||||
if (!Success)
|
||||
return 1;
|
||||
|
||||
// Execute the frontend actions.
|
||||
Success = ExecuteCompilerInvocation(Clang.get());
|
||||
|
||||
// If any timers were active but haven't been destroyed yet, print their
|
||||
// results now. This happens in -disable-free mode.
|
||||
llvm::TimerGroup::printAll(llvm::errs());
|
||||
|
||||
// Our error handler depends on the Diagnostics object, which we're
|
||||
// potentially about to delete. Uninstall the handler now so that any
|
||||
// later errors use the default handling behavior instead.
|
||||
llvm::remove_fatal_error_handler();
|
||||
|
||||
// When running with -disable-free, don't do any destruction or shutdown.
|
||||
if (Clang->getFrontendOpts().DisableFree) {
|
||||
BuryPointer(std::move(Clang));
|
||||
return !Success;
|
||||
}
|
||||
|
||||
return !Success;
|
||||
}
|
||||
@ -1,572 +0,0 @@
|
||||
//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This is the entry point to the clang -cc1as functionality, which implements
|
||||
// the direct interface to the LLVM MC based assembler.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "clang/Basic/Diagnostic.h"
|
||||
#include "clang/Basic/DiagnosticOptions.h"
|
||||
#include "clang/Driver/DriverDiagnostic.h"
|
||||
#include "clang/Driver/Options.h"
|
||||
#include "clang/Frontend/FrontendDiagnostic.h"
|
||||
#include "clang/Frontend/TextDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/Utils.h"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/ADT/StringSwitch.h"
|
||||
#include "llvm/ADT/Triple.h"
|
||||
#include "llvm/IR/DataLayout.h"
|
||||
#include "llvm/MC/MCAsmBackend.h"
|
||||
#include "llvm/MC/MCAsmInfo.h"
|
||||
#include "llvm/MC/MCCodeEmitter.h"
|
||||
#include "llvm/MC/MCContext.h"
|
||||
#include "llvm/MC/MCInstrInfo.h"
|
||||
#include "llvm/MC/MCObjectFileInfo.h"
|
||||
#include "llvm/MC/MCObjectWriter.h"
|
||||
#include "llvm/MC/MCParser/MCAsmParser.h"
|
||||
#include "llvm/MC/MCParser/MCTargetAsmParser.h"
|
||||
#include "llvm/MC/MCRegisterInfo.h"
|
||||
#include "llvm/MC/MCStreamer.h"
|
||||
#include "llvm/MC/MCSubtargetInfo.h"
|
||||
#include "llvm/MC/MCTargetOptions.h"
|
||||
#include "llvm/Option/Arg.h"
|
||||
#include "llvm/Option/ArgList.h"
|
||||
#include "llvm/Option/OptTable.h"
|
||||
#include "llvm/Support/CommandLine.h"
|
||||
#include "llvm/Support/ErrorHandling.h"
|
||||
#include "llvm/Support/FileSystem.h"
|
||||
#include "llvm/Support/FormattedStream.h"
|
||||
#include "llvm/Support/Host.h"
|
||||
#include "llvm/Support/MemoryBuffer.h"
|
||||
#include "llvm/Support/Path.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
#include "llvm/Support/SourceMgr.h"
|
||||
#include "llvm/Support/TargetRegistry.h"
|
||||
#include "llvm/Support/TargetSelect.h"
|
||||
#include "llvm/Support/Timer.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
#include <memory>
|
||||
#include <system_error>
|
||||
using namespace clang;
|
||||
using namespace clang::driver;
|
||||
using namespace clang::driver::options;
|
||||
using namespace llvm;
|
||||
using namespace llvm::opt;
|
||||
|
||||
namespace {
|
||||
|
||||
/// Helper class for representing a single invocation of the assembler.
|
||||
struct AssemblerInvocation {
|
||||
/// @name Target Options
|
||||
/// @{
|
||||
|
||||
/// The name of the target triple to assemble for.
|
||||
std::string Triple;
|
||||
|
||||
/// If given, the name of the target CPU to determine which instructions
|
||||
/// are legal.
|
||||
std::string CPU;
|
||||
|
||||
/// The list of target specific features to enable or disable -- this should
|
||||
/// be a list of strings starting with '+' or '-'.
|
||||
std::vector<std::string> Features;
|
||||
|
||||
/// The list of symbol definitions.
|
||||
std::vector<std::string> SymbolDefs;
|
||||
|
||||
/// @}
|
||||
/// @name Language Options
|
||||
/// @{
|
||||
|
||||
std::vector<std::string> IncludePaths;
|
||||
unsigned NoInitialTextSection : 1;
|
||||
unsigned SaveTemporaryLabels : 1;
|
||||
unsigned GenDwarfForAssembly : 1;
|
||||
unsigned RelaxELFRelocations : 1;
|
||||
unsigned DwarfVersion;
|
||||
std::string DwarfDebugFlags;
|
||||
std::string DwarfDebugProducer;
|
||||
std::string DebugCompilationDir;
|
||||
std::map<const std::string, const std::string> DebugPrefixMap;
|
||||
llvm::DebugCompressionType CompressDebugSections =
|
||||
llvm::DebugCompressionType::None;
|
||||
std::string MainFileName;
|
||||
std::string SplitDwarfFile;
|
||||
|
||||
/// @}
|
||||
/// @name Frontend Options
|
||||
/// @{
|
||||
|
||||
std::string InputFile;
|
||||
std::vector<std::string> LLVMArgs;
|
||||
std::string OutputPath;
|
||||
enum FileType {
|
||||
FT_Asm, ///< Assembly (.s) output, transliterate mode.
|
||||
FT_Null, ///< No output, for timing purposes.
|
||||
FT_Obj ///< Object file output.
|
||||
};
|
||||
FileType OutputType;
|
||||
unsigned ShowHelp : 1;
|
||||
unsigned ShowVersion : 1;
|
||||
|
||||
/// @}
|
||||
/// @name Transliterate Options
|
||||
/// @{
|
||||
|
||||
unsigned OutputAsmVariant;
|
||||
unsigned ShowEncoding : 1;
|
||||
unsigned ShowInst : 1;
|
||||
|
||||
/// @}
|
||||
/// @name Assembler Options
|
||||
/// @{
|
||||
|
||||
unsigned RelaxAll : 1;
|
||||
unsigned NoExecStack : 1;
|
||||
unsigned FatalWarnings : 1;
|
||||
unsigned IncrementalLinkerCompatible : 1;
|
||||
|
||||
/// The name of the relocation model to use.
|
||||
std::string RelocationModel;
|
||||
|
||||
/// @}
|
||||
|
||||
public:
|
||||
AssemblerInvocation() {
|
||||
Triple = "";
|
||||
NoInitialTextSection = 0;
|
||||
InputFile = "-";
|
||||
OutputPath = "-";
|
||||
OutputType = FT_Asm;
|
||||
OutputAsmVariant = 0;
|
||||
ShowInst = 0;
|
||||
ShowEncoding = 0;
|
||||
RelaxAll = 0;
|
||||
NoExecStack = 0;
|
||||
FatalWarnings = 0;
|
||||
IncrementalLinkerCompatible = 0;
|
||||
DwarfVersion = 0;
|
||||
}
|
||||
|
||||
static bool CreateFromArgs(AssemblerInvocation &Res,
|
||||
ArrayRef<const char *> Argv,
|
||||
DiagnosticsEngine &Diags);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
|
||||
ArrayRef<const char *> Argv,
|
||||
DiagnosticsEngine &Diags) {
|
||||
bool Success = true;
|
||||
|
||||
// Parse the arguments.
|
||||
std::unique_ptr<OptTable> OptTbl(createDriverOptTable());
|
||||
|
||||
const unsigned IncludedFlagsBitmask = options::CC1AsOption;
|
||||
unsigned MissingArgIndex, MissingArgCount;
|
||||
InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount,
|
||||
IncludedFlagsBitmask);
|
||||
|
||||
// Check for missing argument error.
|
||||
if (MissingArgCount) {
|
||||
Diags.Report(diag::err_drv_missing_argument)
|
||||
<< Args.getArgString(MissingArgIndex) << MissingArgCount;
|
||||
Success = false;
|
||||
}
|
||||
|
||||
// Issue errors on unknown arguments.
|
||||
for (const Arg *A : Args.filtered(OPT_UNKNOWN)) {
|
||||
auto ArgString = A->getAsString(Args);
|
||||
std::string Nearest;
|
||||
if (OptTbl->findNearest(ArgString, Nearest, IncludedFlagsBitmask) > 1)
|
||||
Diags.Report(diag::err_drv_unknown_argument) << ArgString;
|
||||
else
|
||||
Diags.Report(diag::err_drv_unknown_argument_with_suggestion)
|
||||
<< ArgString << Nearest;
|
||||
Success = false;
|
||||
}
|
||||
|
||||
// Construct the invocation.
|
||||
|
||||
// Target Options
|
||||
Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple));
|
||||
Opts.CPU = Args.getLastArgValue(OPT_target_cpu);
|
||||
Opts.Features = Args.getAllArgValues(OPT_target_feature);
|
||||
|
||||
// Use the default target triple if unspecified.
|
||||
if (Opts.Triple.empty())
|
||||
Opts.Triple = llvm::sys::getDefaultTargetTriple();
|
||||
|
||||
// Language Options
|
||||
Opts.IncludePaths = Args.getAllArgValues(OPT_I);
|
||||
Opts.NoInitialTextSection = Args.hasArg(OPT_n);
|
||||
Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels);
|
||||
// Any DebugInfoKind implies GenDwarfForAssembly.
|
||||
Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ);
|
||||
|
||||
if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections,
|
||||
OPT_compress_debug_sections_EQ)) {
|
||||
if (A->getOption().getID() == OPT_compress_debug_sections) {
|
||||
// TODO: be more clever about the compression type auto-detection
|
||||
Opts.CompressDebugSections = llvm::DebugCompressionType::GNU;
|
||||
} else {
|
||||
Opts.CompressDebugSections =
|
||||
llvm::StringSwitch<llvm::DebugCompressionType>(A->getValue())
|
||||
.Case("none", llvm::DebugCompressionType::None)
|
||||
.Case("zlib", llvm::DebugCompressionType::Z)
|
||||
.Case("zlib-gnu", llvm::DebugCompressionType::GNU)
|
||||
.Default(llvm::DebugCompressionType::None);
|
||||
}
|
||||
}
|
||||
|
||||
Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations);
|
||||
Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags);
|
||||
Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags);
|
||||
Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer);
|
||||
Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir);
|
||||
Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name);
|
||||
|
||||
for (const auto &Arg : Args.getAllArgValues(OPT_fdebug_prefix_map_EQ))
|
||||
Opts.DebugPrefixMap.insert(StringRef(Arg).split('='));
|
||||
|
||||
// Frontend Options
|
||||
if (Args.hasArg(OPT_INPUT)) {
|
||||
bool First = true;
|
||||
for (const Arg *A : Args.filtered(OPT_INPUT)) {
|
||||
if (First) {
|
||||
Opts.InputFile = A->getValue();
|
||||
First = false;
|
||||
} else {
|
||||
Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args);
|
||||
Success = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
|
||||
Opts.OutputPath = Args.getLastArgValue(OPT_o);
|
||||
Opts.SplitDwarfFile = Args.getLastArgValue(OPT_split_dwarf_file);
|
||||
if (Arg *A = Args.getLastArg(OPT_filetype)) {
|
||||
StringRef Name = A->getValue();
|
||||
unsigned OutputType = StringSwitch<unsigned>(Name)
|
||||
.Case("asm", FT_Asm)
|
||||
.Case("null", FT_Null)
|
||||
.Case("obj", FT_Obj)
|
||||
.Default(~0U);
|
||||
if (OutputType == ~0U) {
|
||||
Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name;
|
||||
Success = false;
|
||||
} else
|
||||
Opts.OutputType = FileType(OutputType);
|
||||
}
|
||||
Opts.ShowHelp = Args.hasArg(OPT_help);
|
||||
Opts.ShowVersion = Args.hasArg(OPT_version);
|
||||
|
||||
// Transliterate Options
|
||||
Opts.OutputAsmVariant =
|
||||
getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags);
|
||||
Opts.ShowEncoding = Args.hasArg(OPT_show_encoding);
|
||||
Opts.ShowInst = Args.hasArg(OPT_show_inst);
|
||||
|
||||
// Assemble Options
|
||||
Opts.RelaxAll = Args.hasArg(OPT_mrelax_all);
|
||||
Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack);
|
||||
Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings);
|
||||
Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic");
|
||||
Opts.IncrementalLinkerCompatible =
|
||||
Args.hasArg(OPT_mincremental_linker_compatible);
|
||||
Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym);
|
||||
|
||||
return Success;
|
||||
}
|
||||
|
||||
static std::unique_ptr<raw_fd_ostream>
|
||||
getOutputStream(StringRef Path, DiagnosticsEngine &Diags, bool Binary) {
|
||||
// Make sure that the Out file gets unlinked from the disk if we get a
|
||||
// SIGINT.
|
||||
if (Path != "-")
|
||||
sys::RemoveFileOnSignal(Path);
|
||||
|
||||
std::error_code EC;
|
||||
auto Out = llvm::make_unique<raw_fd_ostream>(
|
||||
Path, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text));
|
||||
if (EC) {
|
||||
Diags.Report(diag::err_fe_unable_to_open_output) << Path << EC.message();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return Out;
|
||||
}
|
||||
|
||||
static bool ExecuteAssembler(AssemblerInvocation &Opts,
|
||||
DiagnosticsEngine &Diags) {
|
||||
// Get the target specific parser.
|
||||
std::string Error;
|
||||
const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error);
|
||||
if (!TheTarget)
|
||||
return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
|
||||
|
||||
ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer =
|
||||
MemoryBuffer::getFileOrSTDIN(Opts.InputFile);
|
||||
|
||||
if (std::error_code EC = Buffer.getError()) {
|
||||
Error = EC.message();
|
||||
return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile;
|
||||
}
|
||||
|
||||
SourceMgr SrcMgr;
|
||||
|
||||
// Tell SrcMgr about this buffer, which is what the parser will pick up.
|
||||
SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc());
|
||||
|
||||
// Record the location of the include directories so that the lexer can find
|
||||
// it later.
|
||||
SrcMgr.setIncludeDirs(Opts.IncludePaths);
|
||||
|
||||
std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(Opts.Triple));
|
||||
assert(MRI && "Unable to create target register info!");
|
||||
|
||||
std::unique_ptr<MCAsmInfo> MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple));
|
||||
assert(MAI && "Unable to create target asm info!");
|
||||
|
||||
// Ensure MCAsmInfo initialization occurs before any use, otherwise sections
|
||||
// may be created with a combination of default and explicit settings.
|
||||
MAI->setCompressDebugSections(Opts.CompressDebugSections);
|
||||
|
||||
MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations);
|
||||
|
||||
bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj;
|
||||
if (Opts.OutputPath.empty())
|
||||
Opts.OutputPath = "-";
|
||||
std::unique_ptr<raw_fd_ostream> FDOS =
|
||||
getOutputStream(Opts.OutputPath, Diags, IsBinary);
|
||||
if (!FDOS)
|
||||
return true;
|
||||
std::unique_ptr<raw_fd_ostream> DwoOS;
|
||||
if (!Opts.SplitDwarfFile.empty())
|
||||
DwoOS = getOutputStream(Opts.SplitDwarfFile, Diags, IsBinary);
|
||||
|
||||
// FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and
|
||||
// MCObjectFileInfo needs a MCContext reference in order to initialize itself.
|
||||
std::unique_ptr<MCObjectFileInfo> MOFI(new MCObjectFileInfo());
|
||||
|
||||
MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr);
|
||||
|
||||
bool PIC = false;
|
||||
if (Opts.RelocationModel == "static") {
|
||||
PIC = false;
|
||||
} else if (Opts.RelocationModel == "pic") {
|
||||
PIC = true;
|
||||
} else {
|
||||
assert(Opts.RelocationModel == "dynamic-no-pic" &&
|
||||
"Invalid PIC model!");
|
||||
PIC = false;
|
||||
}
|
||||
|
||||
MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx);
|
||||
if (Opts.SaveTemporaryLabels)
|
||||
Ctx.setAllowTemporaryLabels(false);
|
||||
if (Opts.GenDwarfForAssembly)
|
||||
Ctx.setGenDwarfForAssembly(true);
|
||||
if (!Opts.DwarfDebugFlags.empty())
|
||||
Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags));
|
||||
if (!Opts.DwarfDebugProducer.empty())
|
||||
Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer));
|
||||
if (!Opts.DebugCompilationDir.empty())
|
||||
Ctx.setCompilationDir(Opts.DebugCompilationDir);
|
||||
if (!Opts.DebugPrefixMap.empty())
|
||||
for (const auto &KV : Opts.DebugPrefixMap)
|
||||
Ctx.addDebugPrefixMapEntry(KV.first, KV.second);
|
||||
if (!Opts.MainFileName.empty())
|
||||
Ctx.setMainFileName(StringRef(Opts.MainFileName));
|
||||
Ctx.setDwarfVersion(Opts.DwarfVersion);
|
||||
|
||||
// Build up the feature string from the target feature list.
|
||||
std::string FS;
|
||||
if (!Opts.Features.empty()) {
|
||||
FS = Opts.Features[0];
|
||||
for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i)
|
||||
FS += "," + Opts.Features[i];
|
||||
}
|
||||
|
||||
std::unique_ptr<MCStreamer> Str;
|
||||
|
||||
std::unique_ptr<MCInstrInfo> MCII(TheTarget->createMCInstrInfo());
|
||||
std::unique_ptr<MCSubtargetInfo> STI(
|
||||
TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS));
|
||||
|
||||
raw_pwrite_stream *Out = FDOS.get();
|
||||
std::unique_ptr<buffer_ostream> BOS;
|
||||
|
||||
// FIXME: There is a bit of code duplication with addPassesToEmitFile.
|
||||
if (Opts.OutputType == AssemblerInvocation::FT_Asm) {
|
||||
MCInstPrinter *IP = TheTarget->createMCInstPrinter(
|
||||
llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI);
|
||||
|
||||
std::unique_ptr<MCCodeEmitter> CE;
|
||||
if (Opts.ShowEncoding)
|
||||
CE.reset(TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx));
|
||||
MCTargetOptions MCOptions;
|
||||
std::unique_ptr<MCAsmBackend> MAB(
|
||||
TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
|
||||
|
||||
auto FOut = llvm::make_unique<formatted_raw_ostream>(*Out);
|
||||
Str.reset(TheTarget->createAsmStreamer(
|
||||
Ctx, std::move(FOut), /*asmverbose*/ true,
|
||||
/*useDwarfDirectory*/ true, IP, std::move(CE), std::move(MAB),
|
||||
Opts.ShowInst));
|
||||
} else if (Opts.OutputType == AssemblerInvocation::FT_Null) {
|
||||
Str.reset(createNullStreamer(Ctx));
|
||||
} else {
|
||||
assert(Opts.OutputType == AssemblerInvocation::FT_Obj &&
|
||||
"Invalid file type!");
|
||||
if (!FDOS->supportsSeeking()) {
|
||||
BOS = make_unique<buffer_ostream>(*FDOS);
|
||||
Out = BOS.get();
|
||||
}
|
||||
|
||||
std::unique_ptr<MCCodeEmitter> CE(
|
||||
TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx));
|
||||
MCTargetOptions MCOptions;
|
||||
std::unique_ptr<MCAsmBackend> MAB(
|
||||
TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
|
||||
std::unique_ptr<MCObjectWriter> OW =
|
||||
DwoOS ? MAB->createDwoObjectWriter(*Out, *DwoOS)
|
||||
: MAB->createObjectWriter(*Out);
|
||||
|
||||
Triple T(Opts.Triple);
|
||||
Str.reset(TheTarget->createMCObjectStreamer(
|
||||
T, Ctx, std::move(MAB), std::move(OW), std::move(CE), *STI,
|
||||
Opts.RelaxAll, Opts.IncrementalLinkerCompatible,
|
||||
/*DWARFMustBeAtTheEnd*/ true));
|
||||
Str.get()->InitSections(Opts.NoExecStack);
|
||||
}
|
||||
|
||||
// Assembly to object compilation should leverage assembly info.
|
||||
Str->setUseAssemblerInfoForParsing(true);
|
||||
|
||||
bool Failed = false;
|
||||
|
||||
std::unique_ptr<MCAsmParser> Parser(
|
||||
createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI));
|
||||
|
||||
// FIXME: init MCTargetOptions from sanitizer flags here.
|
||||
MCTargetOptions Options;
|
||||
std::unique_ptr<MCTargetAsmParser> TAP(
|
||||
TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options));
|
||||
if (!TAP)
|
||||
Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
|
||||
|
||||
// Set values for symbols, if any.
|
||||
for (auto &S : Opts.SymbolDefs) {
|
||||
auto Pair = StringRef(S).split('=');
|
||||
auto Sym = Pair.first;
|
||||
auto Val = Pair.second;
|
||||
int64_t Value = 1;
|
||||
// We have already error checked this in the driver.
|
||||
Val.getAsInteger(0, Value);
|
||||
Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value);
|
||||
}
|
||||
|
||||
if (!Failed) {
|
||||
Parser->setTargetParser(*TAP.get());
|
||||
Failed = Parser->Run(Opts.NoInitialTextSection);
|
||||
}
|
||||
|
||||
// Close Streamer first.
|
||||
// It might have a reference to the output stream.
|
||||
Str.reset();
|
||||
// Close the output stream early.
|
||||
BOS.reset();
|
||||
FDOS.reset();
|
||||
|
||||
// Delete output file if there were errors.
|
||||
if (Failed) {
|
||||
if (Opts.OutputPath != "-")
|
||||
sys::fs::remove(Opts.OutputPath);
|
||||
if (!Opts.SplitDwarfFile.empty() && Opts.SplitDwarfFile != "-")
|
||||
sys::fs::remove(Opts.SplitDwarfFile);
|
||||
}
|
||||
|
||||
return Failed;
|
||||
}
|
||||
|
||||
static void LLVMErrorHandler(void *UserData, const std::string &Message,
|
||||
bool GenCrashDiag) {
|
||||
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
|
||||
|
||||
Diags.Report(diag::err_fe_error_backend) << Message;
|
||||
|
||||
// We cannot recover from llvm errors.
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
|
||||
// Initialize targets and assembly printers/parsers.
|
||||
InitializeAllTargetInfos();
|
||||
InitializeAllTargetMCs();
|
||||
InitializeAllAsmParsers();
|
||||
|
||||
// Construct our diagnostic client.
|
||||
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
|
||||
TextDiagnosticPrinter *DiagClient
|
||||
= new TextDiagnosticPrinter(errs(), &*DiagOpts);
|
||||
DiagClient->setPrefix("clang -cc1as");
|
||||
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
|
||||
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
|
||||
|
||||
// Set an error handler, so that any LLVM backend diagnostics go through our
|
||||
// error handler.
|
||||
ScopedFatalErrorHandler FatalErrorHandler
|
||||
(LLVMErrorHandler, static_cast<void*>(&Diags));
|
||||
|
||||
// Parse the arguments.
|
||||
AssemblerInvocation Asm;
|
||||
if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags))
|
||||
return 1;
|
||||
|
||||
if (Asm.ShowHelp) {
|
||||
std::unique_ptr<OptTable> Opts(driver::createDriverOptTable());
|
||||
Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler",
|
||||
/*Include=*/driver::options::CC1AsOption, /*Exclude=*/0,
|
||||
/*ShowAllAliases=*/false);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Honor -version.
|
||||
//
|
||||
// FIXME: Use a better -version message?
|
||||
if (Asm.ShowVersion) {
|
||||
llvm::cl::PrintVersionMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Honor -mllvm.
|
||||
//
|
||||
// FIXME: Remove this, one day.
|
||||
if (!Asm.LLVMArgs.empty()) {
|
||||
unsigned NumArgs = Asm.LLVMArgs.size();
|
||||
auto Args = llvm::make_unique<const char*[]>(NumArgs + 2);
|
||||
Args[0] = "clang (LLVM option parsing)";
|
||||
for (unsigned i = 0; i != NumArgs; ++i)
|
||||
Args[i + 1] = Asm.LLVMArgs[i].c_str();
|
||||
Args[NumArgs + 1] = nullptr;
|
||||
llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get());
|
||||
}
|
||||
|
||||
// Execute the invocation, unless there were parsing errors.
|
||||
bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags);
|
||||
|
||||
// If any timers were active but haven't been destroyed yet, print their
|
||||
// results now.
|
||||
TimerGroup::printAll(errs());
|
||||
|
||||
return !!Failed;
|
||||
}
|
||||
@ -1,196 +0,0 @@
|
||||
//===-- cc1gen_reproducer_main.cpp - Clang reproducer generator ----------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This is the entry point to the clang -cc1gen-reproducer functionality, which
|
||||
// generates reproducers for invocations for clang-based tools.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "clang/Basic/Diagnostic.h"
|
||||
#include "clang/Basic/LLVM.h"
|
||||
#include "clang/Basic/VirtualFileSystem.h"
|
||||
#include "clang/Driver/Compilation.h"
|
||||
#include "clang/Driver/Driver.h"
|
||||
#include "llvm/ADT/ArrayRef.h"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/Support/FileSystem.h"
|
||||
#include "llvm/Support/TargetSelect.h"
|
||||
#include "llvm/Support/YAMLTraits.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
|
||||
using namespace clang;
|
||||
|
||||
namespace {
|
||||
|
||||
struct UnsavedFileHash {
|
||||
std::string Name;
|
||||
std::string MD5;
|
||||
};
|
||||
|
||||
struct ClangInvocationInfo {
|
||||
std::string Toolchain;
|
||||
std::string LibclangOperation;
|
||||
std::string LibclangOptions;
|
||||
std::vector<std::string> Arguments;
|
||||
std::vector<std::string> InvocationArguments;
|
||||
std::vector<UnsavedFileHash> UnsavedFileHashes;
|
||||
bool Dump = false;
|
||||
};
|
||||
|
||||
} // end anonymous namespace
|
||||
|
||||
LLVM_YAML_IS_SEQUENCE_VECTOR(UnsavedFileHash)
|
||||
|
||||
namespace llvm {
|
||||
namespace yaml {
|
||||
|
||||
template <> struct MappingTraits<UnsavedFileHash> {
|
||||
static void mapping(IO &IO, UnsavedFileHash &Info) {
|
||||
IO.mapRequired("name", Info.Name);
|
||||
IO.mapRequired("md5", Info.MD5);
|
||||
}
|
||||
};
|
||||
|
||||
template <> struct MappingTraits<ClangInvocationInfo> {
|
||||
static void mapping(IO &IO, ClangInvocationInfo &Info) {
|
||||
IO.mapRequired("toolchain", Info.Toolchain);
|
||||
IO.mapOptional("libclang.operation", Info.LibclangOperation);
|
||||
IO.mapOptional("libclang.opts", Info.LibclangOptions);
|
||||
IO.mapRequired("args", Info.Arguments);
|
||||
IO.mapOptional("invocation-args", Info.InvocationArguments);
|
||||
IO.mapOptional("unsaved_file_hashes", Info.UnsavedFileHashes);
|
||||
}
|
||||
};
|
||||
|
||||
} // end namespace yaml
|
||||
} // end namespace llvm
|
||||
|
||||
static std::string generateReproducerMetaInfo(const ClangInvocationInfo &Info) {
|
||||
std::string Result;
|
||||
llvm::raw_string_ostream OS(Result);
|
||||
OS << '{';
|
||||
bool NeedComma = false;
|
||||
auto EmitKey = [&](StringRef Key) {
|
||||
if (NeedComma)
|
||||
OS << ", ";
|
||||
NeedComma = true;
|
||||
OS << '"' << Key << "\": ";
|
||||
};
|
||||
auto EmitStringKey = [&](StringRef Key, StringRef Value) {
|
||||
if (Value.empty())
|
||||
return;
|
||||
EmitKey(Key);
|
||||
OS << '"' << Value << '"';
|
||||
};
|
||||
EmitStringKey("libclang.operation", Info.LibclangOperation);
|
||||
EmitStringKey("libclang.opts", Info.LibclangOptions);
|
||||
if (!Info.InvocationArguments.empty()) {
|
||||
EmitKey("invocation-args");
|
||||
OS << '[';
|
||||
for (const auto &Arg : llvm::enumerate(Info.InvocationArguments)) {
|
||||
if (Arg.index())
|
||||
OS << ',';
|
||||
OS << '"' << Arg.value() << '"';
|
||||
}
|
||||
OS << ']';
|
||||
}
|
||||
OS << '}';
|
||||
// FIXME: Compare unsaved file hashes and report mismatch in the reproducer.
|
||||
if (Info.Dump)
|
||||
llvm::outs() << "REPRODUCER METAINFO: " << OS.str() << "\n";
|
||||
return std::move(OS.str());
|
||||
}
|
||||
|
||||
/// Generates a reproducer for a set of arguments from a specific invocation.
|
||||
static llvm::Optional<driver::Driver::CompilationDiagnosticReport>
|
||||
generateReproducerForInvocationArguments(ArrayRef<const char *> Argv,
|
||||
const ClangInvocationInfo &Info) {
|
||||
using namespace driver;
|
||||
auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(Argv[0]);
|
||||
|
||||
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions;
|
||||
|
||||
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
|
||||
DiagnosticsEngine Diags(DiagID, &*DiagOpts, new IgnoringDiagConsumer());
|
||||
ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false);
|
||||
Driver TheDriver(Argv[0], llvm::sys::getDefaultTargetTriple(), Diags);
|
||||
TheDriver.setTargetAndMode(TargetAndMode);
|
||||
|
||||
std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(Argv));
|
||||
if (C && !C->containsError()) {
|
||||
for (const auto &J : C->getJobs()) {
|
||||
if (const Command *Cmd = dyn_cast<Command>(&J)) {
|
||||
Driver::CompilationDiagnosticReport Report;
|
||||
TheDriver.generateCompilationDiagnostics(
|
||||
*C, *Cmd, generateReproducerMetaInfo(Info), &Report);
|
||||
return Report;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return None;
|
||||
}
|
||||
|
||||
std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes);
|
||||
|
||||
static void printReproducerInformation(
|
||||
llvm::raw_ostream &OS, const ClangInvocationInfo &Info,
|
||||
const driver::Driver::CompilationDiagnosticReport &Report) {
|
||||
OS << "REPRODUCER:\n";
|
||||
OS << "{\n";
|
||||
OS << R"("files":[)";
|
||||
for (const auto &File : llvm::enumerate(Report.TemporaryFiles)) {
|
||||
if (File.index())
|
||||
OS << ',';
|
||||
OS << '"' << File.value() << '"';
|
||||
}
|
||||
OS << "]\n}\n";
|
||||
}
|
||||
|
||||
int cc1gen_reproducer_main(ArrayRef<const char *> Argv, const char *Argv0,
|
||||
void *MainAddr) {
|
||||
if (Argv.size() < 1) {
|
||||
llvm::errs() << "error: missing invocation file\n";
|
||||
return 1;
|
||||
}
|
||||
// Parse the invocation descriptor.
|
||||
StringRef Input = Argv[0];
|
||||
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
|
||||
llvm::MemoryBuffer::getFile(Input);
|
||||
if (!Buffer) {
|
||||
llvm::errs() << "error: failed to read " << Input << ": "
|
||||
<< Buffer.getError().message() << "\n";
|
||||
return 1;
|
||||
}
|
||||
llvm::yaml::Input YAML(Buffer.get()->getBuffer());
|
||||
ClangInvocationInfo InvocationInfo;
|
||||
YAML >> InvocationInfo;
|
||||
if (Argv.size() > 1 && Argv[1] == StringRef("-v"))
|
||||
InvocationInfo.Dump = true;
|
||||
|
||||
// Create an invocation that will produce the reproducer.
|
||||
std::vector<const char *> DriverArgs;
|
||||
for (const auto &Arg : InvocationInfo.Arguments)
|
||||
DriverArgs.push_back(Arg.c_str());
|
||||
std::string Path = GetExecutablePath(Argv0, /*CanonicalPrefixes=*/true);
|
||||
DriverArgs[0] = Path.c_str();
|
||||
llvm::Optional<driver::Driver::CompilationDiagnosticReport> Report =
|
||||
generateReproducerForInvocationArguments(DriverArgs, InvocationInfo);
|
||||
|
||||
// Emit the information about the reproduce files to stdout.
|
||||
int Result = 1;
|
||||
if (Report) {
|
||||
printReproducerInformation(llvm::outs(), InvocationInfo, *Report);
|
||||
Result = 0;
|
||||
}
|
||||
|
||||
// Remove the input file.
|
||||
llvm::sys::fs::remove(Input);
|
||||
return Result;
|
||||
}
|
||||
@ -1,514 +0,0 @@
|
||||
//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This is the entry point to the clang driver; it is a thin wrapper
|
||||
// for functionality in the Driver clang library.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "clang/Driver/Driver.h"
|
||||
#include "clang/Basic/DiagnosticOptions.h"
|
||||
#include "clang/Driver/Compilation.h"
|
||||
#include "clang/Driver/DriverDiagnostic.h"
|
||||
#include "clang/Driver/Options.h"
|
||||
#include "clang/Driver/ToolChain.h"
|
||||
#include "clang/Frontend/ChainedDiagnosticConsumer.h"
|
||||
#include "clang/Frontend/CompilerInvocation.h"
|
||||
#include "clang/Frontend/SerializedDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/TextDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/Utils.h"
|
||||
#include "llvm/ADT/ArrayRef.h"
|
||||
#include "llvm/ADT/SmallString.h"
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
#include "llvm/Option/ArgList.h"
|
||||
#include "llvm/Option/OptTable.h"
|
||||
#include "llvm/Option/Option.h"
|
||||
#include "llvm/Support/CommandLine.h"
|
||||
#include "llvm/Support/ErrorHandling.h"
|
||||
#include "llvm/Support/FileSystem.h"
|
||||
#include "llvm/Support/Host.h"
|
||||
#include "llvm/Support/InitLLVM.h"
|
||||
#include "llvm/Support/Path.h"
|
||||
#include "llvm/Support/Process.h"
|
||||
#include "llvm/Support/Program.h"
|
||||
#include "llvm/Support/Regex.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
#include "llvm/Support/StringSaver.h"
|
||||
#include "llvm/Support/TargetSelect.h"
|
||||
#include "llvm/Support/Timer.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <system_error>
|
||||
using namespace clang;
|
||||
using namespace clang::driver;
|
||||
using namespace llvm::opt;
|
||||
|
||||
std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) {
|
||||
if (!CanonicalPrefixes) {
|
||||
SmallString<128> ExecutablePath(Argv0);
|
||||
// Do a PATH lookup if Argv0 isn't a valid path.
|
||||
if (!llvm::sys::fs::exists(ExecutablePath))
|
||||
if (llvm::ErrorOr<std::string> P =
|
||||
llvm::sys::findProgramByName(ExecutablePath))
|
||||
ExecutablePath = *P;
|
||||
return ExecutablePath.str();
|
||||
}
|
||||
|
||||
// This just needs to be some symbol in the binary; C++ doesn't
|
||||
// allow taking the address of ::main however.
|
||||
void *P = (void*) (intptr_t) GetExecutablePath;
|
||||
return llvm::sys::fs::getMainExecutable(Argv0, P);
|
||||
}
|
||||
|
||||
static const char *GetStableCStr(std::set<std::string> &SavedStrings,
|
||||
StringRef S) {
|
||||
return SavedStrings.insert(S).first->c_str();
|
||||
}
|
||||
|
||||
/// ApplyQAOverride - Apply a list of edits to the input argument lists.
|
||||
///
|
||||
/// The input string is a space separate list of edits to perform,
|
||||
/// they are applied in order to the input argument lists. Edits
|
||||
/// should be one of the following forms:
|
||||
///
|
||||
/// '#': Silence information about the changes to the command line arguments.
|
||||
///
|
||||
/// '^': Add FOO as a new argument at the beginning of the command line.
|
||||
///
|
||||
/// '+': Add FOO as a new argument at the end of the command line.
|
||||
///
|
||||
/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command
|
||||
/// line.
|
||||
///
|
||||
/// 'xOPTION': Removes all instances of the literal argument OPTION.
|
||||
///
|
||||
/// 'XOPTION': Removes all instances of the literal argument OPTION,
|
||||
/// and the following argument.
|
||||
///
|
||||
/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox'
|
||||
/// at the end of the command line.
|
||||
///
|
||||
/// \param OS - The stream to write edit information to.
|
||||
/// \param Args - The vector of command line arguments.
|
||||
/// \param Edit - The override command to perform.
|
||||
/// \param SavedStrings - Set to use for storing string representations.
|
||||
static void ApplyOneQAOverride(raw_ostream &OS,
|
||||
SmallVectorImpl<const char*> &Args,
|
||||
StringRef Edit,
|
||||
std::set<std::string> &SavedStrings) {
|
||||
// This does not need to be efficient.
|
||||
|
||||
if (Edit[0] == '^') {
|
||||
const char *Str =
|
||||
GetStableCStr(SavedStrings, Edit.substr(1));
|
||||
OS << "### Adding argument " << Str << " at beginning\n";
|
||||
Args.insert(Args.begin() + 1, Str);
|
||||
} else if (Edit[0] == '+') {
|
||||
const char *Str =
|
||||
GetStableCStr(SavedStrings, Edit.substr(1));
|
||||
OS << "### Adding argument " << Str << " at end\n";
|
||||
Args.push_back(Str);
|
||||
} else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") &&
|
||||
Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) {
|
||||
StringRef MatchPattern = Edit.substr(2).split('/').first;
|
||||
StringRef ReplPattern = Edit.substr(2).split('/').second;
|
||||
ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1);
|
||||
|
||||
for (unsigned i = 1, e = Args.size(); i != e; ++i) {
|
||||
// Ignore end-of-line response file markers
|
||||
if (Args[i] == nullptr)
|
||||
continue;
|
||||
std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]);
|
||||
|
||||
if (Repl != Args[i]) {
|
||||
OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n";
|
||||
Args[i] = GetStableCStr(SavedStrings, Repl);
|
||||
}
|
||||
}
|
||||
} else if (Edit[0] == 'x' || Edit[0] == 'X') {
|
||||
auto Option = Edit.substr(1);
|
||||
for (unsigned i = 1; i < Args.size();) {
|
||||
if (Option == Args[i]) {
|
||||
OS << "### Deleting argument " << Args[i] << '\n';
|
||||
Args.erase(Args.begin() + i);
|
||||
if (Edit[0] == 'X') {
|
||||
if (i < Args.size()) {
|
||||
OS << "### Deleting argument " << Args[i] << '\n';
|
||||
Args.erase(Args.begin() + i);
|
||||
} else
|
||||
OS << "### Invalid X edit, end of command line!\n";
|
||||
}
|
||||
} else
|
||||
++i;
|
||||
}
|
||||
} else if (Edit[0] == 'O') {
|
||||
for (unsigned i = 1; i < Args.size();) {
|
||||
const char *A = Args[i];
|
||||
// Ignore end-of-line response file markers
|
||||
if (A == nullptr)
|
||||
continue;
|
||||
if (A[0] == '-' && A[1] == 'O' &&
|
||||
(A[2] == '\0' ||
|
||||
(A[3] == '\0' && (A[2] == 's' || A[2] == 'z' ||
|
||||
('0' <= A[2] && A[2] <= '9'))))) {
|
||||
OS << "### Deleting argument " << Args[i] << '\n';
|
||||
Args.erase(Args.begin() + i);
|
||||
} else
|
||||
++i;
|
||||
}
|
||||
OS << "### Adding argument " << Edit << " at end\n";
|
||||
Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str()));
|
||||
} else {
|
||||
OS << "### Unrecognized edit: " << Edit << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
/// ApplyQAOverride - Apply a comma separate list of edits to the
|
||||
/// input argument lists. See ApplyOneQAOverride.
|
||||
static void ApplyQAOverride(SmallVectorImpl<const char*> &Args,
|
||||
const char *OverrideStr,
|
||||
std::set<std::string> &SavedStrings) {
|
||||
raw_ostream *OS = &llvm::errs();
|
||||
|
||||
if (OverrideStr[0] == '#') {
|
||||
++OverrideStr;
|
||||
OS = &llvm::nulls();
|
||||
}
|
||||
|
||||
*OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n";
|
||||
|
||||
// This does not need to be efficient.
|
||||
|
||||
const char *S = OverrideStr;
|
||||
while (*S) {
|
||||
const char *End = ::strchr(S, ' ');
|
||||
if (!End)
|
||||
End = S + strlen(S);
|
||||
if (End != S)
|
||||
ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings);
|
||||
S = End;
|
||||
if (*S != '\0')
|
||||
++S;
|
||||
}
|
||||
}
|
||||
|
||||
extern int cc1_main(ArrayRef<const char *> Argv, const char *Argv0,
|
||||
void *MainAddr);
|
||||
extern int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0,
|
||||
void *MainAddr);
|
||||
extern int cc1gen_reproducer_main(ArrayRef<const char *> Argv,
|
||||
const char *Argv0, void *MainAddr);
|
||||
|
||||
static void insertTargetAndModeArgs(const ParsedClangName &NameParts,
|
||||
SmallVectorImpl<const char *> &ArgVector,
|
||||
std::set<std::string> &SavedStrings) {
|
||||
// Put target and mode arguments at the start of argument list so that
|
||||
// arguments specified in command line could override them. Avoid putting
|
||||
// them at index 0, as an option like '-cc1' must remain the first.
|
||||
int InsertionPoint = 0;
|
||||
if (ArgVector.size() > 0)
|
||||
++InsertionPoint;
|
||||
|
||||
if (NameParts.DriverMode) {
|
||||
// Add the mode flag to the arguments.
|
||||
ArgVector.insert(ArgVector.begin() + InsertionPoint,
|
||||
GetStableCStr(SavedStrings, NameParts.DriverMode));
|
||||
}
|
||||
|
||||
if (NameParts.TargetIsValid) {
|
||||
const char *arr[] = {"-target", GetStableCStr(SavedStrings,
|
||||
NameParts.TargetPrefix)};
|
||||
ArgVector.insert(ArgVector.begin() + InsertionPoint,
|
||||
std::begin(arr), std::end(arr));
|
||||
}
|
||||
}
|
||||
|
||||
static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver,
|
||||
SmallVectorImpl<const char *> &Opts) {
|
||||
llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts);
|
||||
// The first instance of '#' should be replaced with '=' in each option.
|
||||
for (const char *Opt : Opts)
|
||||
if (char *NumberSignPtr = const_cast<char *>(::strchr(Opt, '#')))
|
||||
*NumberSignPtr = '=';
|
||||
}
|
||||
|
||||
static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) {
|
||||
// Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE.
|
||||
TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS");
|
||||
if (TheDriver.CCPrintOptions)
|
||||
TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE");
|
||||
|
||||
// Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE.
|
||||
TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS");
|
||||
if (TheDriver.CCPrintHeaders)
|
||||
TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE");
|
||||
|
||||
// Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE.
|
||||
TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS");
|
||||
if (TheDriver.CCLogDiagnostics)
|
||||
TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE");
|
||||
}
|
||||
|
||||
static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient,
|
||||
const std::string &Path) {
|
||||
// If the clang binary happens to be named cl.exe for compatibility reasons,
|
||||
// use clang-cl.exe as the prefix to avoid confusion between clang and MSVC.
|
||||
StringRef ExeBasename(llvm::sys::path::filename(Path));
|
||||
if (ExeBasename.equals_lower("cl.exe"))
|
||||
ExeBasename = "clang-cl.exe";
|
||||
DiagClient->setPrefix(ExeBasename);
|
||||
}
|
||||
|
||||
// This lets us create the DiagnosticsEngine with a properly-filled-out
|
||||
// DiagnosticOptions instance.
|
||||
static DiagnosticOptions *
|
||||
CreateAndPopulateDiagOpts(ArrayRef<const char *> argv) {
|
||||
auto *DiagOpts = new DiagnosticOptions;
|
||||
std::unique_ptr<OptTable> Opts(createDriverOptTable());
|
||||
unsigned MissingArgIndex, MissingArgCount;
|
||||
InputArgList Args =
|
||||
Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount);
|
||||
// We ignore MissingArgCount and the return value of ParseDiagnosticArgs.
|
||||
// Any errors that would be diagnosed here will also be diagnosed later,
|
||||
// when the DiagnosticsEngine actually exists.
|
||||
(void)ParseDiagnosticArgs(*DiagOpts, Args);
|
||||
return DiagOpts;
|
||||
}
|
||||
|
||||
static void SetInstallDir(SmallVectorImpl<const char *> &argv,
|
||||
Driver &TheDriver, bool CanonicalPrefixes) {
|
||||
// Attempt to find the original path used to invoke the driver, to determine
|
||||
// the installed path. We do this manually, because we want to support that
|
||||
// path being a symlink.
|
||||
SmallString<128> InstalledPath(argv[0]);
|
||||
|
||||
// Do a PATH lookup, if there are no directory components.
|
||||
if (llvm::sys::path::filename(InstalledPath) == InstalledPath)
|
||||
if (llvm::ErrorOr<std::string> Tmp = llvm::sys::findProgramByName(
|
||||
llvm::sys::path::filename(InstalledPath.str())))
|
||||
InstalledPath = *Tmp;
|
||||
|
||||
// FIXME: We don't actually canonicalize this, we just make it absolute.
|
||||
if (CanonicalPrefixes)
|
||||
llvm::sys::fs::make_absolute(InstalledPath);
|
||||
|
||||
StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath));
|
||||
if (llvm::sys::fs::exists(InstalledPathParent))
|
||||
TheDriver.setInstalledDir(InstalledPathParent);
|
||||
}
|
||||
|
||||
static int ExecuteCC1Tool(ArrayRef<const char *> argv, StringRef Tool) {
|
||||
void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath;
|
||||
if (Tool == "")
|
||||
return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP);
|
||||
if (Tool == "as")
|
||||
return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP);
|
||||
if (Tool == "gen-reproducer")
|
||||
return cc1gen_reproducer_main(argv.slice(2), argv[0], GetExecutablePathVP);
|
||||
|
||||
// Reject unknown tools.
|
||||
llvm::errs() << "error: unknown integrated tool '" << Tool << "'. "
|
||||
<< "Valid tools include '-cc1' and '-cc1as'.\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
int mainEntryClickHouseClang(int argc_, /* const */ char **argv_) {
|
||||
llvm::InitLLVM X(argc_, argv_);
|
||||
SmallVector<const char *, 256> argv(argv_, argv_ + argc_);
|
||||
|
||||
if (llvm::sys::Process::FixupStandardFileDescriptors())
|
||||
return 1;
|
||||
|
||||
llvm::InitializeAllTargets();
|
||||
auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]);
|
||||
|
||||
llvm::BumpPtrAllocator A;
|
||||
llvm::StringSaver Saver(A);
|
||||
|
||||
// Parse response files using the GNU syntax, unless we're in CL mode. There
|
||||
// are two ways to put clang in CL compatibility mode: argv[0] is either
|
||||
// clang-cl or cl, or --driver-mode=cl is on the command line. The normal
|
||||
// command line parsing can't happen until after response file parsing, so we
|
||||
// have to manually search for a --driver-mode=cl argument the hard way.
|
||||
// Finally, our -cc1 tools don't care which tokenization mode we use because
|
||||
// response files written by clang will tokenize the same way in either mode.
|
||||
bool ClangCLMode = false;
|
||||
if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") ||
|
||||
std::find_if(argv.begin(), argv.end(), [](const char *F) {
|
||||
return F && strcmp(F, "--driver-mode=cl") == 0;
|
||||
}) != argv.end()) {
|
||||
ClangCLMode = true;
|
||||
}
|
||||
enum { Default, POSIX, Windows } RSPQuoting = Default;
|
||||
for (const char *F : argv) {
|
||||
if (strcmp(F, "--rsp-quoting=posix") == 0)
|
||||
RSPQuoting = POSIX;
|
||||
else if (strcmp(F, "--rsp-quoting=windows") == 0)
|
||||
RSPQuoting = Windows;
|
||||
}
|
||||
|
||||
// Determines whether we want nullptr markers in argv to indicate response
|
||||
// files end-of-lines. We only use this for the /LINK driver argument with
|
||||
// clang-cl.exe on Windows.
|
||||
bool MarkEOLs = ClangCLMode;
|
||||
|
||||
llvm::cl::TokenizerCallback Tokenizer;
|
||||
if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode))
|
||||
Tokenizer = &llvm::cl::TokenizeWindowsCommandLine;
|
||||
else
|
||||
Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
|
||||
|
||||
if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1"))
|
||||
MarkEOLs = false;
|
||||
llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs);
|
||||
|
||||
// Handle -cc1 integrated tools, even if -cc1 was expanded from a response
|
||||
// file.
|
||||
auto FirstArg = std::find_if(argv.begin() + 1, argv.end(),
|
||||
[](const char *A) { return A != nullptr; });
|
||||
if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) {
|
||||
// If -cc1 came from a response file, remove the EOL sentinels.
|
||||
if (MarkEOLs) {
|
||||
auto newEnd = std::remove(argv.begin(), argv.end(), nullptr);
|
||||
argv.resize(newEnd - argv.begin());
|
||||
}
|
||||
return ExecuteCC1Tool(argv, argv[1] + 4);
|
||||
}
|
||||
|
||||
bool CanonicalPrefixes = true;
|
||||
for (int i = 1, size = argv.size(); i < size; ++i) {
|
||||
// Skip end-of-line response file markers
|
||||
if (argv[i] == nullptr)
|
||||
continue;
|
||||
if (StringRef(argv[i]) == "-no-canonical-prefixes") {
|
||||
CanonicalPrefixes = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle CL and _CL_ which permits additional command line options to be
|
||||
// prepended or appended.
|
||||
if (ClangCLMode) {
|
||||
// Arguments in "CL" are prepended.
|
||||
llvm::Optional<std::string> OptCL = llvm::sys::Process::GetEnv("CL");
|
||||
if (OptCL.hasValue()) {
|
||||
SmallVector<const char *, 8> PrependedOpts;
|
||||
getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts);
|
||||
|
||||
// Insert right after the program name to prepend to the argument list.
|
||||
argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end());
|
||||
}
|
||||
// Arguments in "_CL_" are appended.
|
||||
llvm::Optional<std::string> Opt_CL_ = llvm::sys::Process::GetEnv("_CL_");
|
||||
if (Opt_CL_.hasValue()) {
|
||||
SmallVector<const char *, 8> AppendedOpts;
|
||||
getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts);
|
||||
|
||||
// Insert at the end of the argument list to append.
|
||||
argv.append(AppendedOpts.begin(), AppendedOpts.end());
|
||||
}
|
||||
}
|
||||
|
||||
std::set<std::string> SavedStrings;
|
||||
// Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the
|
||||
// scenes.
|
||||
if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) {
|
||||
// FIXME: Driver shouldn't take extra initial argument.
|
||||
ApplyQAOverride(argv, OverrideStr, SavedStrings);
|
||||
}
|
||||
|
||||
std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes);
|
||||
|
||||
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts =
|
||||
CreateAndPopulateDiagOpts(argv);
|
||||
|
||||
TextDiagnosticPrinter *DiagClient
|
||||
= new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
|
||||
FixupDiagPrefixExeName(DiagClient, Path);
|
||||
|
||||
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
|
||||
|
||||
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
|
||||
|
||||
if (!DiagOpts->DiagnosticSerializationFile.empty()) {
|
||||
auto SerializedConsumer =
|
||||
clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile,
|
||||
&*DiagOpts, /*MergeChildRecords=*/true);
|
||||
Diags.setClient(new ChainedDiagnosticConsumer(
|
||||
Diags.takeClient(), std::move(SerializedConsumer)));
|
||||
}
|
||||
|
||||
ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false);
|
||||
|
||||
Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags);
|
||||
SetInstallDir(argv, TheDriver, CanonicalPrefixes);
|
||||
TheDriver.setTargetAndMode(TargetAndMode);
|
||||
|
||||
insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings);
|
||||
|
||||
SetBackdoorDriverOutputsFromEnvVars(TheDriver);
|
||||
|
||||
std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(argv));
|
||||
int Res = 1;
|
||||
if (C && !C->containsError()) {
|
||||
SmallVector<std::pair<int, const Command *>, 4> FailingCommands;
|
||||
Res = TheDriver.ExecuteCompilation(*C, FailingCommands);
|
||||
|
||||
// Force a crash to test the diagnostics.
|
||||
if (TheDriver.GenReproducer) {
|
||||
Diags.Report(diag::err_drv_force_crash)
|
||||
<< !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH");
|
||||
|
||||
// Pretend that every command failed.
|
||||
FailingCommands.clear();
|
||||
for (const auto &J : C->getJobs())
|
||||
if (const Command *C = dyn_cast<Command>(&J))
|
||||
FailingCommands.push_back(std::make_pair(-1, C));
|
||||
}
|
||||
|
||||
for (const auto &P : FailingCommands) {
|
||||
int CommandRes = P.first;
|
||||
const Command *FailingCommand = P.second;
|
||||
if (!Res)
|
||||
Res = CommandRes;
|
||||
|
||||
// If result status is < 0, then the driver command signalled an error.
|
||||
// If result status is 70, then the driver command reported a fatal error.
|
||||
// On Windows, abort will return an exit code of 3. In these cases,
|
||||
// generate additional diagnostic information if possible.
|
||||
bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70;
|
||||
#ifdef _WIN32
|
||||
DiagnoseCrash |= CommandRes == 3;
|
||||
#endif
|
||||
if (DiagnoseCrash) {
|
||||
TheDriver.generateCompilationDiagnostics(*C, *FailingCommand);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Diags.getClient()->finish();
|
||||
|
||||
// If any timers were active but haven't been destroyed yet, print their
|
||||
// results now. This happens in -disable-free mode.
|
||||
llvm::TimerGroup::printAll(llvm::errs());
|
||||
|
||||
#ifdef _WIN32
|
||||
// Exit status should not be negative on Win32, unless abnormal termination.
|
||||
// Once abnormal termiation was caught, negative status should not be
|
||||
// propagated.
|
||||
if (Res < 0)
|
||||
Res = 1;
|
||||
#endif
|
||||
|
||||
// If we have multiple failing commands, we return the result of the first
|
||||
// failing command.
|
||||
return Res;
|
||||
}
|
||||
@ -1,150 +0,0 @@
|
||||
//===- tools/lld/lld.cpp - Linker Driver Dispatcher -----------------------===//
|
||||
//
|
||||
// The LLVM Linker
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This file contains the main function of the lld executable. The main
|
||||
// function is a thin wrapper which dispatches to the platform specific
|
||||
// driver.
|
||||
//
|
||||
// lld is a single executable that contains four different linkers for ELF,
|
||||
// COFF, WebAssembly and Mach-O. The main function dispatches according to
|
||||
// argv[0] (i.e. command name). The most common name for each target is shown
|
||||
// below:
|
||||
//
|
||||
// - ld.lld: ELF (Unix)
|
||||
// - ld64: Mach-O (macOS)
|
||||
// - lld-link: COFF (Windows)
|
||||
// - ld-wasm: WebAssembly
|
||||
//
|
||||
// lld can be invoked as "lld" along with "-flavor" option. This is for
|
||||
// backward compatibility and not recommended.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "lld/Common/Driver.h"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/ADT/StringSwitch.h"
|
||||
#include "llvm/ADT/Twine.h"
|
||||
#include "llvm/Support/InitLLVM.h"
|
||||
#include "llvm/Support/Path.h"
|
||||
#include <cstdlib>
|
||||
using namespace lld;
|
||||
using namespace llvm;
|
||||
using namespace llvm::sys;
|
||||
|
||||
/*
|
||||
|
||||
enum Flavor {
|
||||
Invalid,
|
||||
Gnu, // -flavor gnu
|
||||
WinLink, // -flavor link
|
||||
Darwin, // -flavor darwin
|
||||
Wasm, // -flavor wasm
|
||||
};
|
||||
|
||||
LLVM_ATTRIBUTE_NORETURN static void die(const Twine &S) {
|
||||
errs() << S << "\n";
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static Flavor getFlavor(StringRef S) {
|
||||
return StringSwitch<Flavor>(S)
|
||||
.CasesLower("ld", "ld.lld", "gnu", Gnu)
|
||||
.CasesLower("wasm", "ld-wasm", Wasm)
|
||||
.CaseLower("link", WinLink)
|
||||
.CasesLower("ld64", "ld64.lld", "darwin", Darwin)
|
||||
.Default(Invalid);
|
||||
}
|
||||
|
||||
static bool isPETarget(const std::vector<const char *> &V) {
|
||||
for (auto It = V.begin(); It + 1 != V.end(); ++It) {
|
||||
if (StringRef(*It) != "-m")
|
||||
continue;
|
||||
StringRef S = *(It + 1);
|
||||
return S == "i386pe" || S == "i386pep" || S == "thumb2pe" || S == "arm64pe";
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static Flavor parseProgname(StringRef Progname) {
|
||||
#if __APPLE__
|
||||
// Use Darwin driver for "ld" on Darwin.
|
||||
if (Progname == "ld")
|
||||
return Darwin;
|
||||
#endif
|
||||
|
||||
#if LLVM_ON_UNIX
|
||||
// Use GNU driver for "ld" on other Unix-like system.
|
||||
if (Progname == "ld")
|
||||
return Gnu;
|
||||
#endif
|
||||
|
||||
// Progname may be something like "lld-gnu". Parse it.
|
||||
SmallVector<StringRef, 3> V;
|
||||
Progname.split(V, "-");
|
||||
for (StringRef S : V)
|
||||
if (Flavor F = getFlavor(S))
|
||||
return F;
|
||||
return Invalid;
|
||||
}
|
||||
|
||||
static Flavor parseFlavor(std::vector<const char *> &V) {
|
||||
// Parse -flavor option.
|
||||
if (V.size() > 1 && V[1] == StringRef("-flavor")) {
|
||||
if (V.size() <= 2)
|
||||
die("missing arg value for '-flavor'");
|
||||
Flavor F = getFlavor(V[2]);
|
||||
if (F == Invalid)
|
||||
die("Unknown flavor: " + StringRef(V[2]));
|
||||
V.erase(V.begin() + 1, V.begin() + 3);
|
||||
return F;
|
||||
}
|
||||
|
||||
// Deduct the flavor from argv[0].
|
||||
StringRef Arg0 = path::filename(V[0]);
|
||||
if (Arg0.endswith_lower(".exe"))
|
||||
Arg0 = Arg0.drop_back(4);
|
||||
return parseProgname(Arg0);
|
||||
}
|
||||
*/
|
||||
|
||||
// If this function returns true, lld calls _exit() so that it quickly
|
||||
// exits without invoking destructors of globally allocated objects.
|
||||
//
|
||||
// We don't want to do that if we are running tests though, because
|
||||
// doing that breaks leak sanitizer. So, lit sets this environment variable,
|
||||
// and we use it to detect whether we are running tests or not.
|
||||
static bool canExitEarly() { return StringRef(getenv("LLD_IN_TEST")) != "1"; }
|
||||
|
||||
/// Universal linker main(). This linker emulates the gnu, darwin, or
|
||||
/// windows linker based on the argv[0] or -flavor option.
|
||||
int mainEntryClickHouseLLD(int Argc, /* const */ char **Argv) {
|
||||
InitLLVM X(Argc, Argv);
|
||||
|
||||
std::vector<const char *> Args(Argv, Argv + Argc);
|
||||
/*
|
||||
switch (parseFlavor(Args)) {
|
||||
case Gnu:
|
||||
if (isPETarget(Args))
|
||||
return !mingw::link(Args);
|
||||
*/
|
||||
return !elf::link(Args, canExitEarly());
|
||||
/*
|
||||
case WinLink:
|
||||
return !coff::link(Args, canExitEarly());
|
||||
case Darwin:
|
||||
return !mach_o::link(Args, canExitEarly());
|
||||
case Wasm:
|
||||
return !wasm::link(Args, canExitEarly());
|
||||
default:
|
||||
die("lld is a generic driver.\n"
|
||||
"Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-lld"
|
||||
" (WebAssembly) instead");
|
||||
}
|
||||
*/
|
||||
}
|
||||
@ -1,49 +0,0 @@
|
||||
add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG)
|
||||
|
||||
link_directories(${LLVM_LIBRARY_DIRS})
|
||||
|
||||
add_library(clickhouse-compiler-lib
|
||||
driver.cpp
|
||||
cc1_main.cpp
|
||||
cc1as_main.cpp
|
||||
lld.cpp)
|
||||
|
||||
target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0)
|
||||
|
||||
string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti
|
||||
|
||||
llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
|
||||
|
||||
message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
|
||||
|
||||
target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
|
||||
|
||||
# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.
|
||||
|
||||
target_link_libraries(clickhouse-compiler-lib PRIVATE
|
||||
|
||||
clangBasic clangCodeGen clangDriver
|
||||
clangFrontend
|
||||
clangFrontendTool
|
||||
clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend
|
||||
clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers
|
||||
clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic
|
||||
clangCrossTU clangIndex
|
||||
|
||||
lldCOFF
|
||||
lldDriver
|
||||
lldELF
|
||||
lldMinGW
|
||||
lldMachO
|
||||
lldReaderWriter
|
||||
lldYAML
|
||||
lldCommon
|
||||
lldCore
|
||||
|
||||
${REQUIRED_LLVM_LIBRARIES}
|
||||
|
||||
PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARIES} Threads::Threads
|
||||
${MALLOC_LIBRARIES}
|
||||
${GLIBC_COMPATIBILITY_LIBRARIES}
|
||||
${MEMCPY_LIBRARIES}
|
||||
)
|
||||
@ -1,243 +0,0 @@
|
||||
//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This is the entry point to the clang -cc1 functionality, which implements the
|
||||
// core compiler functionality along with a number of additional tools for
|
||||
// demonstration and testing purposes.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "llvm/Option/Arg.h"
|
||||
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
|
||||
#include "clang/Config/config.h"
|
||||
#include "clang/Driver/DriverDiagnostic.h"
|
||||
#include "clang/Driver/Options.h"
|
||||
#include "clang/Frontend/CompilerInstance.h"
|
||||
#include "clang/Frontend/CompilerInvocation.h"
|
||||
#include "clang/Frontend/FrontendDiagnostic.h"
|
||||
#include "clang/Frontend/TextDiagnosticBuffer.h"
|
||||
#include "clang/Frontend/TextDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/Utils.h"
|
||||
#include "clang/FrontendTool/Utils.h"
|
||||
#include "llvm/ADT/Statistic.h"
|
||||
#include "llvm/Config/llvm-config.h"
|
||||
#include "llvm/LinkAllPasses.h"
|
||||
#include "llvm/Option/ArgList.h"
|
||||
#include "llvm/Option/OptTable.h"
|
||||
#include "llvm/Support/Compiler.h"
|
||||
#include "llvm/Support/ErrorHandling.h"
|
||||
#include "llvm/Support/ManagedStatic.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
#include "llvm/Support/TargetSelect.h"
|
||||
#include "llvm/Support/Timer.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
#include <cstdio>
|
||||
|
||||
#ifdef CLANG_HAVE_RLIMITS
|
||||
#include <sys/resource.h>
|
||||
#endif
|
||||
|
||||
// have no .a version in packages
|
||||
#undef LINK_POLLY_INTO_TOOLS
|
||||
|
||||
using namespace clang;
|
||||
using namespace llvm::opt;
|
||||
|
||||
//===----------------------------------------------------------------------===//
|
||||
// Main driver
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
static void LLVMErrorHandler(void *UserData, const std::string &Message,
|
||||
bool GenCrashDiag) {
|
||||
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
|
||||
|
||||
Diags.Report(diag::err_fe_error_backend) << Message;
|
||||
|
||||
// Run the interrupt handlers to make sure any special cleanups get done, in
|
||||
// particular that we remove files registered with RemoveFileOnSignal.
|
||||
llvm::sys::RunInterruptHandlers();
|
||||
|
||||
// We cannot recover from llvm errors. When reporting a fatal error, exit
|
||||
// with status 70 to generate crash diagnostics. For BSD systems this is
|
||||
// defined as an internal software error. Otherwise, exit with status 1.
|
||||
exit(GenCrashDiag ? 70 : 1);
|
||||
}
|
||||
|
||||
#ifdef LINK_POLLY_INTO_TOOLS
|
||||
namespace polly {
|
||||
void initializePollyPasses(llvm::PassRegistry &Registry);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef CLANG_HAVE_RLIMITS
|
||||
// The amount of stack we think is "sufficient". If less than this much is
|
||||
// available, we may be unable to reach our template instantiation depth
|
||||
// limit and other similar limits.
|
||||
// FIXME: Unify this with the stack we request when spawning a thread to build
|
||||
// a module.
|
||||
static const int kSufficientStack = 8 << 20;
|
||||
|
||||
#if defined(__linux__) && defined(__PIE__)
|
||||
static size_t getCurrentStackAllocation() {
|
||||
// If we can't compute the current stack usage, allow for 512K of command
|
||||
// line arguments and environment.
|
||||
size_t Usage = 512 * 1024;
|
||||
if (FILE *StatFile = fopen("/proc/self/stat", "r")) {
|
||||
// We assume that the stack extends from its current address to the end of
|
||||
// the environment space. In reality, there is another string literal (the
|
||||
// program name) after the environment, but this is close enough (we only
|
||||
// need to be within 100K or so).
|
||||
unsigned long StackPtr, EnvEnd;
|
||||
// Disable silly GCC -Wformat warning that complains about length
|
||||
// modifiers on ignored format specifiers. We want to retain these
|
||||
// for documentation purposes even though they have no effect.
|
||||
#if defined(__GNUC__) && !defined(__clang__)
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wformat"
|
||||
#endif
|
||||
if (fscanf(StatFile,
|
||||
"%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu "
|
||||
"%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu "
|
||||
"%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d "
|
||||
"%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d",
|
||||
&StackPtr, &EnvEnd) == 2) {
|
||||
#if defined(__GNUC__) && !defined(__clang__)
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd;
|
||||
}
|
||||
fclose(StatFile);
|
||||
}
|
||||
return Usage;
|
||||
}
|
||||
|
||||
#include <alloca.h>
|
||||
|
||||
LLVM_ATTRIBUTE_NOINLINE
|
||||
static void ensureStackAddressSpace(int ExtraChunks = 0) {
|
||||
// Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary
|
||||
// relatively close to the stack (they are only guaranteed to be 128MiB
|
||||
// apart). This results in crashes if we happen to heap-allocate more than
|
||||
// 128MiB before we reach our stack high-water mark.
|
||||
//
|
||||
// To avoid these crashes, ensure that we have sufficient virtual memory
|
||||
// pages allocated before we start running.
|
||||
size_t Curr = getCurrentStackAllocation();
|
||||
const int kTargetStack = kSufficientStack - 256 * 1024;
|
||||
if (Curr < kTargetStack) {
|
||||
volatile char *volatile Alloc =
|
||||
static_cast<volatile char *>(alloca(kTargetStack - Curr));
|
||||
Alloc[0] = 0;
|
||||
Alloc[kTargetStack - Curr - 1] = 0;
|
||||
}
|
||||
}
|
||||
#else
|
||||
static void ensureStackAddressSpace() {}
|
||||
#endif
|
||||
|
||||
/// Attempt to ensure that we have at least 8MiB of usable stack space.
|
||||
static void ensureSufficientStack() {
|
||||
struct rlimit rlim;
|
||||
if (getrlimit(RLIMIT_STACK, &rlim) != 0)
|
||||
return;
|
||||
|
||||
// Increase the soft stack limit to our desired level, if necessary and
|
||||
// possible.
|
||||
if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) {
|
||||
// Try to allocate sufficient stack.
|
||||
if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack)
|
||||
rlim.rlim_cur = kSufficientStack;
|
||||
else if (rlim.rlim_cur == rlim.rlim_max)
|
||||
return;
|
||||
else
|
||||
rlim.rlim_cur = rlim.rlim_max;
|
||||
|
||||
if (setrlimit(RLIMIT_STACK, &rlim) != 0 ||
|
||||
rlim.rlim_cur != kSufficientStack)
|
||||
return;
|
||||
}
|
||||
|
||||
// We should now have a stack of size at least kSufficientStack. Ensure
|
||||
// that we can actually use that much, if necessary.
|
||||
ensureStackAddressSpace();
|
||||
}
|
||||
#else
|
||||
static void ensureSufficientStack() {}
|
||||
#endif
|
||||
|
||||
int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
|
||||
ensureSufficientStack();
|
||||
|
||||
std::unique_ptr<CompilerInstance> Clang(new CompilerInstance());
|
||||
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
|
||||
|
||||
// Register the support for object-file-wrapped Clang modules.
|
||||
auto PCHOps = Clang->getPCHContainerOperations();
|
||||
PCHOps->registerWriter(llvm::make_unique<ObjectFilePCHContainerWriter>());
|
||||
PCHOps->registerReader(llvm::make_unique<ObjectFilePCHContainerReader>());
|
||||
|
||||
// Initialize targets first, so that --version shows registered targets.
|
||||
llvm::InitializeAllTargets();
|
||||
llvm::InitializeAllTargetMCs();
|
||||
llvm::InitializeAllAsmPrinters();
|
||||
llvm::InitializeAllAsmParsers();
|
||||
|
||||
#ifdef LINK_POLLY_INTO_TOOLS
|
||||
llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry();
|
||||
polly::initializePollyPasses(Registry);
|
||||
#endif
|
||||
|
||||
// Buffer diagnostics from argument parsing so that we can output them using a
|
||||
// well formed diagnostic object.
|
||||
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
|
||||
TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer;
|
||||
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer);
|
||||
bool Success = CompilerInvocation::CreateFromArgs(
|
||||
Clang->getInvocation(), Argv.begin(), Argv.end(), Diags);
|
||||
|
||||
// Infer the builtin include path if unspecified.
|
||||
if (Clang->getHeaderSearchOpts().UseBuiltinIncludes &&
|
||||
Clang->getHeaderSearchOpts().ResourceDir.empty())
|
||||
Clang->getHeaderSearchOpts().ResourceDir =
|
||||
CompilerInvocation::GetResourcesPath(Argv0, MainAddr);
|
||||
|
||||
// Create the actual diagnostics engine.
|
||||
Clang->createDiagnostics();
|
||||
if (!Clang->hasDiagnostics())
|
||||
return 1;
|
||||
|
||||
// Set an error handler, so that any LLVM backend diagnostics go through our
|
||||
// error handler.
|
||||
llvm::install_fatal_error_handler(LLVMErrorHandler,
|
||||
static_cast<void*>(&Clang->getDiagnostics()));
|
||||
|
||||
DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics());
|
||||
if (!Success)
|
||||
return 1;
|
||||
|
||||
// Execute the frontend actions.
|
||||
Success = ExecuteCompilerInvocation(Clang.get());
|
||||
|
||||
// If any timers were active but haven't been destroyed yet, print their
|
||||
// results now. This happens in -disable-free mode.
|
||||
llvm::TimerGroup::printAll(llvm::errs());
|
||||
|
||||
// Our error handler depends on the Diagnostics object, which we're
|
||||
// potentially about to delete. Uninstall the handler now so that any
|
||||
// later errors use the default handling behavior instead.
|
||||
llvm::remove_fatal_error_handler();
|
||||
|
||||
// When running with -disable-free, don't do any destruction or shutdown.
|
||||
if (Clang->getFrontendOpts().DisableFree) {
|
||||
BuryPointer(std::move(Clang));
|
||||
return !Success;
|
||||
}
|
||||
|
||||
return !Success;
|
||||
}
|
||||
@ -1,555 +0,0 @@
|
||||
//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This is the entry point to the clang -cc1as functionality, which implements
|
||||
// the direct interface to the LLVM MC based assembler.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "clang/Basic/Diagnostic.h"
|
||||
#include "clang/Basic/DiagnosticOptions.h"
|
||||
#include "clang/Driver/DriverDiagnostic.h"
|
||||
#include "clang/Driver/Options.h"
|
||||
#include "clang/Frontend/FrontendDiagnostic.h"
|
||||
#include "clang/Frontend/TextDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/Utils.h"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/ADT/StringSwitch.h"
|
||||
#include "llvm/ADT/Triple.h"
|
||||
#include "llvm/IR/DataLayout.h"
|
||||
#include "llvm/MC/MCAsmBackend.h"
|
||||
#include "llvm/MC/MCAsmInfo.h"
|
||||
#include "llvm/MC/MCCodeEmitter.h"
|
||||
#include "llvm/MC/MCContext.h"
|
||||
#include "llvm/MC/MCInstrInfo.h"
|
||||
#include "llvm/MC/MCObjectFileInfo.h"
|
||||
#include "llvm/MC/MCParser/MCAsmParser.h"
|
||||
#include "llvm/MC/MCParser/MCTargetAsmParser.h"
|
||||
#include "llvm/MC/MCRegisterInfo.h"
|
||||
#include "llvm/MC/MCStreamer.h"
|
||||
#include "llvm/MC/MCSubtargetInfo.h"
|
||||
#include "llvm/MC/MCTargetOptions.h"
|
||||
#include "llvm/Option/Arg.h"
|
||||
#include "llvm/Option/ArgList.h"
|
||||
#include "llvm/Option/OptTable.h"
|
||||
#include "llvm/Support/CommandLine.h"
|
||||
#include "llvm/Support/ErrorHandling.h"
|
||||
#include "llvm/Support/FileSystem.h"
|
||||
#include "llvm/Support/FormattedStream.h"
|
||||
#include "llvm/Support/Host.h"
|
||||
#include "llvm/Support/MemoryBuffer.h"
|
||||
#include "llvm/Support/Path.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
#include "llvm/Support/SourceMgr.h"
|
||||
#include "llvm/Support/TargetRegistry.h"
|
||||
#include "llvm/Support/TargetSelect.h"
|
||||
#include "llvm/Support/Timer.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
#include <memory>
|
||||
#include <system_error>
|
||||
using namespace clang;
|
||||
using namespace clang::driver;
|
||||
using namespace clang::driver::options;
|
||||
using namespace llvm;
|
||||
using namespace llvm::opt;
|
||||
|
||||
|
||||
namespace {
|
||||
|
||||
/// \brief Helper class for representing a single invocation of the assembler.
|
||||
struct AssemblerInvocation {
|
||||
/// @name Target Options
|
||||
/// @{
|
||||
|
||||
/// The name of the target triple to assemble for.
|
||||
std::string Triple;
|
||||
|
||||
/// If given, the name of the target CPU to determine which instructions
|
||||
/// are legal.
|
||||
std::string CPU;
|
||||
|
||||
/// The list of target specific features to enable or disable -- this should
|
||||
/// be a list of strings starting with '+' or '-'.
|
||||
std::vector<std::string> Features;
|
||||
|
||||
/// The list of symbol definitions.
|
||||
std::vector<std::string> SymbolDefs;
|
||||
|
||||
/// @}
|
||||
/// @name Language Options
|
||||
/// @{
|
||||
|
||||
std::vector<std::string> IncludePaths;
|
||||
unsigned NoInitialTextSection : 1;
|
||||
unsigned SaveTemporaryLabels : 1;
|
||||
unsigned GenDwarfForAssembly : 1;
|
||||
unsigned RelaxELFRelocations : 1;
|
||||
unsigned DwarfVersion;
|
||||
std::string DwarfDebugFlags;
|
||||
std::string DwarfDebugProducer;
|
||||
std::string DebugCompilationDir;
|
||||
llvm::DebugCompressionType CompressDebugSections =
|
||||
llvm::DebugCompressionType::None;
|
||||
std::string MainFileName;
|
||||
|
||||
/// @}
|
||||
/// @name Frontend Options
|
||||
/// @{
|
||||
|
||||
std::string InputFile;
|
||||
std::vector<std::string> LLVMArgs;
|
||||
std::string OutputPath;
|
||||
enum FileType {
|
||||
FT_Asm, ///< Assembly (.s) output, transliterate mode.
|
||||
FT_Null, ///< No output, for timing purposes.
|
||||
FT_Obj ///< Object file output.
|
||||
};
|
||||
FileType OutputType;
|
||||
unsigned ShowHelp : 1;
|
||||
unsigned ShowVersion : 1;
|
||||
|
||||
/// @}
|
||||
/// @name Transliterate Options
|
||||
/// @{
|
||||
|
||||
unsigned OutputAsmVariant;
|
||||
unsigned ShowEncoding : 1;
|
||||
unsigned ShowInst : 1;
|
||||
|
||||
/// @}
|
||||
/// @name Assembler Options
|
||||
/// @{
|
||||
|
||||
unsigned RelaxAll : 1;
|
||||
unsigned NoExecStack : 1;
|
||||
unsigned FatalWarnings : 1;
|
||||
unsigned IncrementalLinkerCompatible : 1;
|
||||
|
||||
/// The name of the relocation model to use.
|
||||
std::string RelocationModel;
|
||||
|
||||
/// @}
|
||||
|
||||
public:
|
||||
AssemblerInvocation() {
|
||||
Triple = "";
|
||||
NoInitialTextSection = 0;
|
||||
InputFile = "-";
|
||||
OutputPath = "-";
|
||||
OutputType = FT_Asm;
|
||||
OutputAsmVariant = 0;
|
||||
ShowInst = 0;
|
||||
ShowEncoding = 0;
|
||||
RelaxAll = 0;
|
||||
NoExecStack = 0;
|
||||
FatalWarnings = 0;
|
||||
IncrementalLinkerCompatible = 0;
|
||||
DwarfVersion = 0;
|
||||
}
|
||||
|
||||
static bool CreateFromArgs(AssemblerInvocation &Res,
|
||||
ArrayRef<const char *> Argv,
|
||||
DiagnosticsEngine &Diags);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
|
||||
ArrayRef<const char *> Argv,
|
||||
DiagnosticsEngine &Diags) {
|
||||
bool Success = true;
|
||||
|
||||
// Parse the arguments.
|
||||
std::unique_ptr<OptTable> OptTbl(createDriverOptTable());
|
||||
|
||||
const unsigned IncludedFlagsBitmask = options::CC1AsOption;
|
||||
unsigned MissingArgIndex, MissingArgCount;
|
||||
InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount,
|
||||
IncludedFlagsBitmask);
|
||||
|
||||
// Check for missing argument error.
|
||||
if (MissingArgCount) {
|
||||
Diags.Report(diag::err_drv_missing_argument)
|
||||
<< Args.getArgString(MissingArgIndex) << MissingArgCount;
|
||||
Success = false;
|
||||
}
|
||||
|
||||
// Issue errors on unknown arguments.
|
||||
for (const Arg *A : Args.filtered(OPT_UNKNOWN)) {
|
||||
auto ArgString = A->getAsString(Args);
|
||||
std::string Nearest;
|
||||
if (OptTbl->findNearest(ArgString, Nearest, IncludedFlagsBitmask) > 1)
|
||||
Diags.Report(diag::err_drv_unknown_argument) << ArgString;
|
||||
else
|
||||
Diags.Report(diag::err_drv_unknown_argument_with_suggestion)
|
||||
<< ArgString << Nearest;
|
||||
Success = false;
|
||||
}
|
||||
|
||||
// Construct the invocation.
|
||||
|
||||
// Target Options
|
||||
Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple));
|
||||
Opts.CPU = Args.getLastArgValue(OPT_target_cpu);
|
||||
Opts.Features = Args.getAllArgValues(OPT_target_feature);
|
||||
|
||||
// Use the default target triple if unspecified.
|
||||
if (Opts.Triple.empty())
|
||||
Opts.Triple = llvm::sys::getDefaultTargetTriple();
|
||||
|
||||
// Language Options
|
||||
Opts.IncludePaths = Args.getAllArgValues(OPT_I);
|
||||
Opts.NoInitialTextSection = Args.hasArg(OPT_n);
|
||||
Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels);
|
||||
// Any DebugInfoKind implies GenDwarfForAssembly.
|
||||
Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ);
|
||||
|
||||
if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections,
|
||||
OPT_compress_debug_sections_EQ)) {
|
||||
if (A->getOption().getID() == OPT_compress_debug_sections) {
|
||||
// TODO: be more clever about the compression type auto-detection
|
||||
Opts.CompressDebugSections = llvm::DebugCompressionType::GNU;
|
||||
} else {
|
||||
Opts.CompressDebugSections =
|
||||
llvm::StringSwitch<llvm::DebugCompressionType>(A->getValue())
|
||||
.Case("none", llvm::DebugCompressionType::None)
|
||||
.Case("zlib", llvm::DebugCompressionType::Z)
|
||||
.Case("zlib-gnu", llvm::DebugCompressionType::GNU)
|
||||
.Default(llvm::DebugCompressionType::None);
|
||||
}
|
||||
}
|
||||
|
||||
Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations);
|
||||
Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags);
|
||||
Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags);
|
||||
Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer);
|
||||
Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir);
|
||||
Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name);
|
||||
|
||||
// Frontend Options
|
||||
if (Args.hasArg(OPT_INPUT)) {
|
||||
bool First = true;
|
||||
for (const Arg *A : Args.filtered(OPT_INPUT)) {
|
||||
if (First) {
|
||||
Opts.InputFile = A->getValue();
|
||||
First = false;
|
||||
} else {
|
||||
Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args);
|
||||
Success = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
|
||||
Opts.OutputPath = Args.getLastArgValue(OPT_o);
|
||||
if (Arg *A = Args.getLastArg(OPT_filetype)) {
|
||||
StringRef Name = A->getValue();
|
||||
unsigned OutputType = StringSwitch<unsigned>(Name)
|
||||
.Case("asm", FT_Asm)
|
||||
.Case("null", FT_Null)
|
||||
.Case("obj", FT_Obj)
|
||||
.Default(~0U);
|
||||
if (OutputType == ~0U) {
|
||||
Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name;
|
||||
Success = false;
|
||||
} else
|
||||
Opts.OutputType = FileType(OutputType);
|
||||
}
|
||||
Opts.ShowHelp = Args.hasArg(OPT_help);
|
||||
Opts.ShowVersion = Args.hasArg(OPT_version);
|
||||
|
||||
// Transliterate Options
|
||||
Opts.OutputAsmVariant =
|
||||
getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags);
|
||||
Opts.ShowEncoding = Args.hasArg(OPT_show_encoding);
|
||||
Opts.ShowInst = Args.hasArg(OPT_show_inst);
|
||||
|
||||
// Assemble Options
|
||||
Opts.RelaxAll = Args.hasArg(OPT_mrelax_all);
|
||||
Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack);
|
||||
Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings);
|
||||
Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic");
|
||||
Opts.IncrementalLinkerCompatible =
|
||||
Args.hasArg(OPT_mincremental_linker_compatible);
|
||||
Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym);
|
||||
|
||||
return Success;
|
||||
}
|
||||
|
||||
static std::unique_ptr<raw_fd_ostream>
|
||||
getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags,
|
||||
bool Binary) {
|
||||
if (Opts.OutputPath.empty())
|
||||
Opts.OutputPath = "-";
|
||||
|
||||
// Make sure that the Out file gets unlinked from the disk if we get a
|
||||
// SIGINT.
|
||||
if (Opts.OutputPath != "-")
|
||||
sys::RemoveFileOnSignal(Opts.OutputPath);
|
||||
|
||||
std::error_code EC;
|
||||
auto Out = llvm::make_unique<raw_fd_ostream>(
|
||||
Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text));
|
||||
if (EC) {
|
||||
Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath
|
||||
<< EC.message();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return Out;
|
||||
}
|
||||
|
||||
static bool ExecuteAssembler(AssemblerInvocation &Opts,
|
||||
DiagnosticsEngine &Diags) {
|
||||
// Get the target specific parser.
|
||||
std::string Error;
|
||||
const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error);
|
||||
if (!TheTarget)
|
||||
return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
|
||||
|
||||
ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer =
|
||||
MemoryBuffer::getFileOrSTDIN(Opts.InputFile);
|
||||
|
||||
if (std::error_code EC = Buffer.getError()) {
|
||||
Error = EC.message();
|
||||
return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile;
|
||||
}
|
||||
|
||||
SourceMgr SrcMgr;
|
||||
|
||||
// Tell SrcMgr about this buffer, which is what the parser will pick up.
|
||||
SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc());
|
||||
|
||||
// Record the location of the include directories so that the lexer can find
|
||||
// it later.
|
||||
SrcMgr.setIncludeDirs(Opts.IncludePaths);
|
||||
|
||||
std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(Opts.Triple));
|
||||
assert(MRI && "Unable to create target register info!");
|
||||
|
||||
std::unique_ptr<MCAsmInfo> MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple));
|
||||
assert(MAI && "Unable to create target asm info!");
|
||||
|
||||
// Ensure MCAsmInfo initialization occurs before any use, otherwise sections
|
||||
// may be created with a combination of default and explicit settings.
|
||||
MAI->setCompressDebugSections(Opts.CompressDebugSections);
|
||||
|
||||
MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations);
|
||||
|
||||
bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj;
|
||||
std::unique_ptr<raw_fd_ostream> FDOS = getOutputStream(Opts, Diags, IsBinary);
|
||||
if (!FDOS)
|
||||
return true;
|
||||
|
||||
// FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and
|
||||
// MCObjectFileInfo needs a MCContext reference in order to initialize itself.
|
||||
std::unique_ptr<MCObjectFileInfo> MOFI(new MCObjectFileInfo());
|
||||
|
||||
MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr);
|
||||
|
||||
bool PIC = false;
|
||||
if (Opts.RelocationModel == "static") {
|
||||
PIC = false;
|
||||
} else if (Opts.RelocationModel == "pic") {
|
||||
PIC = true;
|
||||
} else {
|
||||
assert(Opts.RelocationModel == "dynamic-no-pic" &&
|
||||
"Invalid PIC model!");
|
||||
PIC = false;
|
||||
}
|
||||
|
||||
MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx);
|
||||
if (Opts.SaveTemporaryLabels)
|
||||
Ctx.setAllowTemporaryLabels(false);
|
||||
if (Opts.GenDwarfForAssembly)
|
||||
Ctx.setGenDwarfForAssembly(true);
|
||||
if (!Opts.DwarfDebugFlags.empty())
|
||||
Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags));
|
||||
if (!Opts.DwarfDebugProducer.empty())
|
||||
Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer));
|
||||
if (!Opts.DebugCompilationDir.empty())
|
||||
Ctx.setCompilationDir(Opts.DebugCompilationDir);
|
||||
if (!Opts.MainFileName.empty())
|
||||
Ctx.setMainFileName(StringRef(Opts.MainFileName));
|
||||
Ctx.setDwarfVersion(Opts.DwarfVersion);
|
||||
|
||||
// Build up the feature string from the target feature list.
|
||||
std::string FS;
|
||||
if (!Opts.Features.empty()) {
|
||||
FS = Opts.Features[0];
|
||||
for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i)
|
||||
FS += "," + Opts.Features[i];
|
||||
}
|
||||
|
||||
std::unique_ptr<MCStreamer> Str;
|
||||
|
||||
std::unique_ptr<MCInstrInfo> MCII(TheTarget->createMCInstrInfo());
|
||||
std::unique_ptr<MCSubtargetInfo> STI(
|
||||
TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS));
|
||||
|
||||
raw_pwrite_stream *Out = FDOS.get();
|
||||
std::unique_ptr<buffer_ostream> BOS;
|
||||
|
||||
// FIXME: There is a bit of code duplication with addPassesToEmitFile.
|
||||
if (Opts.OutputType == AssemblerInvocation::FT_Asm) {
|
||||
MCInstPrinter *IP = TheTarget->createMCInstPrinter(
|
||||
llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI);
|
||||
|
||||
std::unique_ptr<MCCodeEmitter> CE;
|
||||
if (Opts.ShowEncoding)
|
||||
CE.reset(TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx));
|
||||
MCTargetOptions MCOptions;
|
||||
std::unique_ptr<MCAsmBackend> MAB(
|
||||
TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
|
||||
|
||||
auto FOut = llvm::make_unique<formatted_raw_ostream>(*Out);
|
||||
Str.reset(TheTarget->createAsmStreamer(
|
||||
Ctx, std::move(FOut), /*asmverbose*/ true,
|
||||
/*useDwarfDirectory*/ true, IP, std::move(CE), std::move(MAB),
|
||||
Opts.ShowInst));
|
||||
} else if (Opts.OutputType == AssemblerInvocation::FT_Null) {
|
||||
Str.reset(createNullStreamer(Ctx));
|
||||
} else {
|
||||
assert(Opts.OutputType == AssemblerInvocation::FT_Obj &&
|
||||
"Invalid file type!");
|
||||
if (!FDOS->supportsSeeking()) {
|
||||
BOS = make_unique<buffer_ostream>(*FDOS);
|
||||
Out = BOS.get();
|
||||
}
|
||||
|
||||
std::unique_ptr<MCCodeEmitter> CE(
|
||||
TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx));
|
||||
MCTargetOptions MCOptions;
|
||||
std::unique_ptr<MCAsmBackend> MAB(
|
||||
TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
|
||||
|
||||
Triple T(Opts.Triple);
|
||||
Str.reset(TheTarget->createMCObjectStreamer(
|
||||
T, Ctx, std::move(MAB), *Out, std::move(CE), *STI, Opts.RelaxAll,
|
||||
Opts.IncrementalLinkerCompatible,
|
||||
/*DWARFMustBeAtTheEnd*/ true));
|
||||
Str.get()->InitSections(Opts.NoExecStack);
|
||||
}
|
||||
|
||||
// Assembly to object compilation should leverage assembly info.
|
||||
Str->setUseAssemblerInfoForParsing(true);
|
||||
|
||||
bool Failed = false;
|
||||
|
||||
std::unique_ptr<MCAsmParser> Parser(
|
||||
createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI));
|
||||
|
||||
// FIXME: init MCTargetOptions from sanitizer flags here.
|
||||
MCTargetOptions Options;
|
||||
std::unique_ptr<MCTargetAsmParser> TAP(
|
||||
TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options));
|
||||
if (!TAP)
|
||||
Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
|
||||
|
||||
// Set values for symbols, if any.
|
||||
for (auto &S : Opts.SymbolDefs) {
|
||||
auto Pair = StringRef(S).split('=');
|
||||
auto Sym = Pair.first;
|
||||
auto Val = Pair.second;
|
||||
int64_t Value;
|
||||
// We have already error checked this in the driver.
|
||||
Val.getAsInteger(0, Value);
|
||||
Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value);
|
||||
}
|
||||
|
||||
if (!Failed) {
|
||||
Parser->setTargetParser(*TAP.get());
|
||||
Failed = Parser->Run(Opts.NoInitialTextSection);
|
||||
}
|
||||
|
||||
// Close Streamer first.
|
||||
// It might have a reference to the output stream.
|
||||
Str.reset();
|
||||
// Close the output stream early.
|
||||
BOS.reset();
|
||||
FDOS.reset();
|
||||
|
||||
// Delete output file if there were errors.
|
||||
if (Failed && Opts.OutputPath != "-")
|
||||
sys::fs::remove(Opts.OutputPath);
|
||||
|
||||
return Failed;
|
||||
}
|
||||
|
||||
static void LLVMErrorHandler(void *UserData, const std::string &Message,
|
||||
bool /*GenCrashDiag*/) {
|
||||
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
|
||||
|
||||
Diags.Report(diag::err_fe_error_backend) << Message;
|
||||
|
||||
// We cannot recover from llvm errors.
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int cc1as_main(ArrayRef<const char *> Argv, const char */*Argv0*/, void */*MainAddr*/) {
|
||||
// Initialize targets and assembly printers/parsers.
|
||||
InitializeAllTargetInfos();
|
||||
InitializeAllTargetMCs();
|
||||
InitializeAllAsmParsers();
|
||||
|
||||
// Construct our diagnostic client.
|
||||
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
|
||||
TextDiagnosticPrinter *DiagClient
|
||||
= new TextDiagnosticPrinter(errs(), &*DiagOpts);
|
||||
DiagClient->setPrefix("clang -cc1as");
|
||||
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
|
||||
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
|
||||
|
||||
// Set an error handler, so that any LLVM backend diagnostics go through our
|
||||
// error handler.
|
||||
ScopedFatalErrorHandler FatalErrorHandler
|
||||
(LLVMErrorHandler, static_cast<void*>(&Diags));
|
||||
|
||||
// Parse the arguments.
|
||||
AssemblerInvocation Asm;
|
||||
if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags))
|
||||
return 1;
|
||||
|
||||
if (Asm.ShowHelp) {
|
||||
std::unique_ptr<OptTable> Opts(driver::createDriverOptTable());
|
||||
Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler",
|
||||
/*Include=*/driver::options::CC1AsOption, /*Exclude=*/0,
|
||||
/*ShowAllAliases=*/false);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Honor -version.
|
||||
//
|
||||
// FIXME: Use a better -version message?
|
||||
if (Asm.ShowVersion) {
|
||||
llvm::cl::PrintVersionMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Honor -mllvm.
|
||||
//
|
||||
// FIXME: Remove this, one day.
|
||||
if (!Asm.LLVMArgs.empty()) {
|
||||
unsigned NumArgs = Asm.LLVMArgs.size();
|
||||
auto Args = llvm::make_unique<const char*[]>(NumArgs + 2);
|
||||
Args[0] = "clang (LLVM option parsing)";
|
||||
for (unsigned i = 0; i != NumArgs; ++i)
|
||||
Args[i + 1] = Asm.LLVMArgs[i].c_str();
|
||||
Args[NumArgs + 1] = nullptr;
|
||||
llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get());
|
||||
}
|
||||
|
||||
// Execute the invocation, unless there were parsing errors.
|
||||
bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags);
|
||||
|
||||
// If any timers were active but haven't been destroyed yet, print their
|
||||
// results now.
|
||||
TimerGroup::printAll(errs());
|
||||
|
||||
return !!Failed;
|
||||
}
|
||||
@ -1,512 +0,0 @@
|
||||
//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This is the entry point to the clang driver; it is a thin wrapper
|
||||
// for functionality in the Driver clang library.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "clang/Driver/Driver.h"
|
||||
#include "clang/Basic/DiagnosticOptions.h"
|
||||
#include "clang/Driver/Compilation.h"
|
||||
#include "clang/Driver/DriverDiagnostic.h"
|
||||
#include "clang/Driver/Options.h"
|
||||
#include "clang/Driver/ToolChain.h"
|
||||
#include "clang/Frontend/ChainedDiagnosticConsumer.h"
|
||||
#include "clang/Frontend/CompilerInvocation.h"
|
||||
#include "clang/Frontend/SerializedDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/TextDiagnosticPrinter.h"
|
||||
#include "clang/Frontend/Utils.h"
|
||||
#include "llvm/ADT/ArrayRef.h"
|
||||
#include "llvm/ADT/SmallString.h"
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
#include "llvm/Option/ArgList.h"
|
||||
#include "llvm/Option/OptTable.h"
|
||||
#include "llvm/Option/Option.h"
|
||||
#include "llvm/Support/CommandLine.h"
|
||||
#include "llvm/Support/ErrorHandling.h"
|
||||
#include "llvm/Support/FileSystem.h"
|
||||
#include "llvm/Support/Host.h"
|
||||
#include "llvm/Support/InitLLVM.h"
|
||||
#include "llvm/Support/Path.h"
|
||||
#include "llvm/Support/Process.h"
|
||||
#include "llvm/Support/Program.h"
|
||||
#include "llvm/Support/Regex.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
#include "llvm/Support/StringSaver.h"
|
||||
#include "llvm/Support/TargetSelect.h"
|
||||
#include "llvm/Support/Timer.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <system_error>
|
||||
using namespace clang;
|
||||
using namespace clang::driver;
|
||||
using namespace llvm::opt;
|
||||
|
||||
std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) {
|
||||
if (!CanonicalPrefixes) {
|
||||
SmallString<128> ExecutablePath(Argv0);
|
||||
// Do a PATH lookup if Argv0 isn't a valid path.
|
||||
if (!llvm::sys::fs::exists(ExecutablePath))
|
||||
if (llvm::ErrorOr<std::string> P =
|
||||
llvm::sys::findProgramByName(ExecutablePath))
|
||||
ExecutablePath = *P;
|
||||
return ExecutablePath.str();
|
||||
}
|
||||
|
||||
// This just needs to be some symbol in the binary; C++ doesn't
|
||||
// allow taking the address of ::main however.
|
||||
void *P = (void*) (intptr_t) GetExecutablePath;
|
||||
return llvm::sys::fs::getMainExecutable(Argv0, P);
|
||||
}
|
||||
|
||||
static const char *GetStableCStr(std::set<std::string> &SavedStrings,
|
||||
StringRef S) {
|
||||
return SavedStrings.insert(S).first->c_str();
|
||||
}
|
||||
|
||||
/// ApplyQAOverride - Apply a list of edits to the input argument lists.
|
||||
///
|
||||
/// The input string is a space separate list of edits to perform,
|
||||
/// they are applied in order to the input argument lists. Edits
|
||||
/// should be one of the following forms:
|
||||
///
|
||||
/// '#': Silence information about the changes to the command line arguments.
|
||||
///
|
||||
/// '^': Add FOO as a new argument at the beginning of the command line.
|
||||
///
|
||||
/// '+': Add FOO as a new argument at the end of the command line.
|
||||
///
|
||||
/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command
|
||||
/// line.
|
||||
///
|
||||
/// 'xOPTION': Removes all instances of the literal argument OPTION.
|
||||
///
|
||||
/// 'XOPTION': Removes all instances of the literal argument OPTION,
|
||||
/// and the following argument.
|
||||
///
|
||||
/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox'
|
||||
/// at the end of the command line.
|
||||
///
|
||||
/// \param OS - The stream to write edit information to.
|
||||
/// \param Args - The vector of command line arguments.
|
||||
/// \param Edit - The override command to perform.
|
||||
/// \param SavedStrings - Set to use for storing string representations.
|
||||
static void ApplyOneQAOverride(raw_ostream &OS,
|
||||
SmallVectorImpl<const char*> &Args,
|
||||
StringRef Edit,
|
||||
std::set<std::string> &SavedStrings) {
|
||||
// This does not need to be efficient.
|
||||
|
||||
if (Edit[0] == '^') {
|
||||
const char *Str =
|
||||
GetStableCStr(SavedStrings, Edit.substr(1));
|
||||
OS << "### Adding argument " << Str << " at beginning\n";
|
||||
Args.insert(Args.begin() + 1, Str);
|
||||
} else if (Edit[0] == '+') {
|
||||
const char *Str =
|
||||
GetStableCStr(SavedStrings, Edit.substr(1));
|
||||
OS << "### Adding argument " << Str << " at end\n";
|
||||
Args.push_back(Str);
|
||||
} else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") &&
|
||||
Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) {
|
||||
StringRef MatchPattern = Edit.substr(2).split('/').first;
|
||||
StringRef ReplPattern = Edit.substr(2).split('/').second;
|
||||
ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1);
|
||||
|
||||
for (unsigned i = 1, e = Args.size(); i != e; ++i) {
|
||||
// Ignore end-of-line response file markers
|
||||
if (Args[i] == nullptr)
|
||||
continue;
|
||||
std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]);
|
||||
|
||||
if (Repl != Args[i]) {
|
||||
OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n";
|
||||
Args[i] = GetStableCStr(SavedStrings, Repl);
|
||||
}
|
||||
}
|
||||
} else if (Edit[0] == 'x' || Edit[0] == 'X') {
|
||||
auto Option = Edit.substr(1);
|
||||
for (unsigned i = 1; i < Args.size();) {
|
||||
if (Option == Args[i]) {
|
||||
OS << "### Deleting argument " << Args[i] << '\n';
|
||||
Args.erase(Args.begin() + i);
|
||||
if (Edit[0] == 'X') {
|
||||
if (i < Args.size()) {
|
||||
OS << "### Deleting argument " << Args[i] << '\n';
|
||||
Args.erase(Args.begin() + i);
|
||||
} else
|
||||
OS << "### Invalid X edit, end of command line!\n";
|
||||
}
|
||||
} else
|
||||
++i;
|
||||
}
|
||||
} else if (Edit[0] == 'O') {
|
||||
for (unsigned i = 1; i < Args.size();) {
|
||||
const char *A = Args[i];
|
||||
// Ignore end-of-line response file markers
|
||||
if (A == nullptr)
|
||||
continue;
|
||||
if (A[0] == '-' && A[1] == 'O' &&
|
||||
(A[2] == '\0' ||
|
||||
(A[3] == '\0' && (A[2] == 's' || A[2] == 'z' ||
|
||||
('0' <= A[2] && A[2] <= '9'))))) {
|
||||
OS << "### Deleting argument " << Args[i] << '\n';
|
||||
Args.erase(Args.begin() + i);
|
||||
} else
|
||||
++i;
|
||||
}
|
||||
OS << "### Adding argument " << Edit << " at end\n";
|
||||
Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str()));
|
||||
} else {
|
||||
OS << "### Unrecognized edit: " << Edit << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
/// ApplyQAOverride - Apply a comma separate list of edits to the
|
||||
/// input argument lists. See ApplyOneQAOverride.
|
||||
static void ApplyQAOverride(SmallVectorImpl<const char*> &Args,
|
||||
const char *OverrideStr,
|
||||
std::set<std::string> &SavedStrings) {
|
||||
raw_ostream *OS = &llvm::errs();
|
||||
|
||||
if (OverrideStr[0] == '#') {
|
||||
++OverrideStr;
|
||||
OS = &llvm::nulls();
|
||||
}
|
||||
|
||||
*OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n";
|
||||
|
||||
// This does not need to be efficient.
|
||||
|
||||
const char *S = OverrideStr;
|
||||
while (*S) {
|
||||
const char *End = ::strchr(S, ' ');
|
||||
if (!End)
|
||||
End = S + strlen(S);
|
||||
if (End != S)
|
||||
ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings);
|
||||
S = End;
|
||||
if (*S != '\0')
|
||||
++S;
|
||||
}
|
||||
}
|
||||
|
||||
extern int cc1_main(ArrayRef<const char *> Argv, const char *Argv0,
|
||||
void *MainAddr);
|
||||
extern int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0,
|
||||
void *MainAddr);
|
||||
extern int cc1gen_reproducer_main(ArrayRef<const char *> Argv,
|
||||
const char *Argv0, void *MainAddr);
|
||||
|
||||
static void insertTargetAndModeArgs(const ParsedClangName &NameParts,
|
||||
SmallVectorImpl<const char *> &ArgVector,
|
||||
std::set<std::string> &SavedStrings) {
|
||||
// Put target and mode arguments at the start of argument list so that
|
||||
// arguments specified in command line could override them. Avoid putting
|
||||
// them at index 0, as an option like '-cc1' must remain the first.
|
||||
int InsertionPoint = 0;
|
||||
if (ArgVector.size() > 0)
|
||||
++InsertionPoint;
|
||||
|
||||
if (NameParts.DriverMode) {
|
||||
// Add the mode flag to the arguments.
|
||||
ArgVector.insert(ArgVector.begin() + InsertionPoint,
|
||||
GetStableCStr(SavedStrings, NameParts.DriverMode));
|
||||
}
|
||||
|
||||
if (NameParts.TargetIsValid) {
|
||||
const char *arr[] = {"-target", GetStableCStr(SavedStrings,
|
||||
NameParts.TargetPrefix)};
|
||||
ArgVector.insert(ArgVector.begin() + InsertionPoint,
|
||||
std::begin(arr), std::end(arr));
|
||||
}
|
||||
}
|
||||
|
||||
static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver,
|
||||
SmallVectorImpl<const char *> &Opts) {
|
||||
llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts);
|
||||
// The first instance of '#' should be replaced with '=' in each option.
|
||||
for (const char *Opt : Opts)
|
||||
if (char *NumberSignPtr = const_cast<char *>(::strchr(Opt, '#')))
|
||||
*NumberSignPtr = '=';
|
||||
}
|
||||
|
||||
static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) {
|
||||
// Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE.
|
||||
TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS");
|
||||
if (TheDriver.CCPrintOptions)
|
||||
TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE");
|
||||
|
||||
// Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE.
|
||||
TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS");
|
||||
if (TheDriver.CCPrintHeaders)
|
||||
TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE");
|
||||
|
||||
// Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE.
|
||||
TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS");
|
||||
if (TheDriver.CCLogDiagnostics)
|
||||
TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE");
|
||||
}
|
||||
|
||||
static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient,
|
||||
const std::string &Path) {
|
||||
// If the clang binary happens to be named cl.exe for compatibility reasons,
|
||||
// use clang-cl.exe as the prefix to avoid confusion between clang and MSVC.
|
||||
StringRef ExeBasename(llvm::sys::path::filename(Path));
|
||||
if (ExeBasename.equals_lower("cl.exe"))
|
||||
ExeBasename = "clang-cl.exe";
|
||||
DiagClient->setPrefix(ExeBasename);
|
||||
}
|
||||
|
||||
// This lets us create the DiagnosticsEngine with a properly-filled-out
|
||||
// DiagnosticOptions instance.
|
||||
static DiagnosticOptions *
|
||||
CreateAndPopulateDiagOpts(ArrayRef<const char *> argv) {
|
||||
auto *DiagOpts = new DiagnosticOptions;
|
||||
std::unique_ptr<OptTable> Opts(createDriverOptTable());
|
||||
unsigned MissingArgIndex, MissingArgCount;
|
||||
InputArgList Args =
|
||||
Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount);
|
||||
// We ignore MissingArgCount and the return value of ParseDiagnosticArgs.
|
||||
// Any errors that would be diagnosed here will also be diagnosed later,
|
||||
// when the DiagnosticsEngine actually exists.
|
||||
(void)ParseDiagnosticArgs(*DiagOpts, Args);
|
||||
return DiagOpts;
|
||||
}
|
||||
|
||||
static void SetInstallDir(SmallVectorImpl<const char *> &argv,
|
||||
Driver &TheDriver, bool CanonicalPrefixes) {
|
||||
// Attempt to find the original path used to invoke the driver, to determine
|
||||
// the installed path. We do this manually, because we want to support that
|
||||
// path being a symlink.
|
||||
SmallString<128> InstalledPath(argv[0]);
|
||||
|
||||
// Do a PATH lookup, if there are no directory components.
|
||||
if (llvm::sys::path::filename(InstalledPath) == InstalledPath)
|
||||
if (llvm::ErrorOr<std::string> Tmp = llvm::sys::findProgramByName(
|
||||
llvm::sys::path::filename(InstalledPath.str())))
|
||||
InstalledPath = *Tmp;
|
||||
|
||||
// FIXME: We don't actually canonicalize this, we just make it absolute.
|
||||
if (CanonicalPrefixes)
|
||||
llvm::sys::fs::make_absolute(InstalledPath);
|
||||
|
||||
StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath));
|
||||
if (llvm::sys::fs::exists(InstalledPathParent))
|
||||
TheDriver.setInstalledDir(InstalledPathParent);
|
||||
}
|
||||
|
||||
static int ExecuteCC1Tool(ArrayRef<const char *> argv, StringRef Tool) {
|
||||
void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath;
|
||||
if (Tool == "")
|
||||
return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP);
|
||||
if (Tool == "as")
|
||||
return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP);
|
||||
|
||||
// Reject unknown tools.
|
||||
llvm::errs() << "error: unknown integrated tool '" << Tool << "'. "
|
||||
<< "Valid tools include '-cc1' and '-cc1as'.\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
int mainEntryClickHouseClang(int argc_, char **argv_) {
|
||||
llvm::InitLLVM X(argc_, argv_);
|
||||
SmallVector<const char *, 256> argv(argv_, argv_ + argc_);
|
||||
|
||||
if (llvm::sys::Process::FixupStandardFileDescriptors())
|
||||
return 1;
|
||||
|
||||
llvm::InitializeAllTargets();
|
||||
auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]);
|
||||
|
||||
llvm::BumpPtrAllocator A;
|
||||
llvm::StringSaver Saver(A);
|
||||
|
||||
// Parse response files using the GNU syntax, unless we're in CL mode. There
|
||||
// are two ways to put clang in CL compatibility mode: argv[0] is either
|
||||
// clang-cl or cl, or --driver-mode=cl is on the command line. The normal
|
||||
// command line parsing can't happen until after response file parsing, so we
|
||||
// have to manually search for a --driver-mode=cl argument the hard way.
|
||||
// Finally, our -cc1 tools don't care which tokenization mode we use because
|
||||
// response files written by clang will tokenize the same way in either mode.
|
||||
bool ClangCLMode = false;
|
||||
if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") ||
|
||||
std::find_if(argv.begin(), argv.end(), [](const char *F) {
|
||||
return F && strcmp(F, "--driver-mode=cl") == 0;
|
||||
}) != argv.end()) {
|
||||
ClangCLMode = true;
|
||||
}
|
||||
enum { Default, POSIX, Windows } RSPQuoting = Default;
|
||||
for (const char *F : argv) {
|
||||
if (strcmp(F, "--rsp-quoting=posix") == 0)
|
||||
RSPQuoting = POSIX;
|
||||
else if (strcmp(F, "--rsp-quoting=windows") == 0)
|
||||
RSPQuoting = Windows;
|
||||
}
|
||||
|
||||
// Determines whether we want nullptr markers in argv to indicate response
|
||||
// files end-of-lines. We only use this for the /LINK driver argument with
|
||||
// clang-cl.exe on Windows.
|
||||
bool MarkEOLs = ClangCLMode;
|
||||
|
||||
llvm::cl::TokenizerCallback Tokenizer;
|
||||
if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode))
|
||||
Tokenizer = &llvm::cl::TokenizeWindowsCommandLine;
|
||||
else
|
||||
Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
|
||||
|
||||
if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1"))
|
||||
MarkEOLs = false;
|
||||
llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs);
|
||||
|
||||
// Handle -cc1 integrated tools, even if -cc1 was expanded from a response
|
||||
// file.
|
||||
auto FirstArg = std::find_if(argv.begin() + 1, argv.end(),
|
||||
[](const char *A) { return A != nullptr; });
|
||||
if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) {
|
||||
// If -cc1 came from a response file, remove the EOL sentinels.
|
||||
if (MarkEOLs) {
|
||||
auto newEnd = std::remove(argv.begin(), argv.end(), nullptr);
|
||||
argv.resize(newEnd - argv.begin());
|
||||
}
|
||||
return ExecuteCC1Tool(argv, argv[1] + 4);
|
||||
}
|
||||
|
||||
bool CanonicalPrefixes = true;
|
||||
for (int i = 1, size = argv.size(); i < size; ++i) {
|
||||
// Skip end-of-line response file markers
|
||||
if (argv[i] == nullptr)
|
||||
continue;
|
||||
if (StringRef(argv[i]) == "-no-canonical-prefixes") {
|
||||
CanonicalPrefixes = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle CL and _CL_ which permits additional command line options to be
|
||||
// prepended or appended.
|
||||
if (ClangCLMode) {
|
||||
// Arguments in "CL" are prepended.
|
||||
llvm::Optional<std::string> OptCL = llvm::sys::Process::GetEnv("CL");
|
||||
if (OptCL.hasValue()) {
|
||||
SmallVector<const char *, 8> PrependedOpts;
|
||||
getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts);
|
||||
|
||||
// Insert right after the program name to prepend to the argument list.
|
||||
argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end());
|
||||
}
|
||||
// Arguments in "_CL_" are appended.
|
||||
llvm::Optional<std::string> Opt_CL_ = llvm::sys::Process::GetEnv("_CL_");
|
||||
if (Opt_CL_.hasValue()) {
|
||||
SmallVector<const char *, 8> AppendedOpts;
|
||||
getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts);
|
||||
|
||||
// Insert at the end of the argument list to append.
|
||||
argv.append(AppendedOpts.begin(), AppendedOpts.end());
|
||||
}
|
||||
}
|
||||
|
||||
std::set<std::string> SavedStrings;
|
||||
// Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the
|
||||
// scenes.
|
||||
if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) {
|
||||
// FIXME: Driver shouldn't take extra initial argument.
|
||||
ApplyQAOverride(argv, OverrideStr, SavedStrings);
|
||||
}
|
||||
|
||||
std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes);
|
||||
|
||||
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts =
|
||||
CreateAndPopulateDiagOpts(argv);
|
||||
|
||||
TextDiagnosticPrinter *DiagClient
|
||||
= new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
|
||||
FixupDiagPrefixExeName(DiagClient, Path);
|
||||
|
||||
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
|
||||
|
||||
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
|
||||
|
||||
if (!DiagOpts->DiagnosticSerializationFile.empty()) {
|
||||
auto SerializedConsumer =
|
||||
clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile,
|
||||
&*DiagOpts, /*MergeChildRecords=*/true);
|
||||
Diags.setClient(new ChainedDiagnosticConsumer(
|
||||
Diags.takeClient(), std::move(SerializedConsumer)));
|
||||
}
|
||||
|
||||
ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false);
|
||||
|
||||
Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags);
|
||||
SetInstallDir(argv, TheDriver, CanonicalPrefixes);
|
||||
TheDriver.setTargetAndMode(TargetAndMode);
|
||||
|
||||
insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings);
|
||||
|
||||
SetBackdoorDriverOutputsFromEnvVars(TheDriver);
|
||||
|
||||
std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(argv));
|
||||
int Res = 1;
|
||||
if (C && !C->containsError()) {
|
||||
SmallVector<std::pair<int, const Command *>, 4> FailingCommands;
|
||||
Res = TheDriver.ExecuteCompilation(*C, FailingCommands);
|
||||
|
||||
// Force a crash to test the diagnostics.
|
||||
if (TheDriver.GenReproducer) {
|
||||
Diags.Report(diag::err_drv_force_crash)
|
||||
<< !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH");
|
||||
|
||||
// Pretend that every command failed.
|
||||
FailingCommands.clear();
|
||||
for (const auto &J : C->getJobs())
|
||||
if (const Command *C = dyn_cast<Command>(&J))
|
||||
FailingCommands.push_back(std::make_pair(-1, C));
|
||||
}
|
||||
|
||||
for (const auto &P : FailingCommands) {
|
||||
int CommandRes = P.first;
|
||||
const Command *FailingCommand = P.second;
|
||||
if (!Res)
|
||||
Res = CommandRes;
|
||||
|
||||
// If result status is < 0, then the driver command signalled an error.
|
||||
// If result status is 70, then the driver command reported a fatal error.
|
||||
// On Windows, abort will return an exit code of 3. In these cases,
|
||||
// generate additional diagnostic information if possible.
|
||||
bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70;
|
||||
#ifdef _WIN32
|
||||
DiagnoseCrash |= CommandRes == 3;
|
||||
#endif
|
||||
if (DiagnoseCrash) {
|
||||
TheDriver.generateCompilationDiagnostics(*C, *FailingCommand);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Diags.getClient()->finish();
|
||||
|
||||
// If any timers were active but haven't been destroyed yet, print their
|
||||
// results now. This happens in -disable-free mode.
|
||||
llvm::TimerGroup::printAll(llvm::errs());
|
||||
|
||||
#ifdef _WIN32
|
||||
// Exit status should not be negative on Win32, unless abnormal termination.
|
||||
// Once abnormal termiation was caught, negative status should not be
|
||||
// propagated.
|
||||
if (Res < 0)
|
||||
Res = 1;
|
||||
#endif
|
||||
|
||||
// If we have multiple failing commands, we return the result of the first
|
||||
// failing command.
|
||||
return Res;
|
||||
}
|
||||
@ -1,10 +0,0 @@
|
||||
#include "lld/Common/Driver.h"
|
||||
#include "llvm/Support/InitLLVM.h"
|
||||
#include <vector>
|
||||
|
||||
int mainEntryClickHouseLLD(int argc, char ** argv)
|
||||
{
|
||||
llvm::InitLLVM X(argc, argv);
|
||||
std::vector<const char *> args(argv, argv + argc);
|
||||
return !lld::elf::link(args, false);
|
||||
}
|
||||
@ -1 +0,0 @@
|
||||
Compiler-7.0.0
|
||||
@ -1 +0,0 @@
|
||||
Compiler-7.0.0
|
||||
@ -1,2 +0,0 @@
|
||||
int mainEntryClickHouseClang(int argc, char ** argv);
|
||||
int main(int argc_, char ** argv_) { return mainEntryClickHouseClang(argc_, argv_); }
|
||||
@ -1,2 +0,0 @@
|
||||
int mainEntryClickHouseLLD(int argc, char ** argv);
|
||||
int main(int argc_, char ** argv_) { return mainEntryClickHouseLLD(argc_, argv_); }
|
||||
@ -1,100 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
#set -x
|
||||
#echo "Args: $*"; env | sort
|
||||
|
||||
# Этот скрипт собирает все заголовочные файлы, нужные для компиляции некоторого translation unit-а
|
||||
# и копирует их с сохранением путей в директорию DST.
|
||||
# Это затем может быть использовано, чтобы скомпилировать translation unit на другом сервере,
|
||||
# используя ровно такой же набор заголовочных файлов.
|
||||
#
|
||||
# Требуется clang, желательно наиболее свежий (trunk).
|
||||
#
|
||||
# Используется при сборке пакетов.
|
||||
# Заголовочные файлы записываются в пакет clickhouse-common, в директорию /usr/share/clickhouse/headers.
|
||||
#
|
||||
# Если вы хотите установить их самостоятельно, без сборки пакета,
|
||||
# чтобы clickhouse-server видел их там, где ожидается, выполните:
|
||||
#
|
||||
# sudo ./copy_headers.sh . /usr/share/clickhouse/headers/
|
||||
|
||||
SOURCE_PATH=${1:-../../..}
|
||||
DST=${2:-$SOURCE_PATH/../headers}
|
||||
BUILD_PATH=${BUILD_PATH=${3:-$SOURCE_PATH/build}}
|
||||
|
||||
PATH="/usr/local/bin:/usr/local/sbin:/usr/bin:$PATH"
|
||||
|
||||
if [[ -z $CLANG ]]; then
|
||||
CLANG="clang"
|
||||
fi
|
||||
|
||||
START_HEADERS=$(echo \
|
||||
$BUILD_PATH/dbms/src/Common/config_version.h \
|
||||
$SOURCE_PATH/dbms/src/Interpreters/SpecializedAggregator.h \
|
||||
$SOURCE_PATH/dbms/src/AggregateFunctions/AggregateFunction*.h)
|
||||
|
||||
for header in $START_HEADERS; do
|
||||
START_HEADERS_INCLUDE+="-include $header "
|
||||
done
|
||||
|
||||
|
||||
GCC_ROOT=`$CLANG -v 2>&1 | grep "Selected GCC installation"| sed -n -e 's/^.*: //p'`
|
||||
|
||||
# TODO: Does not work on macos?
|
||||
GCC_ROOT=${GCC_ROOT:=/usr/lib/clang/${CMAKE_CXX_COMPILER_VERSION}}
|
||||
|
||||
# Опция -mcx16 для того, чтобы выбиралось больше заголовочных файлов (с запасом).
|
||||
# The latter options are the same that are added while building packages.
|
||||
for src_file in $(echo | $CLANG -M -xc++ -std=c++1z -Wall -Werror -msse2 -msse4 -mcx16 -mpopcnt -O3 -g -fPIC -fstack-protector -D_FORTIFY_SOURCE=2 \
|
||||
-I $GCC_ROOT/include \
|
||||
-I $GCC_ROOT/include-fixed \
|
||||
$(cat "$BUILD_PATH/include_directories.txt") \
|
||||
$START_HEADERS_INCLUDE \
|
||||
- |
|
||||
tr -d '\\' |
|
||||
sed -E -e 's/^-\.o://');
|
||||
do
|
||||
dst_file=$src_file;
|
||||
[ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!")
|
||||
[ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!")
|
||||
dst_file=$(echo $dst_file | sed -E -e 's/build\///') # for simplicity reasons, will put generated headers near the rest.
|
||||
mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')";
|
||||
cp "$src_file" "$DST/$dst_file";
|
||||
done
|
||||
|
||||
|
||||
# Копируем больше заголовочных файлов с интринсиками, так как на серверах, куда будут устанавливаться
|
||||
# заголовочные файлы, будет использоваться опция -march=native.
|
||||
|
||||
for src_file in $(ls -1 $($CLANG -v -xc++ - <<<'' 2>&1 | grep '^ /' | grep 'include' | grep -E '/lib/clang/|/include/clang/')/*.h | grep -vE 'arm|altivec|Intrin');
|
||||
do
|
||||
dst_file=$src_file;
|
||||
[ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!")
|
||||
[ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!")
|
||||
mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')";
|
||||
cp "$src_file" "$DST/$dst_file";
|
||||
done
|
||||
|
||||
if [ -d "$SOURCE_PATH/contrib/boost/libs/smart_ptr/include/boost/smart_ptr/detail" ]; then
|
||||
# Even more platform-specific headers
|
||||
for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/libs/smart_ptr/include/boost/smart_ptr/detail/*);
|
||||
do
|
||||
dst_file=$src_file;
|
||||
[ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!")
|
||||
[ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!")
|
||||
mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')";
|
||||
cp "$src_file" "$DST/$dst_file";
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -d "$SOURCE_PATH/contrib/boost/boost/smart_ptr/detail" ]; then
|
||||
for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/boost/smart_ptr/detail/*);
|
||||
do
|
||||
dst_file=$src_file;
|
||||
[ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!")
|
||||
[ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!")
|
||||
mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')";
|
||||
cp "$src_file" "$DST/$dst_file";
|
||||
done
|
||||
fi
|
||||
@ -56,11 +56,6 @@ int mainEntryClickHouseObfuscator(int argc, char ** argv);
|
||||
#endif
|
||||
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
int mainEntryClickHouseClang(int argc, char ** argv);
|
||||
int mainEntryClickHouseLLD(int argc, char ** argv);
|
||||
#endif
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
@ -100,12 +95,6 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
|
||||
#if ENABLE_CLICKHOUSE_OBFUSCATOR || !defined(ENABLE_CLICKHOUSE_OBFUSCATOR)
|
||||
{"obfuscator", mainEntryClickHouseObfuscator},
|
||||
#endif
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
{"clang", mainEntryClickHouseClang},
|
||||
{"clang++", mainEntryClickHouseClang},
|
||||
{"lld", mainEntryClickHouseLLD},
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
@ -152,11 +141,6 @@ int main(int argc_, char ** argv_)
|
||||
/// will work only after additional call of this function.
|
||||
updatePHDRCache();
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
if (argc_ >= 2 && 0 == strcmp(argv_[1], "-cc1"))
|
||||
return mainEntryClickHouseClang(argc_, argv_);
|
||||
#endif
|
||||
|
||||
#if USE_TCMALLOC
|
||||
/** Without this option, tcmalloc returns memory to OS too frequently for medium-sized memory allocations
|
||||
* (like IO buffers, column vectors, hash tables, etc.),
|
||||
|
||||
@ -97,7 +97,7 @@ void PerformanceTestInfo::applySettings(XMLConfigurationPtr config)
|
||||
}
|
||||
|
||||
extractSettings(config, "settings", config_settings, settings_to_apply);
|
||||
settings.applyChanges(settings_to_apply);
|
||||
settings.loadFromChanges(settings_to_apply);
|
||||
|
||||
if (settings_contain("average_rows_speed_precision"))
|
||||
TestStats::avg_rows_speed_precision =
|
||||
|
||||
@ -182,11 +182,11 @@ void TCPHandler::runImpl()
|
||||
/// Should we send internal logs to client?
|
||||
const auto client_logs_level = query_context->getSettingsRef().send_logs_level;
|
||||
if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_LOGS
|
||||
&& client_logs_level.value != LogsLevel::none)
|
||||
&& client_logs_level != LogsLevel::none)
|
||||
{
|
||||
state.logs_queue = std::make_shared<InternalTextLogsQueue>();
|
||||
state.logs_queue->max_priority = Poco::Logger::parseLevel(client_logs_level.toString());
|
||||
CurrentThread::attachInternalTextLogsQueue(state.logs_queue, client_logs_level.value);
|
||||
CurrentThread::attachInternalTextLogsQueue(state.logs_queue, client_logs_level);
|
||||
}
|
||||
|
||||
query_context->setExternalTablesInitializer([&connection_settings, this] (Context & context)
|
||||
@ -329,7 +329,7 @@ void TCPHandler::readData(const Settings & connection_settings)
|
||||
const auto receive_timeout = query_context->getSettingsRef().receive_timeout.value;
|
||||
|
||||
/// Poll interval should not be greater than receive_timeout
|
||||
const size_t default_poll_interval = connection_settings.poll_interval.value * 1000000;
|
||||
const size_t default_poll_interval = connection_settings.poll_interval * 1000000;
|
||||
size_t current_poll_interval = static_cast<size_t>(receive_timeout.totalMicroseconds());
|
||||
constexpr size_t min_poll_interval = 5000; // 5 ms
|
||||
size_t poll_interval = std::max(min_poll_interval, std::min(default_poll_interval, current_poll_interval));
|
||||
|
||||
@ -176,7 +176,7 @@ struct QuantileExactExclusive : public QuantileExact<Value>
|
||||
}
|
||||
};
|
||||
|
||||
/// QuantileExactInclusive is equivalent to Excel PERCENTILE and PERCENTILE.INC, R-7, SciPy-(1,1)
|
||||
/// QuantileExactInclusive is equivalent to Excel PERCENTILE and PERCENTILE.INC, R-7, SciPy-(1,1)
|
||||
template <typename Value>
|
||||
struct QuantileExactInclusive : public QuantileExact<Value>
|
||||
{
|
||||
|
||||
@ -51,12 +51,6 @@ MemoryTracker * CurrentThread::getMemoryTracker()
|
||||
return ¤t_thread->memory_tracker;
|
||||
}
|
||||
|
||||
Int64 & CurrentThread::getUntrackedMemory()
|
||||
{
|
||||
/// It assumes that (current_thread != nullptr) is already checked with getMemoryTracker()
|
||||
return current_thread->untracked_memory;
|
||||
}
|
||||
|
||||
void CurrentThread::updateProgressIn(const Progress & value)
|
||||
{
|
||||
if (unlikely(!current_thread))
|
||||
|
||||
@ -52,7 +52,12 @@ public:
|
||||
|
||||
static ProfileEvents::Counters & getProfileEvents();
|
||||
static MemoryTracker * getMemoryTracker();
|
||||
static Int64 & getUntrackedMemory();
|
||||
|
||||
static inline Int64 & getUntrackedMemory()
|
||||
{
|
||||
/// It assumes that (current_thread != nullptr) is already checked with getMemoryTracker()
|
||||
return current_thread->untracked_memory;
|
||||
}
|
||||
|
||||
/// Update read and write rows (bytes) statistics (used in system.query_thread_log)
|
||||
static void updateProgressIn(const Progress & value);
|
||||
|
||||
@ -445,6 +445,8 @@ namespace ErrorCodes
|
||||
extern const int CANNOT_PTHREAD_ATTR = 468;
|
||||
extern const int VIOLATED_CONSTRAINT = 469;
|
||||
extern const int QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW = 470;
|
||||
extern const int SETTINGS_ARE_NOT_SUPPORTED = 471;
|
||||
extern const int IMMUTABLE_SETTING = 472;
|
||||
|
||||
extern const int KEEPER_EXCEPTION = 999;
|
||||
extern const int POCO_EXCEPTION = 1000;
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/PODArray.h>
|
||||
#include <Common/OptimizedRegularExpression.h>
|
||||
|
||||
#define MIN_LENGTH_FOR_STRSTR 3
|
||||
@ -413,9 +414,9 @@ unsigned OptimizedRegularExpressionImpl<thread_safe>::match(const char * subject
|
||||
return 0;
|
||||
}
|
||||
|
||||
StringPieceType pieces[MAX_SUBPATTERNS];
|
||||
DB::PODArrayWithStackMemory<StringPieceType, sizeof(StringPieceType) * (MAX_SUBPATTERNS+1)> pieces(limit);
|
||||
|
||||
if (!re2->Match(StringPieceType(subject, subject_size), 0, subject_size, RegexType::UNANCHORED, pieces, limit))
|
||||
if (!re2->Match(StringPieceType(subject, subject_size), 0, subject_size, RegexType::UNANCHORED, pieces.data(), pieces.size()))
|
||||
return 0;
|
||||
else
|
||||
{
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/UTF8Helpers.h>
|
||||
#include <Core/Defines.h>
|
||||
#include <ext/range.h>
|
||||
@ -23,6 +25,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int UNSUPPORTED_PARAMETER;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
|
||||
@ -157,7 +160,7 @@ public:
|
||||
#endif
|
||||
}
|
||||
|
||||
ALWAYS_INLINE bool compare(const UInt8 * pos) const
|
||||
ALWAYS_INLINE bool compare(const UInt8 * /*haystack*/, const UInt8 * /*haystack_end*/, const UInt8 * pos) const
|
||||
{
|
||||
static const Poco::UTF8Encoding utf8;
|
||||
|
||||
@ -374,7 +377,7 @@ public:
|
||||
#endif
|
||||
}
|
||||
|
||||
ALWAYS_INLINE bool compare(const UInt8 * pos) const
|
||||
ALWAYS_INLINE bool compare(const UInt8 * /*haystack*/, const UInt8 * /*haystack_end*/, const UInt8 * pos) const
|
||||
{
|
||||
#ifdef __SSE4_1__
|
||||
if (pageSafe(pos))
|
||||
@ -567,7 +570,7 @@ public:
|
||||
#endif
|
||||
}
|
||||
|
||||
ALWAYS_INLINE bool compare(const UInt8 * pos) const
|
||||
ALWAYS_INLINE bool compare(const UInt8 * /*haystack*/, const UInt8 * /*haystack_end*/, const UInt8 * pos) const
|
||||
{
|
||||
#ifdef __SSE4_1__
|
||||
if (pageSafe(pos))
|
||||
@ -697,11 +700,81 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
// Searches for needle surrounded by token-separators.
|
||||
// Separators are anything inside ASCII (0-128) and not alphanum.
|
||||
// Any value outside of basic ASCII (>=128) is considered a non-separator symbol, hence UTF-8 strings
|
||||
// should work just fine. But any Unicode whitespace is not considered a token separtor.
|
||||
template <typename StringSearcher>
|
||||
class TokenSearcher
|
||||
{
|
||||
StringSearcher searcher;
|
||||
size_t needle_size;
|
||||
|
||||
public:
|
||||
TokenSearcher(const char * const needle_, const size_t needle_size_)
|
||||
: searcher{needle_, needle_size_},
|
||||
needle_size(needle_size_)
|
||||
{
|
||||
if (std::any_of(reinterpret_cast<const UInt8 *>(needle_), reinterpret_cast<const UInt8 *>(needle_) + needle_size_, isTokenSeparator))
|
||||
{
|
||||
throw Exception{"Needle must not contain whitespace or separator characters", ErrorCodes::BAD_ARGUMENTS};
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ALWAYS_INLINE bool compare(const UInt8 * haystack, const UInt8 * haystack_end, const UInt8 * pos) const
|
||||
{
|
||||
// use searcher only if pos is in the beginning of token and pos + searcher.needle_size is end of token.
|
||||
if (isToken(haystack, haystack_end, pos))
|
||||
return searcher.compare(haystack, haystack_end, pos);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
const UInt8 * search(const UInt8 * haystack, const UInt8 * const haystack_end) const
|
||||
{
|
||||
// use searcher.search(), then verify that returned value is a token
|
||||
// if it is not, skip it and re-run
|
||||
|
||||
const UInt8 * pos = haystack;
|
||||
while (pos < haystack_end)
|
||||
{
|
||||
pos = searcher.search(pos, haystack_end);
|
||||
if (pos == haystack_end || isToken(haystack, haystack_end, pos))
|
||||
return pos;
|
||||
|
||||
// assuming that heendle does not contain any token separators.
|
||||
pos += needle_size;
|
||||
}
|
||||
return haystack_end;
|
||||
}
|
||||
|
||||
const UInt8 * search(const UInt8 * haystack, const size_t haystack_size) const
|
||||
{
|
||||
return search(haystack, haystack + haystack_size);
|
||||
}
|
||||
|
||||
ALWAYS_INLINE bool isToken(const UInt8 * haystack, const UInt8 * const haystack_end, const UInt8* p) const
|
||||
{
|
||||
return (p == haystack || isTokenSeparator(*(p - 1)))
|
||||
&& (p + needle_size >= haystack_end || isTokenSeparator(*(p + needle_size)));
|
||||
}
|
||||
|
||||
ALWAYS_INLINE static bool isTokenSeparator(const UInt8 c)
|
||||
{
|
||||
if (isAlphaNumericASCII(c) || !isASCII(c))
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
using ASCIICaseSensitiveStringSearcher = StringSearcher<true, true>;
|
||||
using ASCIICaseInsensitiveStringSearcher = StringSearcher<false, true>;
|
||||
using UTF8CaseSensitiveStringSearcher = StringSearcher<true, false>;
|
||||
using UTF8CaseInsensitiveStringSearcher = StringSearcher<false, false>;
|
||||
using ASCIICaseSensitiveTokenSearcher = TokenSearcher<ASCIICaseSensitiveStringSearcher>;
|
||||
|
||||
|
||||
/** Uses functions from libc.
|
||||
|
||||
@ -327,6 +327,8 @@ protected:
|
||||
FallbackSearcher fallback_searcher;
|
||||
|
||||
public:
|
||||
using Searcher = FallbackSearcher;
|
||||
|
||||
/** haystack_size_hint - the expected total size of the haystack for `search` calls. Optional (zero means unspecified).
|
||||
* If you specify it small enough, the fallback algorithm will be used,
|
||||
* since it is considered that it's useless to waste time initializing the hash table.
|
||||
@ -373,7 +375,7 @@ public:
|
||||
const auto res = pos - (hash[cell_num] - 1);
|
||||
|
||||
/// pointer in the code is always padded array so we can use pagesafe semantics
|
||||
if (fallback_searcher.compare(res))
|
||||
if (fallback_searcher.compare(haystack, haystack_end, res))
|
||||
return res;
|
||||
}
|
||||
}
|
||||
@ -520,7 +522,7 @@ public:
|
||||
{
|
||||
const auto res = pos - (hash[cell_num].off - 1);
|
||||
const size_t ind = hash[cell_num].id;
|
||||
if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(res))
|
||||
if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res))
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -552,7 +554,7 @@ public:
|
||||
{
|
||||
const auto res = pos - (hash[cell_num].off - 1);
|
||||
const size_t ind = hash[cell_num].id;
|
||||
if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(res))
|
||||
if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res))
|
||||
ans = std::min(ans, ind);
|
||||
}
|
||||
}
|
||||
@ -590,7 +592,7 @@ public:
|
||||
{
|
||||
const auto res = pos - (hash[cell_num].off - 1);
|
||||
const size_t ind = hash[cell_num].id;
|
||||
if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(res))
|
||||
if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res))
|
||||
ans = std::min<UInt64>(ans, res - haystack);
|
||||
}
|
||||
}
|
||||
@ -625,7 +627,7 @@ public:
|
||||
{
|
||||
const auto * res = pos - (hash[cell_num].off - 1);
|
||||
const size_t ind = hash[cell_num].id;
|
||||
if (ans[ind] == 0 && res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(res))
|
||||
if (ans[ind] == 0 && res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res))
|
||||
ans[ind] = count_chars(haystack, res);
|
||||
}
|
||||
}
|
||||
@ -650,6 +652,8 @@ using VolnitskyUTF8 = VolnitskyBase<true, false, ASCIICaseSensitiveStringSearche
|
||||
using VolnitskyCaseInsensitive = VolnitskyBase<false, true, ASCIICaseInsensitiveStringSearcher>; /// ignores non-ASCII bytes
|
||||
using VolnitskyCaseInsensitiveUTF8 = VolnitskyBase<false, false, UTF8CaseInsensitiveStringSearcher>;
|
||||
|
||||
using VolnitskyToken = VolnitskyBase<true, true, ASCIICaseSensitiveTokenSearcher>;
|
||||
|
||||
using MultiVolnitsky = MultiVolnitskyBase<true, true, ASCIICaseSensitiveStringSearcher>;
|
||||
using MultiVolnitskyUTF8 = MultiVolnitskyBase<true, false, ASCIICaseSensitiveStringSearcher>;
|
||||
using MultiVolnitskyCaseInsensitive = MultiVolnitskyBase<false, true, ASCIICaseInsensitiveStringSearcher>;
|
||||
|
||||
@ -42,7 +42,8 @@ struct Settings : public SettingsCollection<Settings>
|
||||
* but we are not going to do it, because settings is used everywhere as static struct fields.
|
||||
*/
|
||||
|
||||
#define LIST_OF_SETTINGS(M) \
|
||||
/// M (mutable) for normal settings, IM (immutable) for not updateable settings.
|
||||
#define LIST_OF_SETTINGS(M, IM) \
|
||||
M(SettingUInt64, min_compress_block_size, 65536, "The actual size of the block to compress, if the uncompressed data less than max_compress_block_size is no less than this value and no less than the volume of data for one mark.") \
|
||||
M(SettingUInt64, max_compress_block_size, 1048576, "The maximum size of blocks of uncompressed data before compressing for writing to a table.") \
|
||||
M(SettingUInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size for reading") \
|
||||
@ -85,7 +86,6 @@ struct Settings : public SettingsCollection<Settings>
|
||||
M(SettingTotalsMode, totals_mode, TotalsMode::AFTER_HAVING_EXCLUSIVE, "How to calculate TOTALS when HAVING is present, as well as when max_rows_to_group_by and group_by_overflow_mode = ‘any’ are present.") \
|
||||
M(SettingFloat, totals_auto_threshold, 0.5, "The threshold for totals_mode = 'auto'.") \
|
||||
\
|
||||
M(SettingBool, compile, false, "Whether query compilation is enabled.") \
|
||||
M(SettingBool, allow_suspicious_low_cardinality_types, false, "In CREATE TABLE statement allows specifying LowCardinality modifier for types of small fixed size (8 or less). Enabling this may increase merge times and memory consumption.") \
|
||||
M(SettingBool, compile_expressions, false, "Compile some scalar functions and operators to native code.") \
|
||||
M(SettingUInt64, min_count_to_compile, 3, "The number of structurally identical queries before they are compiled.") \
|
||||
@ -351,6 +351,7 @@ struct Settings : public SettingsCollection<Settings>
|
||||
/** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
|
||||
\
|
||||
M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13") \
|
||||
M(SettingBool, compile, false, "Whether query compilation is enabled. Will be removed after 2020-03-13") \
|
||||
|
||||
DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS)
|
||||
|
||||
|
||||
@ -17,6 +17,10 @@ class Field;
|
||||
class ReadBuffer;
|
||||
class WriteBuffer;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int IMMUTABLE_SETTING;
|
||||
}
|
||||
|
||||
/** One setting for any type.
|
||||
* Stores a value within itself, as well as a flag - whether the value was changed.
|
||||
@ -304,6 +308,7 @@ private:
|
||||
Derived & castToDerived() { return *static_cast<Derived *>(this); }
|
||||
const Derived & castToDerived() const { return *static_cast<const Derived *>(this); }
|
||||
|
||||
using IsChangedFunction = bool (*)(const Derived &);
|
||||
using GetStringFunction = String (*)(const Derived &);
|
||||
using GetFieldFunction = Field (*)(const Derived &);
|
||||
using SetStringFunction = void (*)(Derived &, const String &);
|
||||
@ -314,9 +319,13 @@ private:
|
||||
|
||||
struct MemberInfo
|
||||
{
|
||||
size_t offset_of_changed;
|
||||
IsChangedFunction is_changed;
|
||||
StringRef name;
|
||||
StringRef description;
|
||||
/// Can be updated after first load for config/definition.
|
||||
/// Non updatable settings can be `changed`,
|
||||
/// if they were overwritten in config/definition.
|
||||
const bool updateable;
|
||||
GetStringFunction get_string;
|
||||
GetFieldFunction get_field;
|
||||
SetStringFunction set_string;
|
||||
@ -325,7 +334,7 @@ private:
|
||||
DeserializeFunction deserialize;
|
||||
CastValueWithoutApplyingFunction cast_value_without_applying;
|
||||
|
||||
bool isChanged(const Derived & collection) const { return *reinterpret_cast<const bool*>(reinterpret_cast<const UInt8*>(&collection) + offset_of_changed); }
|
||||
bool isChanged(const Derived & collection) const { return is_changed(collection); }
|
||||
};
|
||||
|
||||
class MemberInfos
|
||||
@ -396,6 +405,7 @@ public:
|
||||
const_reference(const const_reference & src) = default;
|
||||
const StringRef & getName() const { return member->name; }
|
||||
const StringRef & getDescription() const { return member->description; }
|
||||
bool isUpdateable() const { return member->updateable; }
|
||||
bool isChanged() const { return member->isChanged(*collection); }
|
||||
Field getValue() const { return member->get_field(*collection); }
|
||||
String getValueAsString() const { return member->get_string(*collection); }
|
||||
@ -415,6 +425,18 @@ public:
|
||||
reference(const const_reference & src) : const_reference(src) {}
|
||||
void setValue(const Field & value) { this->member->set_field(*const_cast<Derived *>(this->collection), value); }
|
||||
void setValue(const String & value) { this->member->set_string(*const_cast<Derived *>(this->collection), value); }
|
||||
void updateValue(const Field & value)
|
||||
{
|
||||
if (!this->member->updateable)
|
||||
throw Exception("Setting '" + this->member->name.toString() + "' is restricted for updates.", ErrorCodes::IMMUTABLE_SETTING);
|
||||
setValue(value);
|
||||
}
|
||||
void updateValue(const String & value)
|
||||
{
|
||||
if (!this->member->updateable)
|
||||
throw Exception("Setting '" + this->member->name.toString() + "' is restricted for updates.", ErrorCodes::IMMUTABLE_SETTING);
|
||||
setValue(value);
|
||||
}
|
||||
};
|
||||
|
||||
/// Iterator to iterating through all the settings.
|
||||
@ -497,6 +519,15 @@ public:
|
||||
void set(size_t index, const String & value) { (*this)[index].setValue(value); }
|
||||
void set(const String & name, const String & value) { (*this)[name].setValue(value); }
|
||||
|
||||
/// Updates setting's value. Checks it' mutability.
|
||||
void update(size_t index, const Field & value) { (*this)[index].updateValue(value); }
|
||||
|
||||
void update(const String & name, const Field & value) { (*this)[name].updateValue(value); }
|
||||
|
||||
void update(size_t index, const String & value) { (*this)[index].updateValue(value); }
|
||||
|
||||
void update(const String & name, const String & value) { (*this)[name].updateValue(value); }
|
||||
|
||||
/// Returns value of a setting.
|
||||
Field get(size_t index) const { return (*this)[index].getValue(); }
|
||||
Field get(const String & name) const { return (*this)[name].getValue(); }
|
||||
@ -560,18 +591,35 @@ public:
|
||||
return found_changes;
|
||||
}
|
||||
|
||||
/// Applies changes to the settings.
|
||||
void applyChange(const SettingChange & change)
|
||||
/// Applies change to the settings. Doesn't check settings mutability.
|
||||
void loadFromChange(const SettingChange & change)
|
||||
{
|
||||
set(change.name, change.value);
|
||||
}
|
||||
|
||||
void applyChanges(const SettingsChanges & changes)
|
||||
/// Applies changes to the settings. Should be used in initial settings loading.
|
||||
/// (on table creation or loading from config)
|
||||
void loadFromChanges(const SettingsChanges & changes)
|
||||
{
|
||||
for (const SettingChange & change : changes)
|
||||
applyChange(change);
|
||||
loadFromChange(change);
|
||||
}
|
||||
|
||||
/// Applies change to the settings, checks settings mutability.
|
||||
void updateFromChange(const SettingChange & change)
|
||||
{
|
||||
update(change.name, change.value);
|
||||
}
|
||||
|
||||
/// Applies changes to the settings. Should be used for settigns update.
|
||||
/// (ALTER MODIFY SETTINGS)
|
||||
void updateFromChanges(const SettingsChanges & changes)
|
||||
{
|
||||
for (const SettingChange & change : changes)
|
||||
updateFromChange(change);
|
||||
}
|
||||
|
||||
|
||||
void copyChangesFrom(const Derived & src)
|
||||
{
|
||||
for (const auto & member : members())
|
||||
@ -615,7 +663,7 @@ public:
|
||||
};
|
||||
|
||||
#define DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS_MACRO) \
|
||||
LIST_OF_SETTINGS_MACRO(DECLARE_SETTINGS_COLLECTION_DECLARE_VARIABLES_HELPER_)
|
||||
LIST_OF_SETTINGS_MACRO(DECLARE_SETTINGS_COLLECTION_DECLARE_VARIABLES_HELPER_, DECLARE_SETTINGS_COLLECTION_DECLARE_VARIABLES_HELPER_)
|
||||
|
||||
|
||||
#define IMPLEMENT_SETTINGS_COLLECTION(DERIVED_CLASS_NAME, LIST_OF_SETTINGS_MACRO) \
|
||||
@ -625,9 +673,9 @@ public:
|
||||
using Derived = DERIVED_CLASS_NAME; \
|
||||
struct Functions \
|
||||
{ \
|
||||
LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_) \
|
||||
LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_, IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_) \
|
||||
}; \
|
||||
LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_ADD_MEMBER_INFO_HELPER_) \
|
||||
LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_, IMPLEMENT_SETTINGS_COLLECTION_ADD_IMMUTABLE_MEMBER_INFO_HELPER_) \
|
||||
}
|
||||
|
||||
|
||||
@ -645,13 +693,19 @@ public:
|
||||
static Field NAME##_castValueWithoutApplying(const Field & value) { TYPE temp{DEFAULT}; temp.set(value); return temp.toField(); }
|
||||
|
||||
|
||||
#define IMPLEMENT_SETTINGS_COLLECTION_ADD_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \
|
||||
static_assert(std::is_same_v<decltype(std::declval<Derived>().NAME.changed), bool>); \
|
||||
add({offsetof(Derived, NAME.changed), \
|
||||
StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), \
|
||||
#define IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \
|
||||
add({[](const Derived & d) { return d.NAME.changed; }, \
|
||||
StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), true, \
|
||||
&Functions::NAME##_getString, &Functions::NAME##_getField, \
|
||||
&Functions::NAME##_setString, &Functions::NAME##_setField, \
|
||||
&Functions::NAME##_serialize, &Functions::NAME##_deserialize, \
|
||||
&Functions::NAME##_castValueWithoutApplying });
|
||||
|
||||
#define IMPLEMENT_SETTINGS_COLLECTION_ADD_IMMUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \
|
||||
add({[](const Derived & d) { return d.NAME.changed; }, \
|
||||
StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), false, \
|
||||
&Functions::NAME##_getString, &Functions::NAME##_getField, \
|
||||
&Functions::NAME##_setString, &Functions::NAME##_setField, \
|
||||
&Functions::NAME##_serialize, &Functions::NAME##_deserialize, \
|
||||
&Functions::NAME##_castValueWithoutApplying });
|
||||
}
|
||||
|
||||
@ -234,7 +234,7 @@ void DataTypeEnum<Type>::deserializeBinaryBulk(
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
void DataTypeEnum<Type>::serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf, size_t & value_index) const
|
||||
void DataTypeEnum<Type>::serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf, size_t & value_index) const
|
||||
{
|
||||
if (value_index)
|
||||
return;
|
||||
|
||||
@ -434,6 +434,74 @@ struct MultiSearchFirstIndexImpl
|
||||
}
|
||||
};
|
||||
|
||||
/** Token search the string, means that needle must be surrounded by some separator chars, like whitespace or puctuation.
|
||||
*/
|
||||
template <bool negate_result = false>
|
||||
struct HasTokenImpl
|
||||
{
|
||||
using ResultType = UInt8;
|
||||
|
||||
static void vector_constant(
|
||||
const ColumnString::Chars & data, const ColumnString::Offsets & offsets, const std::string & pattern, PaddedPODArray<UInt8> & res)
|
||||
{
|
||||
if (offsets.empty())
|
||||
return;
|
||||
|
||||
const UInt8 * begin = data.data();
|
||||
const UInt8 * pos = begin;
|
||||
const UInt8 * end = pos + data.size();
|
||||
|
||||
/// The current index in the array of strings.
|
||||
size_t i = 0;
|
||||
|
||||
VolnitskyToken searcher(pattern.data(), pattern.size(), end - pos);
|
||||
|
||||
/// We will search for the next occurrence in all rows at once.
|
||||
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
|
||||
{
|
||||
/// Let's determine which index it refers to.
|
||||
while (begin + offsets[i] <= pos)
|
||||
{
|
||||
res[i] = negate_result;
|
||||
++i;
|
||||
}
|
||||
|
||||
/// We check that the entry does not pass through the boundaries of strings.
|
||||
if (pos + pattern.size() < begin + offsets[i])
|
||||
res[i] = !negate_result;
|
||||
else
|
||||
res[i] = negate_result;
|
||||
|
||||
pos = begin + offsets[i];
|
||||
++i;
|
||||
}
|
||||
|
||||
/// Tail, in which there can be no substring.
|
||||
if (i < res.size())
|
||||
memset(&res[i], negate_result, (res.size() - i) * sizeof(res[0]));
|
||||
}
|
||||
|
||||
static void constant_constant(const std::string & data, const std::string & pattern, UInt8 & res)
|
||||
{
|
||||
VolnitskyToken searcher(pattern.data(), pattern.size(), data.size());
|
||||
const auto found = searcher.search(data.c_str(), data.size()) != data.end().base();
|
||||
res = negate_result ^ found;
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
static void vector_vector(Args &&...)
|
||||
{
|
||||
throw Exception("Function 'hasToken' does not support non-constant needle argument", ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
|
||||
/// Search different needles in single haystack.
|
||||
template <typename... Args>
|
||||
static void constant_vector(Args &&...)
|
||||
{
|
||||
throw Exception("Function 'hasToken' does not support non-constant needle argument", ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
struct NamePosition
|
||||
{
|
||||
@ -516,6 +584,11 @@ struct NameMultiSearchFirstPositionCaseInsensitiveUTF8
|
||||
static constexpr auto name = "multiSearchFirstPositionCaseInsensitiveUTF8";
|
||||
};
|
||||
|
||||
struct NameHasToken
|
||||
{
|
||||
static constexpr auto name = "hasToken";
|
||||
};
|
||||
|
||||
|
||||
using FunctionPosition = FunctionsStringSearch<PositionImpl<PositionCaseSensitiveASCII>, NamePosition>;
|
||||
using FunctionPositionUTF8 = FunctionsStringSearch<PositionImpl<PositionCaseSensitiveUTF8>, NamePositionUTF8>;
|
||||
@ -542,6 +615,7 @@ using FunctionMultiSearchFirstPositionUTF8 = FunctionsMultiStringSearch<MultiSea
|
||||
using FunctionMultiSearchFirstPositionCaseInsensitive = FunctionsMultiStringSearch<MultiSearchFirstPositionImpl<PositionCaseInsensitiveASCII>, NameMultiSearchFirstPositionCaseInsensitive>;
|
||||
using FunctionMultiSearchFirstPositionCaseInsensitiveUTF8 = FunctionsMultiStringSearch<MultiSearchFirstPositionImpl<PositionCaseInsensitiveUTF8>, NameMultiSearchFirstPositionCaseInsensitiveUTF8>;
|
||||
|
||||
using FunctionHasToken = FunctionsStringSearch<HasTokenImpl<false>, NameHasToken>;
|
||||
|
||||
void registerFunctionsStringSearch(FunctionFactory & factory)
|
||||
{
|
||||
@ -570,6 +644,8 @@ void registerFunctionsStringSearch(FunctionFactory & factory)
|
||||
factory.registerFunction<FunctionMultiSearchFirstPositionCaseInsensitive>();
|
||||
factory.registerFunction<FunctionMultiSearchFirstPositionCaseInsensitiveUTF8>();
|
||||
|
||||
factory.registerFunction<FunctionHasToken>();
|
||||
|
||||
factory.registerAlias("locate", NamePosition::name, FunctionFactory::CaseInsensitive);
|
||||
}
|
||||
}
|
||||
|
||||
@ -18,7 +18,7 @@ const UInt8 geohash_base32_decode_lookup_table[256] = {
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 10, 11, 12, 13, 14, 15, 16, 0xFF, 17, 18, 0xFF, 19, 20, 0xFF,
|
||||
|
||||
@ -1,10 +1,8 @@
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/FunctionStringToString.h>
|
||||
#include <common/find_symbols.h>
|
||||
|
||||
#ifdef __SSE4_2__
|
||||
#include <nmmintrin.h>
|
||||
#endif
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -60,7 +58,7 @@ public:
|
||||
execute(reinterpret_cast<const UInt8 *>(&data[prev_offset]), offsets[i] - prev_offset - 1, start, length);
|
||||
|
||||
res_data.resize(res_data.size() + length + 1);
|
||||
memcpy(&res_data[res_offset], start, length);
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], start, length);
|
||||
res_offset += length + 1;
|
||||
res_data[res_offset - 1] = '\0';
|
||||
|
||||
@ -77,59 +75,27 @@ public:
|
||||
private:
|
||||
static void execute(const UInt8 * data, size_t size, const UInt8 *& res_data, size_t & res_size)
|
||||
{
|
||||
size_t chars_to_trim_left = 0;
|
||||
size_t chars_to_trim_right = 0;
|
||||
char whitespace = ' ';
|
||||
#ifdef __SSE4_2__
|
||||
const auto bytes_sse = sizeof(__m128i);
|
||||
const auto size_sse = size - (size % bytes_sse);
|
||||
const auto whitespace_mask = _mm_set1_epi8(whitespace);
|
||||
constexpr auto base_sse_mode = _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_EACH | _SIDD_NEGATIVE_POLARITY;
|
||||
auto mask = bytes_sse;
|
||||
#endif
|
||||
const char * char_data = reinterpret_cast<const char *>(data);
|
||||
const char * char_end = char_data + size;
|
||||
|
||||
if constexpr (mode::trim_left)
|
||||
{
|
||||
#ifdef __SSE4_2__
|
||||
/// skip whitespace from left in blocks of up to 16 characters
|
||||
|
||||
/// Avoid gcc bug: _mm_cmpistri: error: the third argument must be an 8-bit immediate
|
||||
enum { left_sse_mode = base_sse_mode | _SIDD_LEAST_SIGNIFICANT };
|
||||
while (mask == bytes_sse && chars_to_trim_left < size_sse)
|
||||
{
|
||||
const auto chars = _mm_loadu_si128(reinterpret_cast<const __m128i *>(data + chars_to_trim_left));
|
||||
mask = _mm_cmpistri(whitespace_mask, chars, left_sse_mode);
|
||||
chars_to_trim_left += mask;
|
||||
}
|
||||
#endif
|
||||
/// skip remaining whitespace from left, character by character
|
||||
while (chars_to_trim_left < size && data[chars_to_trim_left] == whitespace)
|
||||
++chars_to_trim_left;
|
||||
const char * found = find_first_not_symbols<' '>(char_data, char_end);
|
||||
size_t num_chars = found - char_data;
|
||||
char_data += num_chars;
|
||||
}
|
||||
|
||||
if constexpr (mode::trim_right)
|
||||
{
|
||||
const auto trim_right_size = size - chars_to_trim_left;
|
||||
#ifdef __SSE4_2__
|
||||
/// try to skip whitespace from right in blocks of up to 16 characters
|
||||
|
||||
/// Avoid gcc bug: _mm_cmpistri: error: the third argument must be an 8-bit immediate
|
||||
enum { right_sse_mode = base_sse_mode | _SIDD_MOST_SIGNIFICANT };
|
||||
const auto trim_right_size_sse = trim_right_size - (trim_right_size % bytes_sse);
|
||||
while (mask == bytes_sse && chars_to_trim_right < trim_right_size_sse)
|
||||
{
|
||||
const auto chars = _mm_loadu_si128(reinterpret_cast<const __m128i *>(data + size - chars_to_trim_right - bytes_sse));
|
||||
mask = _mm_cmpistri(whitespace_mask, chars, right_sse_mode);
|
||||
chars_to_trim_right += mask;
|
||||
}
|
||||
#endif
|
||||
/// skip remaining whitespace from right, character by character
|
||||
while (chars_to_trim_right < trim_right_size && data[size - chars_to_trim_right - 1] == whitespace)
|
||||
++chars_to_trim_right;
|
||||
const char * found = find_last_not_symbols_or_null<' '>(char_data, char_end);
|
||||
if (found)
|
||||
char_end = found + 1;
|
||||
else
|
||||
char_end = char_data;
|
||||
}
|
||||
|
||||
res_data = data + chars_to_trim_left;
|
||||
res_size = size - chars_to_trim_left - chars_to_trim_right;
|
||||
res_data = reinterpret_cast<const UInt8 *>(char_data);
|
||||
res_size = char_end - char_data;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -40,7 +40,10 @@ HDFSBuilderPtr createHDFSBuilder(const Poco::URI & uri)
|
||||
hdfsBuilderSetUserName(builder.get(), user.c_str());
|
||||
}
|
||||
hdfsBuilderSetNameNode(builder.get(), host.c_str());
|
||||
hdfsBuilderSetNameNodePort(builder.get(), port);
|
||||
if (port != 0)
|
||||
{
|
||||
hdfsBuilderSetNameNodePort(builder.get(), port);
|
||||
}
|
||||
return builder;
|
||||
}
|
||||
|
||||
|
||||
@ -25,10 +25,6 @@
|
||||
#include <Common/assert_cast.h>
|
||||
#include <common/demangle.h>
|
||||
|
||||
#if __has_include(<Interpreters/config_compile.h>)
|
||||
#include <Interpreters/config_compile.h>
|
||||
#endif
|
||||
|
||||
|
||||
namespace ProfileEvents
|
||||
{
|
||||
@ -47,7 +43,6 @@ namespace DB
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_COMPILE_CODE;
|
||||
extern const int TOO_MANY_ROWS;
|
||||
extern const int EMPTY_DATA_PASSED;
|
||||
extern const int CANNOT_MERGE_DIFFERENT_AGGREGATED_DATA_VARIANTS;
|
||||
@ -195,200 +190,6 @@ Aggregator::Aggregator(const Params & params_)
|
||||
}
|
||||
|
||||
|
||||
void Aggregator::compileIfPossible(AggregatedDataVariants::Type type)
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
if (compiled_if_possible)
|
||||
return;
|
||||
|
||||
compiled_if_possible = true;
|
||||
|
||||
#if !defined(INTERNAL_COMPILER_HEADERS)
|
||||
throw Exception("Cannot compile code: Compiler disabled", ErrorCodes::CANNOT_COMPILE_CODE);
|
||||
#else
|
||||
std::string method_typename_single_level;
|
||||
std::string method_typename_two_level;
|
||||
|
||||
if (false) {}
|
||||
#define M(NAME) \
|
||||
else if (type == AggregatedDataVariants::Type::NAME) \
|
||||
{ \
|
||||
method_typename_single_level = "decltype(AggregatedDataVariants::" #NAME ")::element_type"; \
|
||||
method_typename_two_level = "decltype(AggregatedDataVariants::" #NAME "_two_level)::element_type"; \
|
||||
}
|
||||
|
||||
APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M)
|
||||
#undef M
|
||||
|
||||
#define M(NAME) \
|
||||
else if (type == AggregatedDataVariants::Type::NAME) \
|
||||
method_typename_single_level = "decltype(AggregatedDataVariants::" #NAME ")::element_type";
|
||||
|
||||
APPLY_FOR_VARIANTS_NOT_CONVERTIBLE_TO_TWO_LEVEL(M)
|
||||
#undef M
|
||||
else if (type == AggregatedDataVariants::Type::without_key) {}
|
||||
else
|
||||
throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT);
|
||||
|
||||
auto compiler_headers = Poco::Util::Application::instance().config().getString("compiler_headers", INTERNAL_COMPILER_HEADERS);
|
||||
|
||||
/// List of types of aggregate functions.
|
||||
std::stringstream aggregate_functions_typenames_str;
|
||||
std::stringstream aggregate_functions_headers_args;
|
||||
for (size_t i = 0; i < params.aggregates_size; ++i)
|
||||
{
|
||||
IAggregateFunction & func = *aggregate_functions[i];
|
||||
|
||||
int status = 0;
|
||||
std::string type_name = demangle(typeid(func).name(), status);
|
||||
|
||||
if (status)
|
||||
throw Exception("Cannot compile code: cannot demangle name " + String(typeid(func).name())
|
||||
+ ", status: " + toString(status), ErrorCodes::CANNOT_COMPILE_CODE);
|
||||
|
||||
aggregate_functions_typenames_str << ((i != 0) ? ", " : "") << type_name;
|
||||
|
||||
std::string header_path = func.getHeaderFilePath();
|
||||
auto pos = header_path.find("/AggregateFunctions/");
|
||||
|
||||
if (pos == std::string::npos)
|
||||
throw Exception("Cannot compile code: unusual path of header file for aggregate function: " + header_path,
|
||||
ErrorCodes::CANNOT_COMPILE_CODE);
|
||||
|
||||
aggregate_functions_headers_args << "-include '" << compiler_headers << "/dbms/src";
|
||||
aggregate_functions_headers_args.write(&header_path[pos], header_path.size() - pos);
|
||||
aggregate_functions_headers_args << "' ";
|
||||
}
|
||||
|
||||
aggregate_functions_headers_args << "-include '" << compiler_headers << "/dbms/src/Interpreters/SpecializedAggregator.h'";
|
||||
|
||||
std::string aggregate_functions_typenames = aggregate_functions_typenames_str.str();
|
||||
|
||||
std::stringstream key_str;
|
||||
key_str << "Aggregate: ";
|
||||
if (!method_typename_single_level.empty())
|
||||
key_str << method_typename_single_level + ", ";
|
||||
key_str << aggregate_functions_typenames;
|
||||
std::string key = key_str.str();
|
||||
|
||||
auto get_code = [method_typename_single_level, method_typename_two_level, aggregate_functions_typenames]
|
||||
{
|
||||
/// A short piece of code, which is an explicit instantiation of the template.
|
||||
std::stringstream code;
|
||||
code << /// No explicit inclusion of the header file. It is included using the -include compiler option.
|
||||
"namespace DB\n"
|
||||
"{\n"
|
||||
"\n";
|
||||
|
||||
/// There can be up to two instantiations for the template - for normal and two_level options.
|
||||
auto append_code_for_specialization =
|
||||
[&code, &aggregate_functions_typenames] (const std::string & method_typename, const std::string & suffix)
|
||||
{
|
||||
code <<
|
||||
"template void Aggregator::executeSpecialized<\n"
|
||||
" " << method_typename << ", TypeList<" << aggregate_functions_typenames << ">>(\n"
|
||||
" " << method_typename << " &, Arena *, size_t, ColumnRawPtrs &,\n"
|
||||
" AggregateColumns &, bool, AggregateDataPtr) const;\n"
|
||||
"\n"
|
||||
"static void wrapper" << suffix << "(\n"
|
||||
" const Aggregator & aggregator,\n"
|
||||
" " << method_typename << " & method,\n"
|
||||
" Arena * arena,\n"
|
||||
" size_t rows,\n"
|
||||
" ColumnRawPtrs & key_columns,\n"
|
||||
" Aggregator::AggregateColumns & aggregate_columns,\n"
|
||||
" bool no_more_keys,\n"
|
||||
" AggregateDataPtr overflow_row)\n"
|
||||
"{\n"
|
||||
" aggregator.executeSpecialized<\n"
|
||||
" " << method_typename << ", TypeList<" << aggregate_functions_typenames << ">>(\n"
|
||||
" method, arena, rows, key_columns, aggregate_columns, no_more_keys, overflow_row);\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"void * getPtr" << suffix << "() __attribute__((__visibility__(\"default\")));\n"
|
||||
"void * getPtr" << suffix << "()\n" /// Without this wrapper, it's not clear how to get the desired symbol from the compiled library.
|
||||
"{\n"
|
||||
" return reinterpret_cast<void *>(&wrapper" << suffix << ");\n"
|
||||
"}\n";
|
||||
};
|
||||
|
||||
if (!method_typename_single_level.empty())
|
||||
append_code_for_specialization(method_typename_single_level, "");
|
||||
else
|
||||
{
|
||||
/// For `without_key` method.
|
||||
code <<
|
||||
"template void Aggregator::executeSpecializedWithoutKey<\n"
|
||||
" " << "TypeList<" << aggregate_functions_typenames << ">>(\n"
|
||||
" AggregatedDataWithoutKey &, size_t, AggregateColumns &, Arena *) const;\n"
|
||||
"\n"
|
||||
"static void wrapper(\n"
|
||||
" const Aggregator & aggregator,\n"
|
||||
" AggregatedDataWithoutKey & method,\n"
|
||||
" size_t rows,\n"
|
||||
" Aggregator::AggregateColumns & aggregate_columns,\n"
|
||||
" Arena * arena)\n"
|
||||
"{\n"
|
||||
" aggregator.executeSpecializedWithoutKey<\n"
|
||||
" TypeList<" << aggregate_functions_typenames << ">>(\n"
|
||||
" method, rows, aggregate_columns, arena);\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"void * getPtr() __attribute__((__visibility__(\"default\")));\n"
|
||||
"void * getPtr()\n"
|
||||
"{\n"
|
||||
" return reinterpret_cast<void *>(&wrapper);\n"
|
||||
"}\n";
|
||||
}
|
||||
|
||||
if (!method_typename_two_level.empty())
|
||||
append_code_for_specialization(method_typename_two_level, "TwoLevel");
|
||||
else
|
||||
{
|
||||
/// The stub.
|
||||
code <<
|
||||
"void * getPtrTwoLevel() __attribute__((__visibility__(\"default\")));\n"
|
||||
"void * getPtrTwoLevel()\n"
|
||||
"{\n"
|
||||
" return nullptr;\n"
|
||||
"}\n";
|
||||
}
|
||||
|
||||
code <<
|
||||
"}\n";
|
||||
|
||||
return code.str();
|
||||
};
|
||||
|
||||
auto compiled_data_owned_by_callback = compiled_data;
|
||||
auto on_ready = [compiled_data_owned_by_callback] (SharedLibraryPtr & lib)
|
||||
{
|
||||
if (compiled_data_owned_by_callback.unique()) /// Aggregator is already destroyed.
|
||||
return;
|
||||
|
||||
compiled_data_owned_by_callback->compiled_aggregator = lib;
|
||||
compiled_data_owned_by_callback->compiled_method_ptr = lib->get<void * (*) ()>("_ZN2DB6getPtrEv")();
|
||||
compiled_data_owned_by_callback->compiled_two_level_method_ptr = lib->get<void * (*) ()>("_ZN2DB14getPtrTwoLevelEv")();
|
||||
};
|
||||
|
||||
/** If the library has already been compiled, a non-zero SharedLibraryPtr is returned.
|
||||
* If the library was not compiled, then the counter is incremented, and nullptr is returned.
|
||||
* If the counter has reached the value min_count_to_compile, then the compilation starts asynchronously (in a separate thread)
|
||||
* at the end of which `on_ready` callback is called.
|
||||
*/
|
||||
aggregate_functions_headers_args << " -Wno-unused-function";
|
||||
SharedLibraryPtr lib = params.compiler->getOrCount(key, params.min_count_to_compile,
|
||||
aggregate_functions_headers_args.str(),
|
||||
get_code, on_ready);
|
||||
|
||||
/// If the result is already ready.
|
||||
if (lib)
|
||||
on_ready(lib);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
AggregatedDataVariants::Type Aggregator::chooseAggregationMethod()
|
||||
{
|
||||
/// If no keys. All aggregating to single row.
|
||||
@ -720,9 +521,6 @@ bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & re
|
||||
result.keys_size = params.keys_size;
|
||||
result.key_sizes = key_sizes;
|
||||
LOG_TRACE(log, "Aggregation method: " << result.getMethodName());
|
||||
|
||||
if (params.compiler)
|
||||
compileIfPossible(result.type);
|
||||
}
|
||||
|
||||
if (isCancelled())
|
||||
@ -794,67 +592,21 @@ bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & re
|
||||
/// For the case when there are no keys (all aggregate into one row).
|
||||
if (result.type == AggregatedDataVariants::Type::without_key)
|
||||
{
|
||||
/// If there is a dynamically compiled code.
|
||||
if (compiled_data->compiled_method_ptr)
|
||||
{
|
||||
reinterpret_cast<
|
||||
void (*)(const Aggregator &, AggregatedDataWithoutKey &, size_t, AggregateColumns &, Arena *)>
|
||||
(compiled_data->compiled_method_ptr)(*this, result.without_key, rows, aggregate_columns, result.aggregates_pool);
|
||||
}
|
||||
else
|
||||
executeWithoutKeyImpl(result.without_key, rows, aggregate_functions_instructions.data(), result.aggregates_pool);
|
||||
executeWithoutKeyImpl(result.without_key, rows, aggregate_functions_instructions.data(), result.aggregates_pool);
|
||||
}
|
||||
else
|
||||
{
|
||||
/// This is where data is written that does not fit in `max_rows_to_group_by` with `group_by_overflow_mode = any`.
|
||||
AggregateDataPtr overflow_row_ptr = params.overflow_row ? result.without_key : nullptr;
|
||||
|
||||
bool is_two_level = result.isTwoLevel();
|
||||
|
||||
/// Compiled code, for the normal structure.
|
||||
if (!is_two_level && compiled_data->compiled_method_ptr)
|
||||
{
|
||||
#define M(NAME, IS_TWO_LEVEL) \
|
||||
else if (result.type == AggregatedDataVariants::Type::NAME) \
|
||||
reinterpret_cast<void (*)( \
|
||||
const Aggregator &, decltype(result.NAME)::element_type &, \
|
||||
Arena *, size_t, ColumnRawPtrs &, AggregateColumns &, \
|
||||
bool, AggregateDataPtr)>(compiled_data->compiled_method_ptr) \
|
||||
(*this, *result.NAME, result.aggregates_pool, rows, key_columns, aggregate_columns, \
|
||||
no_more_keys, overflow_row_ptr);
|
||||
|
||||
if (false) {}
|
||||
APPLY_FOR_AGGREGATED_VARIANTS(M)
|
||||
#undef M
|
||||
}
|
||||
/// Compiled code, for a two-level structure.
|
||||
else if (is_two_level && compiled_data->compiled_two_level_method_ptr)
|
||||
{
|
||||
#define M(NAME) \
|
||||
else if (result.type == AggregatedDataVariants::Type::NAME) \
|
||||
reinterpret_cast<void (*)( \
|
||||
const Aggregator &, decltype(result.NAME)::element_type &, \
|
||||
Arena *, size_t, ColumnRawPtrs &, AggregateColumns &, \
|
||||
bool, AggregateDataPtr)>(compiled_data->compiled_two_level_method_ptr) \
|
||||
(*this, *result.NAME, result.aggregates_pool, rows, key_columns, aggregate_columns, \
|
||||
no_more_keys, overflow_row_ptr);
|
||||
|
||||
if (false) {}
|
||||
APPLY_FOR_VARIANTS_TWO_LEVEL(M)
|
||||
#undef M
|
||||
}
|
||||
/// When there is no dynamically compiled code.
|
||||
else
|
||||
{
|
||||
#define M(NAME, IS_TWO_LEVEL) \
|
||||
else if (result.type == AggregatedDataVariants::Type::NAME) \
|
||||
executeImpl(*result.NAME, result.aggregates_pool, rows, key_columns, aggregate_functions_instructions.data(), \
|
||||
no_more_keys, overflow_row_ptr);
|
||||
|
||||
if (false) {}
|
||||
APPLY_FOR_AGGREGATED_VARIANTS(M)
|
||||
if (false) {}
|
||||
APPLY_FOR_AGGREGATED_VARIANTS(M)
|
||||
#undef M
|
||||
}
|
||||
}
|
||||
|
||||
size_t result_size = result.sizeWithoutOverflowRow();
|
||||
|
||||
@ -24,7 +24,6 @@
|
||||
|
||||
#include <Interpreters/AggregateDescription.h>
|
||||
#include <Interpreters/AggregationCommon.h>
|
||||
#include <Interpreters/Compiler.h>
|
||||
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnFixedString.h>
|
||||
@ -778,10 +777,6 @@ public:
|
||||
const size_t max_rows_to_group_by;
|
||||
const OverflowMode group_by_overflow_mode;
|
||||
|
||||
/// For dynamic compilation.
|
||||
Compiler * compiler;
|
||||
const UInt32 min_count_to_compile;
|
||||
|
||||
/// Two-level aggregation settings (used for a large number of keys).
|
||||
/** With how many keys or the size of the aggregation state in bytes,
|
||||
* two-level aggregation begins to be used. Enough to reach of at least one of the thresholds.
|
||||
@ -805,7 +800,6 @@ public:
|
||||
const Block & src_header_,
|
||||
const ColumnNumbers & keys_, const AggregateDescriptions & aggregates_,
|
||||
bool overflow_row_, size_t max_rows_to_group_by_, OverflowMode group_by_overflow_mode_,
|
||||
Compiler * compiler_, UInt32 min_count_to_compile_,
|
||||
size_t group_by_two_level_threshold_, size_t group_by_two_level_threshold_bytes_,
|
||||
size_t max_bytes_before_external_group_by_,
|
||||
bool empty_result_for_aggregation_by_empty_set_,
|
||||
@ -813,7 +807,6 @@ public:
|
||||
: src_header(src_header_),
|
||||
keys(keys_), aggregates(aggregates_), keys_size(keys.size()), aggregates_size(aggregates.size()),
|
||||
overflow_row(overflow_row_), max_rows_to_group_by(max_rows_to_group_by_), group_by_overflow_mode(group_by_overflow_mode_),
|
||||
compiler(compiler_), min_count_to_compile(min_count_to_compile_),
|
||||
group_by_two_level_threshold(group_by_two_level_threshold_), group_by_two_level_threshold_bytes(group_by_two_level_threshold_bytes_),
|
||||
max_bytes_before_external_group_by(max_bytes_before_external_group_by_),
|
||||
empty_result_for_aggregation_by_empty_set(empty_result_for_aggregation_by_empty_set_),
|
||||
@ -824,7 +817,7 @@ public:
|
||||
/// Only parameters that matter during merge.
|
||||
Params(const Block & intermediate_header_,
|
||||
const ColumnNumbers & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, size_t max_threads_)
|
||||
: Params(Block(), keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, nullptr, 0, 0, 0, 0, false, "", max_threads_)
|
||||
: Params(Block(), keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, 0, 0, 0, false, "", max_threads_)
|
||||
{
|
||||
intermediate_header = intermediate_header_;
|
||||
}
|
||||
@ -956,26 +949,6 @@ protected:
|
||||
|
||||
Logger * log = &Logger::get("Aggregator");
|
||||
|
||||
/** Dynamically compiled library for aggregation, if any.
|
||||
* The meaning of dynamic compilation is to specialize code
|
||||
* for a specific list of aggregate functions.
|
||||
* This allows you to expand the loop to create and update states of aggregate functions,
|
||||
* and also use inline-code instead of virtual calls.
|
||||
*/
|
||||
struct CompiledData
|
||||
{
|
||||
SharedLibraryPtr compiled_aggregator;
|
||||
|
||||
/// Obtained with dlsym. It is still necessary to make reinterpret_cast to the function pointer.
|
||||
void * compiled_method_ptr = nullptr;
|
||||
void * compiled_two_level_method_ptr = nullptr;
|
||||
};
|
||||
/// shared_ptr - to pass into a callback, that can survive Aggregator.
|
||||
std::shared_ptr<CompiledData> compiled_data { new CompiledData };
|
||||
|
||||
bool compiled_if_possible = false;
|
||||
void compileIfPossible(AggregatedDataVariants::Type type);
|
||||
|
||||
/// Returns true if you can abort the current task.
|
||||
CancellationHook isCancelled;
|
||||
|
||||
@ -1037,35 +1010,6 @@ protected:
|
||||
Method & method,
|
||||
IBlockOutputStream & out);
|
||||
|
||||
public:
|
||||
/// Templates that are instantiated by dynamic code compilation - see SpecializedAggregator.h
|
||||
|
||||
template <typename Method, typename AggregateFunctionsList>
|
||||
void executeSpecialized(
|
||||
Method & method,
|
||||
Arena * aggregates_pool,
|
||||
size_t rows,
|
||||
ColumnRawPtrs & key_columns,
|
||||
AggregateColumns & aggregate_columns,
|
||||
bool no_more_keys,
|
||||
AggregateDataPtr overflow_row) const;
|
||||
|
||||
template <bool no_more_keys, typename Method, typename AggregateFunctionsList>
|
||||
void executeSpecializedCase(
|
||||
Method & method,
|
||||
typename Method::State & state,
|
||||
Arena * aggregates_pool,
|
||||
size_t rows,
|
||||
AggregateColumns & aggregate_columns,
|
||||
AggregateDataPtr overflow_row) const;
|
||||
|
||||
template <typename AggregateFunctionsList>
|
||||
void executeSpecializedWithoutKey(
|
||||
AggregatedDataWithoutKey & res,
|
||||
size_t rows,
|
||||
AggregateColumns & aggregate_columns,
|
||||
Arena * arena) const;
|
||||
|
||||
protected:
|
||||
/// Merge NULL key data from hash table `src` into `dst`.
|
||||
template <typename Method, typename Table>
|
||||
|
||||
@ -1,70 +1,3 @@
|
||||
|
||||
if (OS_FREEBSD)
|
||||
set (PATH_SHARE "/usr/local/share" CACHE STRING "")
|
||||
else ()
|
||||
set (PATH_SHARE "/usr/share" CACHE STRING "")
|
||||
endif ()
|
||||
|
||||
set (INTERNAL_COMPILER_BIN_ROOT "${CMAKE_INSTALL_FULL_BINDIR}/" CACHE STRING "")
|
||||
set (INTERNAL_COMPILER_EXECUTABLE "clickhouse-clang" CACHE STRING "")
|
||||
set (INTERNAL_LINKER_EXECUTABLE "clickhouse-lld" CACHE STRING "")
|
||||
|
||||
# Disabling leak reporting for these tools
|
||||
if (SANITIZE STREQUAL "address")
|
||||
# Note that this doesn't work for setuid and setcap binaries
|
||||
set(INTERNAL_COMPILER_ENV "env ASAN_OPTIONS=detect_leaks=0" CACHE STRING "")
|
||||
else ()
|
||||
set(INTERNAL_COMPILER_ENV "" CACHE STRING "")
|
||||
endif ()
|
||||
|
||||
set (INTERNAL_COMPILER_NO_WARNING OFF CACHE INTERNAL "")
|
||||
set (INTERNAL_COMPILER_HEADERS_DIR "headers" CACHE STRING "")
|
||||
set (INTERNAL_COMPILER_HEADERS_RELATIVE "${INTERNAL_COMPILER_HEADERS_DIR}/${VERSION_STRING}" CACHE STRING "")
|
||||
set (INTERNAL_COMPILER_HEADERS "${PATH_SHARE}/clickhouse/${INTERNAL_COMPILER_HEADERS_RELATIVE}" CACHE STRING "")
|
||||
|
||||
if(OS_FREEBSD)
|
||||
set(INTERNAL_COMPILER_HEADERS_ROOT "" CACHE STRING "")
|
||||
else()
|
||||
set(INTERNAL_COMPILER_HEADERS_ROOT "${INTERNAL_COMPILER_HEADERS}" CACHE STRING "")
|
||||
set(INTERNAL_COMPILER_CUSTOM_ROOT ON CACHE INTERNAL "")
|
||||
endif()
|
||||
|
||||
if(NOT INTERNAL_COMPILER_FLAGS)
|
||||
set(INTERNAL_COMPILER_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UC}} ${CXX_FLAGS_INTERNAL_COMPILER} -x c++ -march=native -fPIC -fvisibility=hidden -fno-implement-inlines -Wno-unused-command-line-argument -Bprefix=${PATH_SHARE}/clickhouse" CACHE STRING "")
|
||||
if(INTERNAL_COMPILER_CUSTOM_ROOT)
|
||||
set(INTERNAL_COMPILER_FLAGS "${INTERNAL_COMPILER_FLAGS} -nostdinc -nostdinc++")
|
||||
if(INTERNAL_COMPILER_HEADERS_ROOT)
|
||||
set(INTERNAL_COMPILER_FLAGS "${INTERNAL_COMPILER_FLAGS} -isysroot=${INTERNAL_COMPILER_HEADERS_ROOT}")
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
# TODO: use libs from package: -nodefaultlibs -lm -lc -lgcc_s -lgcc -lc++ -lc++abi
|
||||
|
||||
string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS})
|
||||
string(REPLACE "-no-pie" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS})
|
||||
if (INTERNAL_COMPILER_NO_WARNING)
|
||||
string (REPLACE "-Wall" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS})
|
||||
string (REPLACE "-Wextra" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS})
|
||||
string (REPLACE "-Werror" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS})
|
||||
endif ()
|
||||
|
||||
list(GET Poco_INCLUDE_DIRS 0 Poco_Foundation_INCLUDE_DIR)
|
||||
list(GET Poco_INCLUDE_DIRS 1 Poco_Util_INCLUDE_DIR)
|
||||
|
||||
if (NOT DOUBLE_CONVERSION_INCLUDE_DIR)
|
||||
get_target_property(DOUBLE_CONVERSION_INCLUDE_DIR ${DOUBLE_CONVERSION_LIBRARIES} INTERFACE_INCLUDE_DIRECTORIES)
|
||||
endif ()
|
||||
|
||||
string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR ${DOUBLE_CONVERSION_INCLUDE_DIR})
|
||||
string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_Boost_INCLUDE_DIRS ${Boost_INCLUDE_DIRS})
|
||||
string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_Poco_Foundation_INCLUDE_DIR ${Poco_Foundation_INCLUDE_DIR})
|
||||
string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_Poco_Util_INCLUDE_DIR ${Poco_Util_INCLUDE_DIR})
|
||||
|
||||
message (STATUS "Using internal=${USE_INTERNAL_LLVM_LIBRARY} compiler=${USE_EMBEDDED_COMPILER}: headers=${INTERNAL_COMPILER_HEADERS} root=${INTERNAL_COMPILER_HEADERS_ROOT}: ${INTERNAL_COMPILER_ENV} ${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE} ${INTERNAL_COMPILER_FLAGS}; ${INTERNAL_LINKER_EXECUTABLE}")
|
||||
|
||||
set (CONFIG_COMPILE ${ClickHouse_BINARY_DIR}/dbms/src/Interpreters/config_compile.h)
|
||||
configure_file (${ClickHouse_SOURCE_DIR}/dbms/src/Interpreters/config_compile.h.in ${CONFIG_COMPILE})
|
||||
|
||||
if (ENABLE_TESTS)
|
||||
add_subdirectory (tests)
|
||||
endif ()
|
||||
|
||||
@ -28,7 +28,7 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin
|
||||
/// Set as unchanged to avoid sending to remote server.
|
||||
new_settings.max_concurrent_queries_for_user.changed = false;
|
||||
new_settings.max_memory_usage_for_user.changed = false;
|
||||
new_settings.max_memory_usage_for_all_queries.changed = false;
|
||||
new_settings.max_memory_usage_for_all_queries = false;
|
||||
|
||||
Context new_context(context);
|
||||
new_context.setSettings(new_settings);
|
||||
|
||||
@ -1,326 +0,0 @@
|
||||
#include <Poco/DirectoryIterator.h>
|
||||
#include <Poco/Util/Application.h>
|
||||
#include <ext/unlock_guard.h>
|
||||
#include <Common/ClickHouseRevision.h>
|
||||
#include <Common/config.h>
|
||||
#include <Common/SipHash.h>
|
||||
#include <Common/ShellCommand.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <IO/Operators.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <IO/ReadBufferFromFileDescriptor.h>
|
||||
#include <IO/WriteBufferFromFile.h>
|
||||
#include <Interpreters/Compiler.h>
|
||||
|
||||
#if __has_include(<Interpreters/config_compile.h>)
|
||||
#include <Interpreters/config_compile.h>
|
||||
#endif
|
||||
|
||||
namespace ProfileEvents
|
||||
{
|
||||
extern const Event CompileAttempt;
|
||||
extern const Event CompileSuccess;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_DLOPEN;
|
||||
extern const int CANNOT_COMPILE_CODE;
|
||||
}
|
||||
|
||||
Compiler::Compiler(const std::string & path_, size_t threads)
|
||||
: path(path_), pool(threads)
|
||||
{
|
||||
Poco::File(path).createDirectory();
|
||||
|
||||
Poco::DirectoryIterator dir_end;
|
||||
for (Poco::DirectoryIterator dir_it(path); dir_end != dir_it; ++dir_it)
|
||||
{
|
||||
const std::string & name = dir_it.name();
|
||||
if (endsWith(name, ".so"))
|
||||
{
|
||||
files.insert(name.substr(0, name.size() - 3));
|
||||
}
|
||||
}
|
||||
|
||||
LOG_INFO(log, "Having " << files.size() << " compiled files from previous start.");
|
||||
}
|
||||
|
||||
Compiler::~Compiler()
|
||||
{
|
||||
LOG_DEBUG(log, "Waiting for threads to finish.");
|
||||
pool.wait();
|
||||
}
|
||||
|
||||
|
||||
static Compiler::HashedKey getHash(const std::string & key)
|
||||
{
|
||||
SipHash hash;
|
||||
|
||||
auto revision = ClickHouseRevision::get();
|
||||
hash.update(revision);
|
||||
hash.update(key.data(), key.size());
|
||||
|
||||
Compiler::HashedKey res;
|
||||
hash.get128(res.low, res.high);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
/// Without .so extension.
|
||||
static std::string hashedKeyToFileName(Compiler::HashedKey hashed_key)
|
||||
{
|
||||
WriteBufferFromOwnString out;
|
||||
out << hashed_key.low << '_' << hashed_key.high;
|
||||
return out.str();
|
||||
}
|
||||
|
||||
|
||||
SharedLibraryPtr Compiler::getOrCount(
|
||||
const std::string & key,
|
||||
UInt32 min_count_to_compile,
|
||||
const std::string & additional_compiler_flags,
|
||||
CodeGenerator get_code,
|
||||
ReadyCallback on_ready)
|
||||
{
|
||||
HashedKey hashed_key = getHash(key);
|
||||
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
UInt32 count = ++counts[hashed_key];
|
||||
|
||||
/// Is there a ready open library? Or, if the library is in the process of compiling, there will be nullptr.
|
||||
Libraries::iterator libraries_it = libraries.find(hashed_key);
|
||||
if (libraries.end() != libraries_it)
|
||||
{
|
||||
if (!libraries_it->second)
|
||||
LOG_INFO(log, "Library " << hashedKeyToFileName(hashed_key) << " is already compiling or compilation was failed.");
|
||||
|
||||
/// TODO In this case, after the compilation is finished, the callback will not be called.
|
||||
|
||||
return libraries_it->second;
|
||||
}
|
||||
|
||||
/// Is there a file with the library left over from the previous launch?
|
||||
std::string file_name = hashedKeyToFileName(hashed_key);
|
||||
Files::iterator files_it = files.find(file_name);
|
||||
if (files.end() != files_it)
|
||||
{
|
||||
std::string so_file_path = path + '/' + file_name + ".so";
|
||||
LOG_INFO(log, "Loading existing library " << so_file_path);
|
||||
|
||||
SharedLibraryPtr lib;
|
||||
|
||||
try
|
||||
{
|
||||
lib = std::make_shared<SharedLibrary>(so_file_path);
|
||||
}
|
||||
catch (const Exception & e)
|
||||
{
|
||||
if (e.code() != ErrorCodes::CANNOT_DLOPEN)
|
||||
throw;
|
||||
|
||||
/// Found broken .so file (or file cannot be dlopened by whatever reason).
|
||||
/// This could happen when filesystem is corrupted after server restart.
|
||||
/// We remove the file - it will be recompiled on next attempt.
|
||||
|
||||
tryLogCurrentException(log);
|
||||
|
||||
files.erase(files_it);
|
||||
Poco::File(so_file_path).remove();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
libraries[hashed_key] = lib;
|
||||
return lib;
|
||||
}
|
||||
|
||||
/// Has min_count_to_compile been reached?
|
||||
if (count >= min_count_to_compile)
|
||||
{
|
||||
/// The min_count_to_compile value of zero indicates the need for synchronous compilation.
|
||||
|
||||
/// Indicates that the library is in the process of compiling.
|
||||
libraries[hashed_key] = nullptr;
|
||||
|
||||
LOG_INFO(log, "Compiling code " << file_name << ", key: " << key);
|
||||
|
||||
if (min_count_to_compile == 0)
|
||||
{
|
||||
{
|
||||
ext::unlock_guard<std::mutex> unlock(mutex);
|
||||
compile(hashed_key, file_name, additional_compiler_flags, get_code, on_ready);
|
||||
}
|
||||
|
||||
return libraries[hashed_key];
|
||||
}
|
||||
else
|
||||
{
|
||||
bool res = pool.trySchedule([=]
|
||||
{
|
||||
try
|
||||
{
|
||||
compile(hashed_key, file_name, additional_compiler_flags, get_code, on_ready);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException("Compiler");
|
||||
}
|
||||
});
|
||||
|
||||
if (!res)
|
||||
LOG_INFO(log, "All threads are busy.");
|
||||
}
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
|
||||
/// This will guarantee that code will compile only when version of headers match version of running server.
|
||||
static void addCodeToAssertHeadersMatch(WriteBuffer & out)
|
||||
{
|
||||
out <<
|
||||
"#include <Common/config_version.h>\n"
|
||||
"#if VERSION_REVISION != " << ClickHouseRevision::get() << "\n"
|
||||
"#define STRING2(x) #x\n"
|
||||
"#define STRING(x) STRING2(x)\n"
|
||||
"#pragma message \"ClickHouse headers revision = \" STRING(VERSION_REVISION) \n"
|
||||
"#error \"ClickHouse headers revision doesn't match runtime revision of the server (" << ClickHouseRevision::get() << ").\"\n"
|
||||
"#endif\n\n";
|
||||
}
|
||||
|
||||
|
||||
void Compiler::compile(
|
||||
HashedKey hashed_key,
|
||||
std::string file_name,
|
||||
const std::string & additional_compiler_flags,
|
||||
CodeGenerator get_code,
|
||||
ReadyCallback on_ready)
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::CompileAttempt);
|
||||
|
||||
#if !defined(INTERNAL_COMPILER_EXECUTABLE)
|
||||
throw Exception("Cannot compile code: Compiler disabled", ErrorCodes::CANNOT_COMPILE_CODE);
|
||||
#else
|
||||
std::string prefix = path + "/" + file_name;
|
||||
std::string cpp_file_path = prefix + ".cpp";
|
||||
std::string so_file_path = prefix + ".so";
|
||||
std::string so_tmp_file_path = prefix + ".so.tmp";
|
||||
|
||||
{
|
||||
WriteBufferFromFile out(cpp_file_path);
|
||||
|
||||
addCodeToAssertHeadersMatch(out);
|
||||
out << get_code();
|
||||
}
|
||||
|
||||
std::stringstream command;
|
||||
|
||||
auto compiler_executable_root = Poco::Util::Application::instance().config().getString("compiler_executable_root", INTERNAL_COMPILER_BIN_ROOT);
|
||||
auto compiler_headers = Poco::Util::Application::instance().config().getString("compiler_headers", INTERNAL_COMPILER_HEADERS);
|
||||
auto compiler_headers_root = Poco::Util::Application::instance().config().getString("compiler_headers_root", INTERNAL_COMPILER_HEADERS_ROOT);
|
||||
LOG_DEBUG(log, "Using internal compiler: compiler_executable_root=" << compiler_executable_root << "; compiler_headers_root=" << compiler_headers_root << "; compiler_headers=" << compiler_headers);
|
||||
|
||||
/// Slightly unconvenient.
|
||||
command <<
|
||||
"("
|
||||
INTERNAL_COMPILER_ENV
|
||||
" " << compiler_executable_root << INTERNAL_COMPILER_EXECUTABLE
|
||||
" " INTERNAL_COMPILER_FLAGS
|
||||
/// It is hard to correctly call a ld program manually, because it is easy to skip critical flags, which might lead to
|
||||
/// unhandled exceptions. Therefore pass path to llvm's lld directly to clang.
|
||||
" -fuse-ld=" << compiler_executable_root << INTERNAL_LINKER_EXECUTABLE
|
||||
" -fdiagnostics-color=never"
|
||||
|
||||
/// Do not use libgcc and startup files. The library will work nevertheless and we avoid extra dependency.
|
||||
" -nodefaultlibs -nostartfiles"
|
||||
|
||||
#if INTERNAL_COMPILER_CUSTOM_ROOT
|
||||
/// To get correct order merge this results carefully:
|
||||
/// echo | clang -x c++ -E -Wp,-v -
|
||||
/// echo | g++ -x c++ -E -Wp,-v -
|
||||
|
||||
" -isystem " << compiler_headers_root << "/usr/include/c++/*"
|
||||
#if defined(CMAKE_LIBRARY_ARCHITECTURE)
|
||||
" -isystem " << compiler_headers_root << "/usr/include/" CMAKE_LIBRARY_ARCHITECTURE "/c++/*"
|
||||
#endif
|
||||
" -isystem " << compiler_headers_root << "/usr/include/c++/*/backward"
|
||||
" -isystem " << compiler_headers_root << "/usr/include/clang/*/include" /// if compiler is clang (from package)
|
||||
" -isystem " << compiler_headers_root << "/usr/local/lib/clang/*/include" /// if clang installed manually
|
||||
" -isystem " << compiler_headers_root << "/usr/lib/clang/*/include" /// if clang build from submodules
|
||||
#if defined(CMAKE_LIBRARY_ARCHITECTURE)
|
||||
" -isystem " << compiler_headers_root << "/usr/lib/gcc/" CMAKE_LIBRARY_ARCHITECTURE "/*/include-fixed"
|
||||
" -isystem " << compiler_headers_root << "/usr/lib/gcc/" CMAKE_LIBRARY_ARCHITECTURE "/*/include"
|
||||
#endif
|
||||
" -isystem " << compiler_headers_root << "/usr/local/include" /// if something installed manually
|
||||
#if defined(CMAKE_LIBRARY_ARCHITECTURE)
|
||||
" -isystem " << compiler_headers_root << "/usr/include/" CMAKE_LIBRARY_ARCHITECTURE
|
||||
#endif
|
||||
" -isystem " << compiler_headers_root << "/usr/include"
|
||||
#endif
|
||||
" -I " << compiler_headers << "/dbms/src/"
|
||||
" -isystem " << compiler_headers << "/contrib/cityhash102/include/"
|
||||
" -isystem " << compiler_headers << "/contrib/libpcg-random/include/"
|
||||
#if USE_MIMALLOC
|
||||
" -isystem " << compiler_headers << "/contrib/mimalloc/include/"
|
||||
#endif
|
||||
" -isystem " << compiler_headers << INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR
|
||||
" -isystem " << compiler_headers << INTERNAL_Poco_Foundation_INCLUDE_DIR
|
||||
" -isystem " << compiler_headers << INTERNAL_Boost_INCLUDE_DIRS
|
||||
" -I " << compiler_headers << "/libs/libcommon/include/"
|
||||
" " << additional_compiler_flags <<
|
||||
" -shared -o " << so_tmp_file_path << " " << cpp_file_path
|
||||
<< " 2>&1"
|
||||
") || echo Return code: $?";
|
||||
|
||||
#ifndef NDEBUG
|
||||
LOG_TRACE(log, "Compile command: " << command.str());
|
||||
#endif
|
||||
|
||||
std::string compile_result;
|
||||
|
||||
{
|
||||
auto process = ShellCommand::execute(command.str());
|
||||
readStringUntilEOF(compile_result, process->out);
|
||||
process->wait();
|
||||
}
|
||||
|
||||
if (!compile_result.empty())
|
||||
{
|
||||
std::string error_message = "Cannot compile code:\n\n" + command.str() + "\n\n" + compile_result;
|
||||
|
||||
Poco::File so_tmp_file(so_tmp_file_path);
|
||||
if (so_tmp_file.exists() && so_tmp_file.canExecute())
|
||||
{
|
||||
/// Compiler may emit information messages. This is suspicious, but we still can use compiled result.
|
||||
LOG_WARNING(log, error_message);
|
||||
}
|
||||
else
|
||||
throw Exception(error_message, ErrorCodes::CANNOT_COMPILE_CODE);
|
||||
}
|
||||
|
||||
/// If there was an error before, the file with the code remains for viewing.
|
||||
Poco::File(cpp_file_path).remove();
|
||||
|
||||
Poco::File(so_tmp_file_path).renameTo(so_file_path);
|
||||
SharedLibraryPtr lib(new SharedLibrary(so_file_path));
|
||||
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
libraries[hashed_key] = lib;
|
||||
}
|
||||
|
||||
LOG_INFO(log, "Compiled code " << file_name);
|
||||
ProfileEvents::increment(ProfileEvents::CompileSuccess);
|
||||
|
||||
on_ready(lib);
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,88 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <boost/core/noncopyable.hpp>
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <mutex>
|
||||
#include <functional>
|
||||
#include <unordered_set>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/UInt128.h>
|
||||
#include <Common/SharedLibrary.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** Lets you compile a piece of code that uses the server's header files into the dynamic library.
|
||||
* Conducts statistic of calls, and initiates compilation only on the N-th call for one key.
|
||||
* Compilation is performed asynchronously, in separate threads, if there are free threads.
|
||||
* NOTE: There is no cleaning of obsolete and unnecessary results.
|
||||
*/
|
||||
class Compiler
|
||||
{
|
||||
public:
|
||||
/** path - path to the directory with temporary files - the results of the compilation.
|
||||
* The compilation results are saved when the server is restarted,
|
||||
* but use the revision number as part of the key. That is, they become obsolete when the server is updated.
|
||||
*/
|
||||
Compiler(const std::string & path_, size_t threads);
|
||||
~Compiler();
|
||||
|
||||
using HashedKey = UInt128;
|
||||
|
||||
using CodeGenerator = std::function<std::string()>;
|
||||
using ReadyCallback = std::function<void(SharedLibraryPtr&)>;
|
||||
|
||||
/** Increase the counter for the given key `key` by one.
|
||||
* If the compilation result already exists (already open, or there is a file with the library),
|
||||
* then return ready SharedLibrary.
|
||||
* Otherwise, if min_count_to_compile == 0, then initiate the compilation in the same thread, wait for it, and return the result.
|
||||
* Otherwise, if the counter has reached min_count_to_compile,
|
||||
* initiate compilation in a separate thread, if there are free threads, and return nullptr.
|
||||
* Otherwise, return nullptr.
|
||||
*/
|
||||
SharedLibraryPtr getOrCount(
|
||||
const std::string & key,
|
||||
UInt32 min_count_to_compile,
|
||||
const std::string & additional_compiler_flags,
|
||||
CodeGenerator get_code,
|
||||
ReadyCallback on_ready);
|
||||
|
||||
private:
|
||||
using Counts = std::unordered_map<HashedKey, UInt32, UInt128Hash>;
|
||||
using Libraries = std::unordered_map<HashedKey, SharedLibraryPtr, UInt128Hash>;
|
||||
using Files = std::unordered_set<std::string>;
|
||||
|
||||
const std::string path;
|
||||
ThreadPool pool;
|
||||
|
||||
/// Number of calls to `getOrCount`.
|
||||
Counts counts;
|
||||
|
||||
/// Compiled and open libraries. Or nullptr for libraries in the compilation process.
|
||||
Libraries libraries;
|
||||
|
||||
/// Compiled files remaining from previous runs, but not yet open.
|
||||
Files files;
|
||||
|
||||
std::mutex mutex;
|
||||
|
||||
Logger * log = &Logger::get("Compiler");
|
||||
|
||||
|
||||
void compile(
|
||||
HashedKey hashed_key,
|
||||
std::string file_name,
|
||||
const std::string & additional_compiler_flags,
|
||||
CodeGenerator get_code,
|
||||
ReadyCallback on_ready);
|
||||
};
|
||||
|
||||
}
|
||||
@ -35,7 +35,6 @@
|
||||
#include <Interpreters/ProcessList.h>
|
||||
#include <Interpreters/Cluster.h>
|
||||
#include <Interpreters/InterserverIOHandler.h>
|
||||
#include <Interpreters/Compiler.h>
|
||||
#include <Interpreters/SettingsConstraints.h>
|
||||
#include <Interpreters/SystemLog.h>
|
||||
#include <Interpreters/Context.h>
|
||||
@ -140,11 +139,10 @@ struct ContextShared
|
||||
std::optional<BackgroundProcessingPool> background_pool; /// The thread pool for the background work performed by the tables.
|
||||
std::optional<BackgroundSchedulePool> schedule_pool; /// A thread pool that can run different jobs in background (used in replicated tables)
|
||||
MultiVersion<Macros> macros; /// Substitutions extracted from config.
|
||||
std::optional<Compiler> compiler; /// Used for dynamic compilation of queries' parts if it necessary.
|
||||
std::unique_ptr<DDLWorker> ddl_worker; /// Process ddl commands from zk.
|
||||
/// Rules for selecting the compression settings, depending on the size of the part.
|
||||
mutable std::unique_ptr<CompressionCodecSelector> compression_codec_selector;
|
||||
std::optional<MergeTreeSettings> merge_tree_settings; /// Settings of MergeTree* engines.
|
||||
MergeTreeSettingsPtr merge_tree_settings; /// Settings of MergeTree* engines.
|
||||
size_t max_table_size_to_drop = 50000000000lu; /// Protects MergeTree tables from accidental DROP (50GB by default)
|
||||
size_t max_partition_size_to_drop = 50000000000lu; /// Protects MergeTree partitions from accidental DROP (50GB by default)
|
||||
String format_schema_path; /// Path to a directory that contains schema files used by input formats.
|
||||
@ -1126,6 +1124,17 @@ void Context::applySettingsChanges(const SettingsChanges & changes)
|
||||
applySettingChange(change);
|
||||
}
|
||||
|
||||
void Context::updateSettingsChanges(const SettingsChanges & changes)
|
||||
{
|
||||
auto lock = getLock();
|
||||
for (const SettingChange & change : changes)
|
||||
{
|
||||
if (change.name == "profile")
|
||||
setProfile(change.value.safeGet<String>());
|
||||
else
|
||||
settings.updateFromChange(change);
|
||||
}
|
||||
}
|
||||
|
||||
void Context::checkSettingsConstraints(const SettingChange & change)
|
||||
{
|
||||
@ -1634,17 +1643,6 @@ void Context::setCluster(const String & cluster_name, const std::shared_ptr<Clus
|
||||
}
|
||||
|
||||
|
||||
Compiler & Context::getCompiler()
|
||||
{
|
||||
auto lock = getLock();
|
||||
|
||||
if (!shared->compiler)
|
||||
shared->compiler.emplace(shared->path + "build/", 1);
|
||||
|
||||
return *shared->compiler;
|
||||
}
|
||||
|
||||
|
||||
void Context::initializeSystemLogs()
|
||||
{
|
||||
auto lock = getLock();
|
||||
@ -1761,8 +1759,9 @@ const MergeTreeSettings & Context::getMergeTreeSettings() const
|
||||
if (!shared->merge_tree_settings)
|
||||
{
|
||||
auto & config = getConfigRef();
|
||||
shared->merge_tree_settings.emplace();
|
||||
shared->merge_tree_settings->loadFromConfig("merge_tree", config);
|
||||
MutableMergeTreeSettingsPtr settings_ptr = MergeTreeSettings::create();
|
||||
settings_ptr->loadFromConfig("merge_tree", config);
|
||||
shared->merge_tree_settings = std::move(settings_ptr);
|
||||
}
|
||||
|
||||
return *shared->merge_tree_settings;
|
||||
|
||||
@ -287,6 +287,9 @@ public:
|
||||
void applySettingChange(const SettingChange & change);
|
||||
void applySettingsChanges(const SettingsChanges & changes);
|
||||
|
||||
/// Update checking that each setting is updatable
|
||||
void updateSettingsChanges(const SettingsChanges & changes);
|
||||
|
||||
/// Checks the constraints.
|
||||
void checkSettingsConstraints(const SettingChange & change);
|
||||
void checkSettingsConstraints(const SettingsChanges & changes);
|
||||
|
||||
@ -1654,7 +1654,6 @@ void InterpreterSelectQuery::executeAggregation(Pipeline & pipeline, const Expre
|
||||
|
||||
Aggregator::Params params(header, keys, aggregates,
|
||||
overflow_row, settings.max_rows_to_group_by, settings.group_by_overflow_mode,
|
||||
settings.compile ? &context.getCompiler() : nullptr, settings.min_count_to_compile,
|
||||
allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold : SettingUInt64(0),
|
||||
allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold_bytes : SettingUInt64(0),
|
||||
settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set,
|
||||
@ -1721,7 +1720,6 @@ void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const
|
||||
|
||||
Aggregator::Params params(header_before_aggregation, keys, aggregates,
|
||||
overflow_row, settings.max_rows_to_group_by, settings.group_by_overflow_mode,
|
||||
settings.compile ? &context.getCompiler() : nullptr, settings.min_count_to_compile,
|
||||
allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold : SettingUInt64(0),
|
||||
allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold_bytes : SettingUInt64(0),
|
||||
settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set,
|
||||
@ -1943,7 +1941,6 @@ void InterpreterSelectQuery::executeRollupOrCube(Pipeline & pipeline, Modificato
|
||||
|
||||
Aggregator::Params params(header, keys, aggregates,
|
||||
false, settings.max_rows_to_group_by, settings.group_by_overflow_mode,
|
||||
settings.compile ? &context.getCompiler() : nullptr, settings.min_count_to_compile,
|
||||
SettingUInt64(0), SettingUInt64(0),
|
||||
settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set,
|
||||
context.getTemporaryPath(), settings.max_threads);
|
||||
@ -1973,7 +1970,6 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPipeline & pipeline, Modif
|
||||
|
||||
Aggregator::Params params(header_before_transform, keys, aggregates,
|
||||
false, settings.max_rows_to_group_by, settings.group_by_overflow_mode,
|
||||
settings.compile ? &context.getCompiler() : nullptr, settings.min_count_to_compile,
|
||||
SettingUInt64(0), SettingUInt64(0),
|
||||
settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set,
|
||||
context.getTemporaryPath(), settings.max_threads);
|
||||
|
||||
@ -10,7 +10,7 @@ BlockIO InterpreterSetQuery::execute()
|
||||
{
|
||||
const auto & ast = query_ptr->as<ASTSetQuery &>();
|
||||
context.checkSettingsConstraints(ast.changes);
|
||||
context.getSessionContext().applySettingsChanges(ast.changes);
|
||||
context.getSessionContext().updateSettingsChanges(ast.changes);
|
||||
return {};
|
||||
}
|
||||
|
||||
@ -19,7 +19,7 @@ void InterpreterSetQuery::executeForCurrentContext()
|
||||
{
|
||||
const auto & ast = query_ptr->as<ASTSetQuery &>();
|
||||
context.checkSettingsConstraints(ast.changes);
|
||||
context.applySettingsChanges(ast.changes);
|
||||
context.updateSettingsChanges(ast.changes);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -30,8 +30,8 @@ bool LogicalExpressionsOptimizer::OrWithExpression::operator<(const OrWithExpres
|
||||
return std::tie(this->or_function, this->expression) < std::tie(rhs.or_function, rhs.expression);
|
||||
}
|
||||
|
||||
LogicalExpressionsOptimizer::LogicalExpressionsOptimizer(ASTSelectQuery * select_query_, ExtractedSettings && settings_)
|
||||
: select_query(select_query_), settings(settings_)
|
||||
LogicalExpressionsOptimizer::LogicalExpressionsOptimizer(ASTSelectQuery * select_query_, UInt64 optimize_min_equality_disjunction_chain_length)
|
||||
: select_query(select_query_), settings(optimize_min_equality_disjunction_chain_length)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@ -36,7 +36,7 @@ class LogicalExpressionsOptimizer final
|
||||
|
||||
public:
|
||||
/// Constructor. Accepts the root of the query DAG.
|
||||
LogicalExpressionsOptimizer(ASTSelectQuery * select_query_, ExtractedSettings && settings_);
|
||||
LogicalExpressionsOptimizer(ASTSelectQuery * select_query_, UInt64 optimize_min_equality_disjunction_chain_length);
|
||||
|
||||
/** Replace all rather long homogeneous OR-chains expr = x1 OR ... OR expr = xN
|
||||
* on the expressions `expr` IN (x1, ..., xN).
|
||||
|
||||
@ -1,215 +0,0 @@
|
||||
#include <Common/TypeList.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Interpreters/Aggregator.h>
|
||||
#include <AggregateFunctions/AggregateFunctionCount.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
/** An aggregation loop template that allows you to generate a custom variant for a specific combination of aggregate functions.
|
||||
* It differs from the usual one in that calls to aggregate functions should be inlined, and the update loop of the aggregate functions should be unrolled.
|
||||
*
|
||||
* Since there are too many possible combinations, it is not possible to generate them all in advance.
|
||||
* This template is intended to instantiate it in runtime,
|
||||
* by running the compiler, compiling shared library, and using it with `dlopen`.
|
||||
*/
|
||||
|
||||
|
||||
struct AggregateFunctionsUpdater
|
||||
{
|
||||
AggregateFunctionsUpdater(
|
||||
const Aggregator::AggregateFunctionsPlainPtrs & aggregate_functions_,
|
||||
const Sizes & offsets_of_aggregate_states_,
|
||||
Aggregator::AggregateColumns & aggregate_columns_,
|
||||
AggregateDataPtr & value_,
|
||||
size_t row_num_,
|
||||
Arena * arena_)
|
||||
: aggregate_functions(aggregate_functions_),
|
||||
offsets_of_aggregate_states(offsets_of_aggregate_states_),
|
||||
aggregate_columns(aggregate_columns_),
|
||||
value(value_), row_num(row_num_), arena(arena_)
|
||||
{
|
||||
}
|
||||
|
||||
template <typename AggregateFunction, size_t column_num>
|
||||
void operator()() ALWAYS_INLINE;
|
||||
|
||||
const Aggregator::AggregateFunctionsPlainPtrs & aggregate_functions;
|
||||
const Sizes & offsets_of_aggregate_states;
|
||||
Aggregator::AggregateColumns & aggregate_columns;
|
||||
AggregateDataPtr & value;
|
||||
size_t row_num;
|
||||
Arena * arena;
|
||||
};
|
||||
|
||||
template <typename AggregateFunction, size_t column_num>
|
||||
void AggregateFunctionsUpdater::operator()()
|
||||
{
|
||||
static_cast<AggregateFunction *>(aggregate_functions[column_num])->add(
|
||||
value + offsets_of_aggregate_states[column_num],
|
||||
aggregate_columns[column_num].data(),
|
||||
row_num, arena);
|
||||
}
|
||||
|
||||
struct AggregateFunctionsCreator
|
||||
{
|
||||
AggregateFunctionsCreator(
|
||||
const Aggregator::AggregateFunctionsPlainPtrs & aggregate_functions_,
|
||||
const Sizes & offsets_of_aggregate_states_,
|
||||
AggregateDataPtr & aggregate_data_)
|
||||
: aggregate_functions(aggregate_functions_),
|
||||
offsets_of_aggregate_states(offsets_of_aggregate_states_),
|
||||
aggregate_data(aggregate_data_)
|
||||
{
|
||||
}
|
||||
|
||||
template <typename AggregateFunction, size_t column_num>
|
||||
void operator()() ALWAYS_INLINE;
|
||||
|
||||
const Aggregator::AggregateFunctionsPlainPtrs & aggregate_functions;
|
||||
const Sizes & offsets_of_aggregate_states;
|
||||
AggregateDataPtr & aggregate_data;
|
||||
};
|
||||
|
||||
template <typename AggregateFunction, size_t column_num>
|
||||
void AggregateFunctionsCreator::operator()()
|
||||
{
|
||||
AggregateFunction * func = static_cast<AggregateFunction *>(aggregate_functions[column_num]);
|
||||
|
||||
try
|
||||
{
|
||||
/** An exception may occur if there is a shortage of memory.
|
||||
* To ensure that everything is properly destroyed, we "roll back" some of the created states.
|
||||
* The code is not very convenient.
|
||||
*/
|
||||
func->create(aggregate_data + offsets_of_aggregate_states[column_num]);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
for (size_t rollback_j = 0; rollback_j < column_num; ++rollback_j)
|
||||
func->destroy(aggregate_data + offsets_of_aggregate_states[rollback_j]);
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
template <typename Method, typename AggregateFunctionsList>
|
||||
void NO_INLINE Aggregator::executeSpecialized(
|
||||
Method & method,
|
||||
Arena * aggregates_pool,
|
||||
size_t rows,
|
||||
ColumnRawPtrs & key_columns,
|
||||
AggregateColumns & aggregate_columns,
|
||||
bool no_more_keys,
|
||||
AggregateDataPtr overflow_row) const
|
||||
{
|
||||
typename Method::State state(key_columns, key_sizes, aggregation_state_cache);
|
||||
|
||||
if (!no_more_keys)
|
||||
executeSpecializedCase<false, Method, AggregateFunctionsList>(
|
||||
method, state, aggregates_pool, rows, aggregate_columns, overflow_row);
|
||||
else
|
||||
executeSpecializedCase<true, Method, AggregateFunctionsList>(
|
||||
method, state, aggregates_pool, rows, aggregate_columns, overflow_row);
|
||||
}
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wuninitialized"
|
||||
|
||||
template <bool no_more_keys, typename Method, typename AggregateFunctionsList>
|
||||
void NO_INLINE Aggregator::executeSpecializedCase(
|
||||
Method & method,
|
||||
typename Method::State & state,
|
||||
Arena * aggregates_pool,
|
||||
size_t rows,
|
||||
AggregateColumns & aggregate_columns,
|
||||
AggregateDataPtr overflow_row) const
|
||||
{
|
||||
/// For all rows.
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
{
|
||||
AggregateDataPtr aggregate_data = nullptr;
|
||||
|
||||
if (!no_more_keys) /// Insert.
|
||||
{
|
||||
auto emplace_result = state.emplaceKey(method.data, i, *aggregates_pool);
|
||||
|
||||
/// If a new key is inserted, initialize the states of the aggregate functions, and possibly something related to the key.
|
||||
if (emplace_result.isInserted())
|
||||
{
|
||||
/// exception-safety - if you can not allocate memory or create states, then destructors will not be called.
|
||||
emplace_result.setMapped(nullptr);
|
||||
|
||||
aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states);
|
||||
AggregateFunctionsList::forEach(AggregateFunctionsCreator(
|
||||
aggregate_functions, offsets_of_aggregate_states, aggregate_data));
|
||||
|
||||
emplace_result.setMapped(aggregate_data);
|
||||
}
|
||||
else
|
||||
aggregate_data = emplace_result.getMapped();
|
||||
}
|
||||
else
|
||||
{
|
||||
/// Add only if the key already exists.
|
||||
auto find_result = state.findKey(method.data, i, *aggregates_pool);
|
||||
if (find_result.isFound())
|
||||
aggregate_data = find_result.getMapped();
|
||||
}
|
||||
|
||||
/// If the key does not fit, and the data does not need to be aggregated in a separate row, then there's nothing to do.
|
||||
if (!aggregate_data && !overflow_row)
|
||||
continue;
|
||||
|
||||
auto value = aggregate_data ? aggregate_data : overflow_row;
|
||||
|
||||
/// Add values into the aggregate functions.
|
||||
AggregateFunctionsList::forEach(AggregateFunctionsUpdater(
|
||||
aggregate_functions, offsets_of_aggregate_states, aggregate_columns, value, i, aggregates_pool));
|
||||
}
|
||||
}
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
template <typename AggregateFunctionsList>
|
||||
void NO_INLINE Aggregator::executeSpecializedWithoutKey(
|
||||
AggregatedDataWithoutKey & res,
|
||||
size_t rows,
|
||||
AggregateColumns & aggregate_columns,
|
||||
Arena * arena) const
|
||||
{
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
{
|
||||
AggregateFunctionsList::forEach(AggregateFunctionsUpdater(
|
||||
aggregate_functions, offsets_of_aggregate_states, aggregate_columns, res, i, arena));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
/** The main code is compiled with gcc 7.
|
||||
* But SpecializedAggregator is compiled using clang 6 into the .so file.
|
||||
* This is done because gcc can not get functions inlined,
|
||||
* which were de-virtualized, in a particular case, and the performance is lower.
|
||||
* And also it's easier to distribute clang for deploy to the servers.
|
||||
*
|
||||
* After switching from gcc 4.8 and gnu++1x to gcc 4.9 and gnu++1y (and then to gcc 5),
|
||||
* an error occurred with `dlopen`: undefined symbol: __cxa_pure_virtual
|
||||
*
|
||||
* Most likely, this is due to the changed version of this symbol:
|
||||
* gcc creates a symbol in .so
|
||||
* U __cxa_pure_virtual@@CXXABI_1.3
|
||||
* but clang creates a symbol
|
||||
* U __cxa_pure_virtual
|
||||
*
|
||||
* But it does not matter for us how the __cxa_pure_virtual function will be implemented,
|
||||
* because it is not called during normal program execution,
|
||||
* and if called - then the program is guaranteed buggy.
|
||||
*
|
||||
* Therefore, we can work around the problem this way
|
||||
*/
|
||||
extern "C" void __attribute__((__visibility__("default"), __noreturn__)) __cxa_pure_virtual() { abort(); }
|
||||
@ -251,7 +251,7 @@ void ThreadStatus::logToQueryThreadLog(QueryThreadLog & thread_log)
|
||||
{
|
||||
elem.client_info = query_context->getClientInfo();
|
||||
|
||||
if (query_context->getSettingsRef().log_profile_events.value != 0)
|
||||
if (query_context->getSettingsRef().log_profile_events != 0)
|
||||
{
|
||||
/// NOTE: Here we are in the same thread, so we can make memcpy()
|
||||
elem.profile_counters = std::make_shared<ProfileEvents::Counters>(performance_counters.getPartiallyAtomicSnapshot());
|
||||
|
||||
@ -1,26 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#cmakedefine CMAKE_LIBRARY_ARCHITECTURE "@CMAKE_LIBRARY_ARCHITECTURE@"
|
||||
#cmakedefine PATH_SHARE "@PATH_SHARE@"
|
||||
#cmakedefine INTERNAL_COMPILER_FLAGS "@INTERNAL_COMPILER_FLAGS@"
|
||||
#cmakedefine INTERNAL_COMPILER_BIN_ROOT "@INTERNAL_COMPILER_BIN_ROOT@"
|
||||
#cmakedefine INTERNAL_LINKER_EXECUTABLE "@INTERNAL_LINKER_EXECUTABLE@"
|
||||
#cmakedefine INTERNAL_COMPILER_EXECUTABLE "@INTERNAL_COMPILER_EXECUTABLE@"
|
||||
#cmakedefine INTERNAL_COMPILER_ENV "@INTERNAL_COMPILER_ENV@"
|
||||
#if !defined(INTERNAL_COMPILER_ENV)
|
||||
# define INTERNAL_COMPILER_ENV ""
|
||||
#endif
|
||||
#cmakedefine INTERNAL_COMPILER_HEADERS "@INTERNAL_COMPILER_HEADERS@"
|
||||
#if !defined(INTERNAL_COMPILER_HEADERS)
|
||||
# define INTERNAL_COMPILER_HEADERS ""
|
||||
#endif
|
||||
#cmakedefine INTERNAL_COMPILER_HEADERS_ROOT "@INTERNAL_COMPILER_HEADERS_ROOT@"
|
||||
#if !defined(INTERNAL_COMPILER_HEADERS_ROOT)
|
||||
# define INTERNAL_COMPILER_HEADERS_ROOT ""
|
||||
#endif
|
||||
|
||||
#cmakedefine01 INTERNAL_COMPILER_CUSTOM_ROOT
|
||||
#cmakedefine INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR "@INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR@"
|
||||
#cmakedefine INTERNAL_Poco_Foundation_INCLUDE_DIR "@INTERNAL_Poco_Foundation_INCLUDE_DIR@"
|
||||
#cmakedefine INTERNAL_Poco_Util_INCLUDE_DIR "@INTERNAL_Poco_Util_INCLUDE_DIR@"
|
||||
#cmakedefine INTERNAL_Boost_INCLUDE_DIRS "@INTERNAL_Boost_INCLUDE_DIRS@"
|
||||
@ -41,9 +41,6 @@ add_executable (two_level_hash_map two_level_hash_map.cpp)
|
||||
target_include_directories (two_level_hash_map SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
|
||||
target_link_libraries (two_level_hash_map PRIVATE dbms)
|
||||
|
||||
add_executable (compiler_test compiler_test.cpp)
|
||||
target_link_libraries (compiler_test PRIVATE dbms)
|
||||
|
||||
add_executable (logical_expressions_optimizer logical_expressions_optimizer.cpp)
|
||||
target_link_libraries (logical_expressions_optimizer PRIVATE dbms clickhouse_parsers)
|
||||
|
||||
|
||||
@ -79,7 +79,7 @@ int main(int argc, char ** argv)
|
||||
|
||||
Aggregator::Params params(
|
||||
stream->getHeader(), {0, 1}, aggregate_descriptions,
|
||||
false, 0, OverflowMode::THROW, nullptr, 0, 0, 0, 0, false, "", 1);
|
||||
false, 0, OverflowMode::THROW, 0, 0, 0, false, "", 1);
|
||||
|
||||
Aggregator aggregator(params);
|
||||
|
||||
|
||||
@ -1,57 +0,0 @@
|
||||
#include <Poco/ConsoleChannel.h>
|
||||
#include <Poco/AutoPtr.h>
|
||||
|
||||
#include <Interpreters/Compiler.h>
|
||||
|
||||
|
||||
int main(int, char **)
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
Poco::AutoPtr<Poco::ConsoleChannel> channel = new Poco::ConsoleChannel(std::cerr);
|
||||
Logger::root().setChannel(channel);
|
||||
Logger::root().setLevel("trace");
|
||||
|
||||
/// Check exception handling and catching
|
||||
try
|
||||
{
|
||||
Compiler compiler(".", 1);
|
||||
|
||||
auto lib = compiler.getOrCount("catch_me_if_you_can", 0, "", []() -> std::string
|
||||
{
|
||||
return
|
||||
"#include <iostream>\n"
|
||||
"void f() __attribute__((__visibility__(\"default\")));\n"
|
||||
"void f()"
|
||||
"{"
|
||||
"try { throw std::runtime_error(\"Catch me if you can\"); }"
|
||||
"catch (const std::runtime_error & e) { std::cout << \"Caught in .so: \" << e.what() << std::endl; throw; }\n"
|
||||
"}"
|
||||
;
|
||||
}, [](SharedLibraryPtr &){});
|
||||
|
||||
auto f = lib->template get<void (*)()>("_Z1fv");
|
||||
|
||||
try
|
||||
{
|
||||
f();
|
||||
}
|
||||
catch (const std::exception & e)
|
||||
{
|
||||
std::cout << "Caught in main(): " << e.what() << "\n";
|
||||
return 0;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
std::cout << "Unknown exception\n";
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
std::cerr << getCurrentExceptionMessage(true) << "\n";
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -45,6 +45,11 @@ ASTPtr ASTAlterCommand::clone() const
|
||||
res->ttl = ttl->clone();
|
||||
res->children.push_back(res->ttl);
|
||||
}
|
||||
if (settings_changes)
|
||||
{
|
||||
res->settings_changes = settings_changes->clone();
|
||||
res->children.push_back(res->settings_changes);
|
||||
}
|
||||
if (values)
|
||||
{
|
||||
res->values = values->clone();
|
||||
@ -222,6 +227,11 @@ void ASTAlterCommand::formatImpl(
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "MODIFY TTL " << (settings.hilite ? hilite_none : "");
|
||||
ttl->formatImpl(settings, state, frame);
|
||||
}
|
||||
else if (type == ASTAlterCommand::MODIFY_SETTING)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "MODIFY SETTING " << (settings.hilite ? hilite_none : "");
|
||||
settings_changes->formatImpl(settings, state, frame);
|
||||
}
|
||||
else if (type == ASTAlterCommand::LIVE_VIEW_REFRESH)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "REFRESH " << (settings.hilite ? hilite_none : "");
|
||||
|
||||
@ -30,6 +30,7 @@ public:
|
||||
COMMENT_COLUMN,
|
||||
MODIFY_ORDER_BY,
|
||||
MODIFY_TTL,
|
||||
MODIFY_SETTING,
|
||||
|
||||
ADD_INDEX,
|
||||
DROP_INDEX,
|
||||
@ -107,6 +108,9 @@ public:
|
||||
/// For MODIFY TTL query
|
||||
ASTPtr ttl;
|
||||
|
||||
/// FOR MODIFY_SETTING
|
||||
ASTPtr settings_changes;
|
||||
|
||||
/** In ALTER CHANNEL, ADD, DROP, SUSPEND, RESUME, REFRESH, MODIFY queries, the list of live views is stored here
|
||||
*/
|
||||
ASTPtr values;
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include <Parsers/ExpressionListParsers.h>
|
||||
#include <Parsers/ParserCreateQuery.h>
|
||||
#include <Parsers/ParserPartition.h>
|
||||
#include <Parsers/ParserSetQuery.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/ASTIndexDeclaration.h>
|
||||
#include <Parsers/ASTAlterQuery.h>
|
||||
@ -28,6 +29,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
ParserKeyword s_comment_column("COMMENT COLUMN");
|
||||
ParserKeyword s_modify_order_by("MODIFY ORDER BY");
|
||||
ParserKeyword s_modify_ttl("MODIFY TTL");
|
||||
ParserKeyword s_modify_setting("MODIFY SETTING");
|
||||
|
||||
ParserKeyword s_add_index("ADD INDEX");
|
||||
ParserKeyword s_drop_index("DROP INDEX");
|
||||
@ -78,6 +80,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
ParserList parser_assignment_list(
|
||||
std::make_unique<ParserAssignment>(), std::make_unique<ParserToken>(TokenType::Comma),
|
||||
/* allow_empty = */ false);
|
||||
ParserSetQuery parser_settings(true);
|
||||
ParserNameList values_p;
|
||||
|
||||
if (is_live_view)
|
||||
@ -386,8 +389,15 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
return false;
|
||||
command->type = ASTAlterCommand::MODIFY_TTL;
|
||||
}
|
||||
else if (s_modify_setting.ignore(pos, expected))
|
||||
{
|
||||
if (!parser_settings.parse(pos, command->settings_changes, expected))
|
||||
return false;
|
||||
command->type = ASTAlterCommand::MODIFY_SETTING;
|
||||
}
|
||||
else
|
||||
return false;
|
||||
|
||||
}
|
||||
|
||||
if (command->col_decl)
|
||||
@ -408,6 +418,8 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
command->children.push_back(command->comment);
|
||||
if (command->ttl)
|
||||
command->children.push_back(command->ttl);
|
||||
if (command->settings_changes)
|
||||
command->children.push_back(command->settings_changes);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -13,6 +13,7 @@ namespace DB
|
||||
* [CLEAR COLUMN [IF EXISTS] col_to_clear [IN PARTITION partition],]
|
||||
* [MODIFY COLUMN [IF EXISTS] col_to_modify type, ...]
|
||||
* [MODIFY PRIMARY KEY (a, b, c...)]
|
||||
* [MODIFY SETTING setting_name=setting_value, ...]
|
||||
* [COMMENT COLUMN [IF EXISTS] col_name string]
|
||||
* [DROP|DETACH|ATTACH PARTITION|PART partition, ...]
|
||||
* [FETCH PARTITION partition FROM ...]
|
||||
|
||||
@ -12,7 +12,7 @@ namespace DB
|
||||
|
||||
ProtobufRowInputFormat::ProtobufRowInputFormat(ReadBuffer & in_, const Block & header_, Params params_, const FormatSchemaInfo & info_)
|
||||
: IRowInputFormat(header_, in_, params_)
|
||||
, data_types(header_.getDataTypes())
|
||||
, data_types(header_.getDataTypes())
|
||||
, reader(in, ProtobufSchemas::instance().getMessageTypeForFormatSchema(info_), header_.getNames())
|
||||
{
|
||||
}
|
||||
|
||||
@ -224,8 +224,6 @@ try
|
||||
overflow_row,
|
||||
max_rows_to_group_by,
|
||||
OverflowMode::THROW,
|
||||
nullptr, /// No compiler
|
||||
0, /// min_count_to_compile
|
||||
group_by_two_level_threshold,
|
||||
group_by_two_level_threshold_bytes,
|
||||
max_bytes_before_external_group_by,
|
||||
@ -298,8 +296,6 @@ try
|
||||
overflow_row,
|
||||
max_rows_to_group_by,
|
||||
OverflowMode::THROW,
|
||||
nullptr, /// No compiler
|
||||
0, /// min_count_to_compile
|
||||
group_by_two_level_threshold,
|
||||
group_by_two_level_threshold_bytes,
|
||||
max_bytes_before_external_group_by,
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTAlterQuery.h>
|
||||
#include <Parsers/ASTColumnDeclaration.h>
|
||||
#include <Parsers/ASTSetQuery.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Compression/CompressionFactory.h>
|
||||
|
||||
@ -29,6 +30,7 @@ namespace ErrorCodes
|
||||
extern const int ILLEGAL_COLUMN;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int UNKNOWN_SETTING;
|
||||
}
|
||||
|
||||
|
||||
@ -199,14 +201,21 @@ std::optional<AlterCommand> AlterCommand::parse(const ASTAlterCommand * command_
|
||||
command.ttl = command_ast->ttl;
|
||||
return command;
|
||||
}
|
||||
else if (command_ast->type == ASTAlterCommand::MODIFY_SETTING)
|
||||
{
|
||||
AlterCommand command;
|
||||
command.type = AlterCommand::MODIFY_SETTING;
|
||||
command.settings_changes = command_ast->settings_changes->as<ASTSetQuery &>().changes;
|
||||
return command;
|
||||
}
|
||||
else
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescription & indices_description,
|
||||
ConstraintsDescription & constraints_description,
|
||||
ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const
|
||||
ConstraintsDescription & constraints_description, ASTPtr & order_by_ast, ASTPtr & primary_key_ast,
|
||||
ASTPtr & ttl_table_ast, SettingsChanges & changes) const
|
||||
{
|
||||
if (type == ADD_COLUMN)
|
||||
{
|
||||
@ -373,23 +382,31 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri
|
||||
{
|
||||
ttl_table_ast = ttl;
|
||||
}
|
||||
else if (type == MODIFY_SETTING)
|
||||
{
|
||||
changes.insert(changes.end(), settings_changes.begin(), settings_changes.end());
|
||||
}
|
||||
else
|
||||
throw Exception("Wrong parameter type in ALTER query", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
bool AlterCommand::isMutable() const
|
||||
{
|
||||
if (type == COMMENT_COLUMN)
|
||||
if (type == COMMENT_COLUMN || type == MODIFY_SETTING)
|
||||
return false;
|
||||
if (type == MODIFY_COLUMN)
|
||||
return data_type.get() || default_expression;
|
||||
// TODO: возможно, здесь нужно дополнить
|
||||
return true;
|
||||
}
|
||||
|
||||
bool AlterCommand::isSettingsAlter() const
|
||||
{
|
||||
return type == MODIFY_SETTING;
|
||||
}
|
||||
|
||||
void AlterCommands::apply(ColumnsDescription & columns_description, IndicesDescription & indices_description,
|
||||
ConstraintsDescription & constraints_description,
|
||||
ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const
|
||||
ConstraintsDescription & constraints_description, ASTPtr & order_by_ast, ASTPtr & primary_key_ast,
|
||||
ASTPtr & ttl_table_ast, SettingsChanges & changes) const
|
||||
{
|
||||
auto new_columns_description = columns_description;
|
||||
auto new_indices_description = indices_description;
|
||||
@ -397,10 +414,11 @@ void AlterCommands::apply(ColumnsDescription & columns_description, IndicesDescr
|
||||
auto new_order_by_ast = order_by_ast;
|
||||
auto new_primary_key_ast = primary_key_ast;
|
||||
auto new_ttl_table_ast = ttl_table_ast;
|
||||
auto new_changes = changes;
|
||||
|
||||
for (const AlterCommand & command : *this)
|
||||
if (!command.ignore)
|
||||
command.apply(new_columns_description, new_indices_description, new_constraints_description, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast);
|
||||
command.apply(new_columns_description, new_indices_description, new_constraints_description, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast, new_changes);
|
||||
|
||||
columns_description = std::move(new_columns_description);
|
||||
indices_description = std::move(new_indices_description);
|
||||
@ -408,6 +426,7 @@ void AlterCommands::apply(ColumnsDescription & columns_description, IndicesDescr
|
||||
order_by_ast = std::move(new_order_by_ast);
|
||||
primary_key_ast = std::move(new_primary_key_ast);
|
||||
ttl_table_ast = std::move(new_ttl_table_ast);
|
||||
changes = std::move(new_changes);
|
||||
}
|
||||
|
||||
void AlterCommands::validate(const IStorage & table, const Context & context)
|
||||
@ -523,6 +542,16 @@ void AlterCommands::validate(const IStorage & table, const Context & context)
|
||||
throw Exception{"Wrong column name. Cannot find column " + command.column_name + " to comment", ErrorCodes::ILLEGAL_COLUMN};
|
||||
}
|
||||
}
|
||||
else if (command.type == AlterCommand::MODIFY_SETTING)
|
||||
{
|
||||
for (const auto & change : command.settings_changes)
|
||||
{
|
||||
if (!table.hasSetting(change.name))
|
||||
{
|
||||
throw Exception{"Storage '" + table.getName() + "' doesn't have setting '" + change.name + "'", ErrorCodes::UNKNOWN_SETTING};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Existing defaulted columns may require default expression extensions with a type conversion,
|
||||
@ -588,7 +617,7 @@ void AlterCommands::validate(const IStorage & table, const Context & context)
|
||||
}
|
||||
}
|
||||
|
||||
void AlterCommands::apply(ColumnsDescription & columns_description) const
|
||||
void AlterCommands::applyForColumnsOnly(ColumnsDescription & columns_description) const
|
||||
{
|
||||
auto out_columns_description = columns_description;
|
||||
IndicesDescription indices_description;
|
||||
@ -596,7 +625,9 @@ void AlterCommands::apply(ColumnsDescription & columns_description) const
|
||||
ASTPtr out_order_by;
|
||||
ASTPtr out_primary_key;
|
||||
ASTPtr out_ttl_table;
|
||||
apply(out_columns_description, indices_description, constraints_description, out_order_by, out_primary_key, out_ttl_table);
|
||||
SettingsChanges out_changes;
|
||||
apply(out_columns_description, indices_description, constraints_description,
|
||||
out_order_by, out_primary_key, out_ttl_table, out_changes);
|
||||
|
||||
if (out_order_by)
|
||||
throw Exception("Storage doesn't support modifying ORDER BY expression", ErrorCodes::NOT_IMPLEMENTED);
|
||||
@ -608,10 +639,40 @@ void AlterCommands::apply(ColumnsDescription & columns_description) const
|
||||
throw Exception("Storage doesn't support modifying constraints", ErrorCodes::NOT_IMPLEMENTED);
|
||||
if (out_ttl_table)
|
||||
throw Exception("Storage doesn't support modifying TTL expression", ErrorCodes::NOT_IMPLEMENTED);
|
||||
if (!out_changes.empty())
|
||||
throw Exception("Storage doesn't support modifying settings", ErrorCodes::NOT_IMPLEMENTED);
|
||||
|
||||
|
||||
columns_description = std::move(out_columns_description);
|
||||
}
|
||||
|
||||
|
||||
void AlterCommands::applyForSettingsOnly(SettingsChanges & changes) const
|
||||
{
|
||||
ColumnsDescription out_columns_description;
|
||||
IndicesDescription indices_description;
|
||||
ConstraintsDescription constraints_description;
|
||||
ASTPtr out_order_by;
|
||||
ASTPtr out_primary_key;
|
||||
ASTPtr out_ttl_table;
|
||||
SettingsChanges out_changes;
|
||||
apply(out_columns_description, indices_description, constraints_description, out_order_by,
|
||||
out_primary_key, out_ttl_table, out_changes);
|
||||
|
||||
if (out_columns_description.begin() != out_columns_description.end())
|
||||
throw Exception("Alter modifying columns, but only settings change applied.", ErrorCodes::LOGICAL_ERROR);
|
||||
if (out_order_by)
|
||||
throw Exception("Alter modifying ORDER BY expression, but only settings change applied.", ErrorCodes::LOGICAL_ERROR);
|
||||
if (out_primary_key)
|
||||
throw Exception("Alter modifying PRIMARY KEY expression, but only settings change applied.", ErrorCodes::LOGICAL_ERROR);
|
||||
if (!indices_description.indices.empty())
|
||||
throw Exception("Alter modifying indices, but only settings change applied.", ErrorCodes::NOT_IMPLEMENTED);
|
||||
if (out_ttl_table)
|
||||
throw Exception("Alter modifying TTL, but only settings change applied.", ErrorCodes::NOT_IMPLEMENTED);
|
||||
|
||||
changes = std::move(out_changes);
|
||||
}
|
||||
|
||||
bool AlterCommands::isMutable() const
|
||||
{
|
||||
for (const auto & param : *this)
|
||||
@ -623,4 +684,8 @@ bool AlterCommands::isMutable() const
|
||||
return false;
|
||||
}
|
||||
|
||||
bool AlterCommands::isSettingsAlter() const
|
||||
{
|
||||
return std::all_of(begin(), end(), [](const AlterCommand & c) { return c.isSettingsAlter(); });
|
||||
}
|
||||
}
|
||||
|
||||
@ -7,6 +7,8 @@
|
||||
#include <Storages/IndicesDescription.h>
|
||||
#include <Storages/ConstraintsDescription.h>
|
||||
|
||||
#include <Common/SettingsChanges.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -30,6 +32,7 @@ struct AlterCommand
|
||||
DROP_CONSTRAINT,
|
||||
MODIFY_TTL,
|
||||
UKNOWN_TYPE,
|
||||
MODIFY_SETTING,
|
||||
};
|
||||
|
||||
Type type = UKNOWN_TYPE;
|
||||
@ -80,6 +83,9 @@ struct AlterCommand
|
||||
/// For ADD and MODIFY
|
||||
CompressionCodecPtr codec;
|
||||
|
||||
/// For MODIFY SETTING
|
||||
SettingsChanges settings_changes;
|
||||
|
||||
AlterCommand() = default;
|
||||
AlterCommand(const Type type_, const String & column_name_, const DataTypePtr & data_type_,
|
||||
const ColumnDefaultKind default_kind_, const ASTPtr & default_expression_,
|
||||
@ -93,11 +99,14 @@ struct AlterCommand
|
||||
static std::optional<AlterCommand> parse(const ASTAlterCommand * command);
|
||||
|
||||
void apply(ColumnsDescription & columns_description, IndicesDescription & indices_description,
|
||||
ConstraintsDescription & constraints_description,
|
||||
ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const;
|
||||
ConstraintsDescription & constraints_description, ASTPtr & order_by_ast,
|
||||
ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast, SettingsChanges & changes) const;
|
||||
|
||||
/// Checks that not only metadata touched by that command
|
||||
bool isMutable() const;
|
||||
|
||||
/// checks that only settings changed by alter
|
||||
bool isSettingsAlter() const;
|
||||
};
|
||||
|
||||
class Context;
|
||||
@ -105,15 +114,18 @@ class Context;
|
||||
class AlterCommands : public std::vector<AlterCommand>
|
||||
{
|
||||
public:
|
||||
/// Used for primitive table engines, where only columns metadata can be changed
|
||||
void applyForColumnsOnly(ColumnsDescription & columns_description) const;
|
||||
void apply(ColumnsDescription & columns_description, IndicesDescription & indices_description,
|
||||
ConstraintsDescription & constraints_description,
|
||||
ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const;
|
||||
ConstraintsDescription & constraints_description, ASTPtr & order_by_ast, ASTPtr & primary_key_ast,
|
||||
ASTPtr & ttl_table_ast, SettingsChanges & changes) const;
|
||||
|
||||
/// For storages that don't support MODIFY_ORDER_BY.
|
||||
void apply(ColumnsDescription & columns_description) const;
|
||||
/// Apply alter commands only for settings. Exception will be thrown if any other part of table structure will be modified.
|
||||
void applyForSettingsOnly(SettingsChanges & changes) const;
|
||||
|
||||
void validate(const IStorage & table, const Context & context);
|
||||
bool isMutable() const;
|
||||
bool isSettingsAlter() const;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
#include <Storages/IStorage.h>
|
||||
|
||||
#include <Storages/AlterCommands.h>
|
||||
#include <Parsers/ASTCreateQuery.h>
|
||||
#include <Parsers/ASTSetQuery.h>
|
||||
|
||||
#include <sparsehash/dense_hash_map>
|
||||
#include <sparsehash/dense_hash_set>
|
||||
@ -18,6 +20,8 @@ namespace ErrorCodes
|
||||
extern const int NO_SUCH_COLUMN_IN_TABLE;
|
||||
extern const int NOT_FOUND_COLUMN_IN_BLOCK;
|
||||
extern const int TYPE_MISMATCH;
|
||||
extern const int SETTINGS_ARE_NOT_SUPPORTED;
|
||||
extern const int UNKNOWN_SETTING;
|
||||
}
|
||||
|
||||
IStorage::IStorage(ColumnsDescription columns_)
|
||||
@ -316,6 +320,13 @@ bool IStorage::isVirtualColumn(const String & column_name) const
|
||||
return getColumns().get(column_name).is_virtual;
|
||||
}
|
||||
|
||||
bool IStorage::hasSetting(const String & /* setting_name */) const
|
||||
{
|
||||
if (!supportsSettings())
|
||||
throw Exception("Storage '" + getName() + "' doesn't support settings.", ErrorCodes::SETTINGS_ARE_NOT_SUPPORTED);
|
||||
return false;
|
||||
}
|
||||
|
||||
TableStructureReadLockHolder IStorage::lockStructureForShare(bool will_add_new_data, const String & query_id)
|
||||
{
|
||||
TableStructureReadLockHolder result;
|
||||
@ -370,6 +381,39 @@ TableStructureWriteLockHolder IStorage::lockExclusively(const String & query_id)
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
void IStorage::alterSettings(
|
||||
const SettingsChanges & new_changes,
|
||||
const String & current_database_name,
|
||||
const String & current_table_name,
|
||||
const Context & context,
|
||||
TableStructureWriteLockHolder & /* table_lock_holder */)
|
||||
{
|
||||
IDatabase::ASTModifier storage_modifier = [&] (IAST & ast)
|
||||
{
|
||||
if (!new_changes.empty())
|
||||
{
|
||||
auto & storage_changes = ast.as<ASTStorage &>().settings->changes;
|
||||
/// Make storage settings unique
|
||||
for (const auto & change : new_changes)
|
||||
{
|
||||
if (hasSetting(change.name))
|
||||
{
|
||||
auto finder = [&change] (const SettingChange & c) { return c.name == change.name; };
|
||||
if (auto it = std::find_if(storage_changes.begin(), storage_changes.end(), finder); it != storage_changes.end())
|
||||
it->value = change.value;
|
||||
else
|
||||
storage_changes.push_back(change);
|
||||
}
|
||||
else
|
||||
throw Exception{"Storage '" + getName() + "' doesn't have setting '" + change.name + "'", ErrorCodes::UNKNOWN_SETTING};
|
||||
}
|
||||
}
|
||||
};
|
||||
context.getDatabase(current_database_name)->alterTable(context, current_table_name, getColumns(), getIndices(), getConstraints(), storage_modifier);
|
||||
}
|
||||
|
||||
|
||||
void IStorage::alter(
|
||||
const AlterCommands & params,
|
||||
const String & database_name,
|
||||
@ -377,17 +421,22 @@ void IStorage::alter(
|
||||
const Context & context,
|
||||
TableStructureWriteLockHolder & table_lock_holder)
|
||||
{
|
||||
for (const auto & param : params)
|
||||
if (params.isSettingsAlter())
|
||||
{
|
||||
if (param.isMutable())
|
||||
throw Exception("Method alter supports only change comment of column for storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
|
||||
SettingsChanges new_changes;
|
||||
params.applyForSettingsOnly(new_changes);
|
||||
alterSettings(new_changes, database_name, table_name, context, table_lock_holder);
|
||||
return;
|
||||
}
|
||||
|
||||
if (params.isMutable())
|
||||
throw Exception("Method alter supports only change comment of column for storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
|
||||
|
||||
lockStructureExclusively(table_lock_holder, context.getCurrentQueryId());
|
||||
auto new_columns = getColumns();
|
||||
auto new_indices = getIndices();
|
||||
auto new_constraints = getConstraints();
|
||||
params.apply(new_columns);
|
||||
params.applyForColumnsOnly(new_columns);
|
||||
context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, new_constraints, {});
|
||||
setColumns(std::move(new_columns));
|
||||
}
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
#include <Common/ActionLock.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/RWLock.h>
|
||||
#include <Common/SettingsChanges.h>
|
||||
#include <Storages/ConstraintsDescription.h>
|
||||
|
||||
#include <optional>
|
||||
@ -97,6 +98,9 @@ public:
|
||||
/// Returns true if the storage supports deduplication of inserted data blocks.
|
||||
virtual bool supportsDeduplication() const { return false; }
|
||||
|
||||
/// Returns true if the storage supports settings.
|
||||
virtual bool supportsSettings() const { return false; }
|
||||
|
||||
/// Optional size information of each physical column.
|
||||
/// Currently it's only used by the MergeTree family for query optimizations.
|
||||
using ColumnSizeByName = std::unordered_map<std::string, ColumnSize>;
|
||||
@ -136,13 +140,15 @@ public: /// thread-unsafe part. lockStructure must be acquired
|
||||
/// If |need_all| is set, then checks that all the columns of the table are in the block.
|
||||
void check(const Block & block, bool need_all = false) const;
|
||||
|
||||
/// Check storage has setting. Exception will be thrown if it doesn't support settings at all.
|
||||
virtual bool hasSetting(const String & setting_name) const;
|
||||
|
||||
protected: /// still thread-unsafe part.
|
||||
void setIndices(IndicesDescription indices_);
|
||||
|
||||
/// Returns whether the column is virtual - by default all columns are real.
|
||||
/// Initially reserved virtual column name may be shadowed by real column.
|
||||
virtual bool isVirtualColumn(const String & column_name) const;
|
||||
|
||||
private:
|
||||
ColumnsDescription columns; /// combined real and virtual columns
|
||||
const ColumnsDescription virtuals = {};
|
||||
@ -287,6 +293,15 @@ public:
|
||||
throw Exception("Partition operations are not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
/** ALTER table settings if possible. Otherwise throws exception.
|
||||
*/
|
||||
virtual void alterSettings(
|
||||
const SettingsChanges & new_changes,
|
||||
const String & current_database_name,
|
||||
const String & current_table_name,
|
||||
const Context & context,
|
||||
TableStructureWriteLockHolder & table_lock_holder);
|
||||
|
||||
/** Perform any background work. For example, combining parts in a MergeTree type table.
|
||||
* Returns whether any work has been done.
|
||||
*/
|
||||
|
||||
@ -22,7 +22,7 @@ void KafkaSettings::loadFromQuery(ASTStorage & storage_def)
|
||||
{
|
||||
try
|
||||
{
|
||||
applyChanges(storage_def.settings->changes);
|
||||
loadFromChanges(storage_def.settings->changes);
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
|
||||
@ -14,7 +14,9 @@ class ASTStorage;
|
||||
struct KafkaSettings : public SettingsCollection<KafkaSettings>
|
||||
{
|
||||
|
||||
#define LIST_OF_KAFKA_SETTINGS(M) \
|
||||
|
||||
/// M (mutable) for normal settings, IM (immutable) for not updateable settings.
|
||||
#define LIST_OF_KAFKA_SETTINGS(M, IM) \
|
||||
M(SettingString, kafka_broker_list, "", "A comma-separated list of brokers for Kafka engine.") \
|
||||
M(SettingString, kafka_topic_list, "", "A list of Kafka topics.") \
|
||||
M(SettingString, kafka_group_name, "", "A group of Kafka consumers.") \
|
||||
|
||||
@ -43,6 +43,7 @@ namespace ErrorCodes
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int UNSUPPORTED_METHOD;
|
||||
}
|
||||
|
||||
namespace
|
||||
@ -367,7 +368,7 @@ bool StorageKafka::streamToViews()
|
||||
const Settings & settings = global_context.getSettingsRef();
|
||||
size_t block_size = max_block_size;
|
||||
if (block_size == 0)
|
||||
block_size = settings.max_block_size.value;
|
||||
block_size = settings.max_block_size;
|
||||
|
||||
// Create a stream for each consumer and join them in a union stream
|
||||
InterpreterInsertQuery interpreter{insert, global_context};
|
||||
@ -406,6 +407,22 @@ bool StorageKafka::streamToViews()
|
||||
}
|
||||
|
||||
|
||||
bool StorageKafka::hasSetting(const String & setting_name) const
|
||||
{
|
||||
return KafkaSettings::findIndex(setting_name) != KafkaSettings::npos;
|
||||
}
|
||||
|
||||
void StorageKafka::alterSettings(
|
||||
const SettingsChanges & /* new_changes */,
|
||||
const String & /* current_database_name */,
|
||||
const String & /* current_table_name */,
|
||||
const Context & /* context */,
|
||||
TableStructureWriteLockHolder & /* table_lock_holder */)
|
||||
{
|
||||
throw Exception("Storage '" + getName() + "' doesn't support settings alter", ErrorCodes::UNSUPPORTED_METHOD);
|
||||
}
|
||||
|
||||
|
||||
void registerStorageKafka(StorageFactory & factory)
|
||||
{
|
||||
factory.registerStorage("Kafka", [](const StorageFactory::Arguments & args)
|
||||
@ -470,7 +487,7 @@ void registerStorageKafka(StorageFactory & factory)
|
||||
#undef CHECK_KAFKA_STORAGE_ARGUMENT
|
||||
|
||||
// Get and check broker list
|
||||
String brokers = kafka_settings.kafka_broker_list.value;
|
||||
String brokers = kafka_settings.kafka_broker_list;
|
||||
if (args_count >= 1)
|
||||
{
|
||||
const auto * ast = engine_args[0]->as<ASTLiteral>();
|
||||
@ -525,7 +542,7 @@ void registerStorageKafka(StorageFactory & factory)
|
||||
}
|
||||
|
||||
// Parse row delimiter (optional)
|
||||
char row_delimiter = kafka_settings.kafka_row_delimiter.value;
|
||||
char row_delimiter = kafka_settings.kafka_row_delimiter;
|
||||
if (args_count >= 5)
|
||||
{
|
||||
engine_args[4] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[4], args.local_context);
|
||||
@ -572,7 +589,7 @@ void registerStorageKafka(StorageFactory & factory)
|
||||
}
|
||||
|
||||
// Parse number of consumers (optional)
|
||||
UInt64 num_consumers = kafka_settings.kafka_num_consumers.value;
|
||||
UInt64 num_consumers = kafka_settings.kafka_num_consumers;
|
||||
if (args_count >= 7)
|
||||
{
|
||||
const auto * ast = engine_args[6]->as<ASTLiteral>();
|
||||
@ -587,7 +604,7 @@ void registerStorageKafka(StorageFactory & factory)
|
||||
}
|
||||
|
||||
// Parse max block size (optional)
|
||||
UInt64 max_block_size = static_cast<size_t>(kafka_settings.kafka_max_block_size.value);
|
||||
UInt64 max_block_size = static_cast<size_t>(kafka_settings.kafka_max_block_size);
|
||||
if (args_count >= 8)
|
||||
{
|
||||
const auto * ast = engine_args[7]->as<ASTLiteral>();
|
||||
@ -602,7 +619,7 @@ void registerStorageKafka(StorageFactory & factory)
|
||||
}
|
||||
}
|
||||
|
||||
size_t skip_broken = static_cast<size_t>(kafka_settings.kafka_skip_broken_messages.value);
|
||||
size_t skip_broken = static_cast<size_t>(kafka_settings.kafka_skip_broken_messages);
|
||||
if (args_count >= 9)
|
||||
{
|
||||
const auto * ast = engine_args[8]->as<ASTLiteral>();
|
||||
|
||||
@ -24,6 +24,7 @@ public:
|
||||
std::string getName() const override { return "Kafka"; }
|
||||
std::string getTableName() const override { return table_name; }
|
||||
std::string getDatabaseName() const override { return database_name; }
|
||||
bool supportsSettings() const override { return true; }
|
||||
|
||||
void startup() override;
|
||||
void shutdown() override;
|
||||
@ -56,6 +57,15 @@ public:
|
||||
const auto & getSchemaName() const { return schema_name; }
|
||||
const auto & skipBroken() const { return skip_broken; }
|
||||
|
||||
bool hasSetting(const String & setting_name) const override;
|
||||
|
||||
void alterSettings(
|
||||
const SettingsChanges & new_changes,
|
||||
const String & current_database_name,
|
||||
const String & current_table_name,
|
||||
const Context & context,
|
||||
TableStructureWriteLockHolder & table_lock_holder) override;
|
||||
|
||||
protected:
|
||||
StorageKafka(
|
||||
const std::string & table_name_,
|
||||
|
||||
@ -54,14 +54,15 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo
|
||||
throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED);
|
||||
|
||||
String part_name = params.get("part");
|
||||
const auto data_settings = data.getCOWSettings();
|
||||
|
||||
/// Validation of the input that may come from malicious replica.
|
||||
MergeTreePartInfo::fromPartName(part_name, data.format_version);
|
||||
|
||||
static std::atomic_uint total_sends {0};
|
||||
|
||||
if ((data.settings.replicated_max_parallel_sends && total_sends >= data.settings.replicated_max_parallel_sends)
|
||||
|| (data.settings.replicated_max_parallel_sends_for_table && data.current_table_sends >= data.settings.replicated_max_parallel_sends_for_table))
|
||||
if ((data_settings->replicated_max_parallel_sends && total_sends >= data_settings->replicated_max_parallel_sends)
|
||||
|| (data_settings->replicated_max_parallel_sends_for_table && data.current_table_sends >= data_settings->replicated_max_parallel_sends_for_table))
|
||||
{
|
||||
response.setStatus(std::to_string(HTTP_TOO_MANY_REQUESTS));
|
||||
response.setReason("Too many concurrent fetches, try again later");
|
||||
@ -174,6 +175,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart(
|
||||
{
|
||||
/// Validation of the input that may come from malicious replica.
|
||||
MergeTreePartInfo::fromPartName(part_name, data.format_version);
|
||||
const auto data_settings = data.getCOWSettings();
|
||||
|
||||
Poco::URI uri;
|
||||
uri.setScheme(interserver_scheme);
|
||||
@ -200,7 +202,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart(
|
||||
timeouts,
|
||||
creds,
|
||||
DBMS_DEFAULT_BUFFER_SIZE,
|
||||
data.settings.replicated_max_parallel_fetches_for_host
|
||||
data_settings->replicated_max_parallel_fetches_for_host
|
||||
};
|
||||
|
||||
static const String TMP_PREFIX = "tmp_fetch_";
|
||||
|
||||
@ -39,7 +39,7 @@ IMergedBlockOutputStream::IMergedBlockOutputStream(
|
||||
, compute_granularity(index_granularity.empty())
|
||||
, codec(std::move(codec_))
|
||||
, skip_indices(indices_to_recalc)
|
||||
, with_final_mark(storage.settings.write_final_mark && can_use_adaptive_granularity)
|
||||
, with_final_mark(storage.getCOWSettings()->write_final_mark && can_use_adaptive_granularity)
|
||||
{
|
||||
if (blocks_are_granules_size && !index_granularity.empty())
|
||||
throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR);
|
||||
@ -139,10 +139,11 @@ void fillIndexGranularityImpl(
|
||||
|
||||
void IMergedBlockOutputStream::fillIndexGranularity(const Block & block)
|
||||
{
|
||||
const auto storage_settings = storage.getCOWSettings();
|
||||
fillIndexGranularityImpl(
|
||||
block,
|
||||
storage.settings.index_granularity_bytes,
|
||||
storage.settings.index_granularity,
|
||||
storage_settings->index_granularity_bytes,
|
||||
storage_settings->index_granularity,
|
||||
blocks_are_granules_size,
|
||||
index_offset,
|
||||
index_granularity,
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTPartition.h>
|
||||
#include <Parsers/ASTSetQuery.h>
|
||||
#include <Parsers/ExpressionListParsers.h>
|
||||
#include <Parsers/parseQuery.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
@ -89,6 +90,7 @@ namespace ErrorCodes
|
||||
extern const int BAD_TTL_EXPRESSION;
|
||||
extern const int INCORRECT_FILE_NAME;
|
||||
extern const int BAD_DATA_PART_NAME;
|
||||
extern const int UNKNOWN_SETTING;
|
||||
}
|
||||
|
||||
|
||||
@ -105,13 +107,12 @@ MergeTreeData::MergeTreeData(
|
||||
const ASTPtr & sample_by_ast_,
|
||||
const ASTPtr & ttl_table_ast_,
|
||||
const MergingParams & merging_params_,
|
||||
const MergeTreeSettings & settings_,
|
||||
MergeTreeSettingsPtr settings_,
|
||||
bool require_part_metadata_,
|
||||
bool attach,
|
||||
BrokenPartCallback broken_part_callback_)
|
||||
: global_context(context_),
|
||||
merging_params(merging_params_),
|
||||
settings(settings_),
|
||||
partition_by_ast(partition_by_ast_),
|
||||
sample_by_ast(sample_by_ast_),
|
||||
ttl_table_ast(ttl_table_ast_),
|
||||
@ -120,9 +121,11 @@ MergeTreeData::MergeTreeData(
|
||||
full_path(full_path_),
|
||||
broken_part_callback(broken_part_callback_),
|
||||
log_name(database_name + "." + table_name), log(&Logger::get(log_name)),
|
||||
guarded_settings(settings_),
|
||||
data_parts_by_info(data_parts_indexes.get<TagByInfo>()),
|
||||
data_parts_by_state_and_info(data_parts_indexes.get<TagByStateAndInfo>())
|
||||
{
|
||||
const auto settings = getCOWSettings();
|
||||
setProperties(order_by_ast_, primary_key_ast_, columns_, indices_, constraints_);
|
||||
setConstraints(constraints_);
|
||||
|
||||
@ -134,7 +137,7 @@ MergeTreeData::MergeTreeData(
|
||||
sampling_expr_column_name = sample_by_ast->getColumnName();
|
||||
|
||||
if (!primary_key_sample.has(sampling_expr_column_name)
|
||||
&& !attach && !settings.compatibility_allow_sampling_expression_not_in_primary_key) /// This is for backward compatibility.
|
||||
&& !attach && !settings->compatibility_allow_sampling_expression_not_in_primary_key) /// This is for backward compatibility.
|
||||
throw Exception("Sampling expression must be present in the primary key", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
auto syntax = SyntaxAnalyzer(global_context).analyze(sample_by_ast, getColumns().getAllPhysical());
|
||||
@ -730,6 +733,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
|
||||
{
|
||||
LOG_DEBUG(log, "Loading data parts");
|
||||
|
||||
const auto settings = getCOWSettings();
|
||||
Strings part_file_names;
|
||||
Poco::DirectoryIterator end;
|
||||
for (Poco::DirectoryIterator it(full_path); it != end; ++it)
|
||||
@ -751,7 +755,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
|
||||
}
|
||||
|
||||
/// Parallel loading of data parts.
|
||||
size_t num_threads = std::min(size_t(settings.max_part_loading_threads), part_file_names.size());
|
||||
size_t num_threads = std::min(size_t(settings->max_part_loading_threads), part_file_names.size());
|
||||
|
||||
std::mutex mutex;
|
||||
|
||||
@ -870,12 +874,12 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
|
||||
|
||||
pool.wait();
|
||||
|
||||
if (has_non_adaptive_parts && has_adaptive_parts && !settings.enable_mixed_granularity_parts)
|
||||
if (has_non_adaptive_parts && has_adaptive_parts && !settings->enable_mixed_granularity_parts)
|
||||
throw Exception("Table contains parts with adaptive and non adaptive marks, but `setting enable_mixed_granularity_parts` is disabled", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
has_non_adaptive_index_granularity_parts = has_non_adaptive_parts;
|
||||
|
||||
if (suspicious_broken_parts > settings.max_suspicious_broken_parts && !skip_sanity_checks)
|
||||
if (suspicious_broken_parts > settings->max_suspicious_broken_parts && !skip_sanity_checks)
|
||||
throw Exception("Suspiciously many (" + toString(suspicious_broken_parts) + ") broken parts to remove.",
|
||||
ErrorCodes::TOO_MANY_UNEXPECTED_DATA_PARTS);
|
||||
|
||||
@ -962,10 +966,11 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life
|
||||
if (!lock.try_lock())
|
||||
return;
|
||||
|
||||
const auto settings = getCOWSettings();
|
||||
time_t current_time = time(nullptr);
|
||||
ssize_t deadline = (custom_directories_lifetime_seconds >= 0)
|
||||
? current_time - custom_directories_lifetime_seconds
|
||||
: current_time - settings.temporary_directories_lifetime.totalSeconds();
|
||||
: current_time - settings->temporary_directories_lifetime.totalSeconds();
|
||||
|
||||
/// Delete temporary directories older than a day.
|
||||
Poco::DirectoryIterator end;
|
||||
@ -1016,7 +1021,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts()
|
||||
|
||||
if (part.unique() && /// Grab only parts that are not used by anyone (SELECTs for example).
|
||||
part_remove_time < now &&
|
||||
now - part_remove_time > settings.old_parts_lifetime.totalSeconds())
|
||||
now - part_remove_time > getCOWSettings()->old_parts_lifetime.totalSeconds())
|
||||
{
|
||||
parts_to_delete.emplace_back(it);
|
||||
}
|
||||
@ -1100,11 +1105,12 @@ void MergeTreeData::clearOldPartsFromFilesystem()
|
||||
|
||||
void MergeTreeData::clearPartsFromFilesystem(const DataPartsVector & parts_to_remove)
|
||||
{
|
||||
if (parts_to_remove.size() > 1 && settings.max_part_removal_threads > 1 && parts_to_remove.size() > settings.concurrent_part_removal_threshold)
|
||||
const auto settings = getCOWSettings();
|
||||
if (parts_to_remove.size() > 1 && settings->max_part_removal_threads > 1 && parts_to_remove.size() > settings->concurrent_part_removal_threshold)
|
||||
{
|
||||
/// Parallel parts removal.
|
||||
|
||||
size_t num_threads = std::min(size_t(settings.max_part_removal_threads), parts_to_remove.size());
|
||||
size_t num_threads = std::min(size_t(settings->max_part_removal_threads), parts_to_remove.size());
|
||||
ThreadPool pool(num_threads);
|
||||
|
||||
/// NOTE: Under heavy system load you may get "Cannot schedule a task" from ThreadPool.
|
||||
@ -1230,7 +1236,8 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c
|
||||
ASTPtr new_order_by_ast = order_by_ast;
|
||||
ASTPtr new_primary_key_ast = primary_key_ast;
|
||||
ASTPtr new_ttl_table_ast = ttl_table_ast;
|
||||
commands.apply(new_columns, new_indices, new_constraints, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast);
|
||||
SettingsChanges new_changes;
|
||||
commands.apply(new_columns, new_indices, new_constraints, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast, new_changes);
|
||||
if (getIndices().empty() && !new_indices.empty() &&
|
||||
!context.getSettingsRef().allow_experimental_data_skipping_indices)
|
||||
throw Exception("You must set the setting `allow_experimental_data_skipping_indices` to 1 " \
|
||||
@ -1317,6 +1324,12 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c
|
||||
|
||||
setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast, /* only_check = */ true);
|
||||
|
||||
for (const auto & setting : new_changes)
|
||||
{
|
||||
if (!hasSetting(setting.name))
|
||||
throw Exception{"Storage '" + getName() + "' doesn't have setting '" + setting.name + "'", ErrorCodes::UNKNOWN_SETTING};
|
||||
}
|
||||
|
||||
/// Check that type conversions are possible.
|
||||
ExpressionActionsPtr unused_expression;
|
||||
NameToNameMap unused_map;
|
||||
@ -1329,6 +1342,7 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name
|
||||
const IndicesASTs & old_indices, const IndicesASTs & new_indices, ExpressionActionsPtr & out_expression,
|
||||
NameToNameMap & out_rename_map, bool & out_force_update_metadata) const
|
||||
{
|
||||
const auto settings = getCOWSettings();
|
||||
out_expression = nullptr;
|
||||
out_rename_map = {};
|
||||
out_force_update_metadata = false;
|
||||
@ -1336,7 +1350,7 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name
|
||||
if (part)
|
||||
part_mrk_file_extension = part->index_granularity_info.marks_file_extension;
|
||||
else
|
||||
part_mrk_file_extension = settings.index_granularity_bytes == 0 ? getNonAdaptiveMrkExtension() : getAdaptiveMrkExtension();
|
||||
part_mrk_file_extension = settings->index_granularity_bytes == 0 ? getNonAdaptiveMrkExtension() : getAdaptiveMrkExtension();
|
||||
|
||||
using NameToType = std::map<String, const IDataType *>;
|
||||
NameToType new_types;
|
||||
@ -1494,6 +1508,7 @@ void MergeTreeData::alterDataPart(
|
||||
bool skip_sanity_checks,
|
||||
AlterDataPartTransactionPtr & transaction)
|
||||
{
|
||||
const auto settings = getCOWSettings();
|
||||
ExpressionActionsPtr expression;
|
||||
const auto & part = transaction->getDataPart();
|
||||
bool force_update_metadata;
|
||||
@ -1509,12 +1524,12 @@ void MergeTreeData::alterDataPart(
|
||||
++num_files_to_remove;
|
||||
|
||||
if (!skip_sanity_checks
|
||||
&& (num_files_to_modify > settings.max_files_to_modify_in_alter_columns
|
||||
|| num_files_to_remove > settings.max_files_to_remove_in_alter_columns))
|
||||
&& (num_files_to_modify > settings->max_files_to_modify_in_alter_columns
|
||||
|| num_files_to_remove > settings->max_files_to_remove_in_alter_columns))
|
||||
{
|
||||
transaction->clear();
|
||||
|
||||
const bool forbidden_because_of_modify = num_files_to_modify > settings.max_files_to_modify_in_alter_columns;
|
||||
const bool forbidden_because_of_modify = num_files_to_modify > settings->max_files_to_modify_in_alter_columns;
|
||||
|
||||
std::stringstream exception_message;
|
||||
exception_message
|
||||
@ -1546,7 +1561,7 @@ void MergeTreeData::alterDataPart(
|
||||
<< " If it is not an error, you could increase merge_tree/"
|
||||
<< (forbidden_because_of_modify ? "max_files_to_modify_in_alter_columns" : "max_files_to_remove_in_alter_columns")
|
||||
<< " parameter in configuration file (current value: "
|
||||
<< (forbidden_because_of_modify ? settings.max_files_to_modify_in_alter_columns : settings.max_files_to_remove_in_alter_columns)
|
||||
<< (forbidden_because_of_modify ? settings->max_files_to_modify_in_alter_columns : settings->max_files_to_remove_in_alter_columns)
|
||||
<< ")";
|
||||
|
||||
throw Exception(exception_message.str(), ErrorCodes::TABLE_DIFFERS_TOO_MUCH);
|
||||
@ -1631,6 +1646,25 @@ void MergeTreeData::alterDataPart(
|
||||
return;
|
||||
}
|
||||
|
||||
void MergeTreeData::alterSettings(
|
||||
const SettingsChanges & new_changes,
|
||||
const String & current_database_name,
|
||||
const String & current_table_name,
|
||||
const Context & context,
|
||||
TableStructureWriteLockHolder & table_lock_holder)
|
||||
{
|
||||
std::lock_guard lock(settings_mutex);
|
||||
MutableMergeTreeSettingsPtr settings = std::move(*guarded_settings.getPtr()).mutate();
|
||||
settings->updateFromChanges(new_changes);
|
||||
IStorage::alterSettings(new_changes, current_database_name, current_table_name, context, table_lock_holder);
|
||||
guarded_settings.setPtr(std::move(settings));
|
||||
}
|
||||
|
||||
bool MergeTreeData::hasSetting(const String & setting_name) const
|
||||
{
|
||||
return MergeTreeSettings::findIndex(setting_name) != MergeTreeSettings::npos;
|
||||
}
|
||||
|
||||
void MergeTreeData::removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr & data_part)
|
||||
{
|
||||
auto & empty_columns = data_part->empty_columns;
|
||||
@ -2307,28 +2341,29 @@ std::optional<Int64> MergeTreeData::getMinPartDataVersion() const
|
||||
}
|
||||
|
||||
|
||||
void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event *until) const
|
||||
void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until) const
|
||||
{
|
||||
const auto settings = getCOWSettings();
|
||||
const size_t parts_count_in_total = getPartsCount();
|
||||
if (parts_count_in_total >= settings.max_parts_in_total)
|
||||
if (parts_count_in_total >= settings->max_parts_in_total)
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::RejectedInserts);
|
||||
throw Exception("Too many parts (" + toString(parts_count_in_total) + ") in all partitions in total. This indicates wrong choice of partition key. The threshold can be modified with 'max_parts_in_total' setting in <merge_tree> element in config.xml or with per-table setting.", ErrorCodes::TOO_MANY_PARTS);
|
||||
}
|
||||
|
||||
const size_t parts_count_in_partition = getMaxPartsCountForPartition();
|
||||
if (parts_count_in_partition < settings.parts_to_delay_insert)
|
||||
if (parts_count_in_partition < settings->parts_to_delay_insert)
|
||||
return;
|
||||
|
||||
if (parts_count_in_partition >= settings.parts_to_throw_insert)
|
||||
if (parts_count_in_partition >= settings->parts_to_throw_insert)
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::RejectedInserts);
|
||||
throw Exception("Too many parts (" + toString(parts_count_in_partition) + "). Merges are processing significantly slower than inserts.", ErrorCodes::TOO_MANY_PARTS);
|
||||
}
|
||||
|
||||
const size_t max_k = settings.parts_to_throw_insert - settings.parts_to_delay_insert; /// always > 0
|
||||
const size_t k = 1 + parts_count_in_partition - settings.parts_to_delay_insert; /// from 1 to max_k
|
||||
const double delay_milliseconds = ::pow(settings.max_delay_to_insert * 1000, static_cast<double>(k) / max_k);
|
||||
const size_t max_k = settings->parts_to_throw_insert - settings->parts_to_delay_insert; /// always > 0
|
||||
const size_t k = 1 + parts_count_in_partition - settings->parts_to_delay_insert; /// from 1 to max_k
|
||||
const double delay_milliseconds = ::pow(settings->max_delay_to_insert * 1000, static_cast<double>(k) / max_k);
|
||||
|
||||
ProfileEvents::increment(ProfileEvents::DelayedInserts);
|
||||
ProfileEvents::increment(ProfileEvents::DelayedInsertsMilliseconds, delay_milliseconds);
|
||||
@ -2346,8 +2381,9 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event *until) const
|
||||
|
||||
void MergeTreeData::throwInsertIfNeeded() const
|
||||
{
|
||||
const auto settings = getCOWSettings();
|
||||
const size_t parts_count_in_total = getPartsCount();
|
||||
if (parts_count_in_total >= settings.max_parts_in_total)
|
||||
if (parts_count_in_total >= settings->max_parts_in_total)
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::RejectedInserts);
|
||||
throw Exception("Too many parts (" + toString(parts_count_in_total) + ") in all partitions in total. This indicates wrong choice of partition key. The threshold can be modified with 'max_parts_in_total' setting in <merge_tree> element in config.xml or with per-table setting.", ErrorCodes::TOO_MANY_PARTS);
|
||||
@ -2355,7 +2391,7 @@ void MergeTreeData::throwInsertIfNeeded() const
|
||||
|
||||
const size_t parts_count_in_partition = getMaxPartsCountForPartition();
|
||||
|
||||
if (parts_count_in_partition >= settings.parts_to_throw_insert)
|
||||
if (parts_count_in_partition >= settings->parts_to_throw_insert)
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::RejectedInserts);
|
||||
throw Exception("Too many parts (" + toString(parts_count_in_partition) + "). Merges are processing significantly slower than inserts.", ErrorCodes::TOO_MANY_PARTS);
|
||||
@ -3040,7 +3076,9 @@ void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String &
|
||||
|
||||
bool MergeTreeData::canReplacePartition(const DataPartPtr & src_part) const
|
||||
{
|
||||
if (!settings.enable_mixed_granularity_parts || settings.index_granularity_bytes == 0)
|
||||
const auto settings = getCOWSettings();
|
||||
|
||||
if (!settings->enable_mixed_granularity_parts || settings->index_granularity_bytes == 0)
|
||||
{
|
||||
if (!canUseAdaptiveGranularity() && src_part->index_granularity_info.is_adaptive)
|
||||
return false;
|
||||
|
||||
@ -331,7 +331,7 @@ public:
|
||||
const ASTPtr & sample_by_ast_, /// nullptr, if sampling is not supported.
|
||||
const ASTPtr & ttl_table_ast_,
|
||||
const MergingParams & merging_params_,
|
||||
const MergeTreeSettings & settings_,
|
||||
MergeTreeSettingsPtr settings_,
|
||||
bool require_part_metadata_,
|
||||
bool attach,
|
||||
BrokenPartCallback broken_part_callback_ = [](const String &){});
|
||||
@ -360,6 +360,8 @@ public:
|
||||
|| merging_params.mode == MergingParams::VersionedCollapsing;
|
||||
}
|
||||
|
||||
bool supportsSettings() const override { return true; }
|
||||
|
||||
bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context &) const override;
|
||||
|
||||
NameAndTypePair getColumn(const String & column_name) const override
|
||||
@ -535,6 +537,18 @@ public:
|
||||
bool skip_sanity_checks,
|
||||
AlterDataPartTransactionPtr& transaction);
|
||||
|
||||
/// Performs ALTER of table settings (MergeTreeSettings). Lightweight operation, affects metadata only.
|
||||
/// Not atomic, have to be done with alter intention lock.
|
||||
void alterSettings(
|
||||
const SettingsChanges & new_changes,
|
||||
const String & current_database_name,
|
||||
const String & current_table_name,
|
||||
const Context & context,
|
||||
TableStructureWriteLockHolder & table_lock_holder) override;
|
||||
|
||||
/// All MergeTreeData children have settings.
|
||||
bool hasSetting(const String & setting_name) const override;
|
||||
|
||||
/// Remove columns, that have been markedd as empty after zeroing values with expired ttl
|
||||
void removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr & data_part);
|
||||
|
||||
@ -606,8 +620,9 @@ public:
|
||||
/// Has additional constraint in replicated version
|
||||
virtual bool canUseAdaptiveGranularity() const
|
||||
{
|
||||
return settings.index_granularity_bytes != 0 &&
|
||||
(settings.enable_mixed_granularity_parts || !has_non_adaptive_index_granularity_parts);
|
||||
const auto settings = getCOWSettings();
|
||||
return settings->index_granularity_bytes != 0 &&
|
||||
(settings->enable_mixed_granularity_parts || !has_non_adaptive_index_granularity_parts);
|
||||
}
|
||||
|
||||
|
||||
@ -660,8 +675,6 @@ public:
|
||||
String sampling_expr_column_name;
|
||||
Names columns_required_for_sampling;
|
||||
|
||||
MergeTreeSettings settings;
|
||||
|
||||
/// Limiting parallel sends per one table, used in DataPartsExchange
|
||||
std::atomic_uint current_table_sends {0};
|
||||
|
||||
@ -670,7 +683,17 @@ public:
|
||||
|
||||
bool has_non_adaptive_index_granularity_parts = false;
|
||||
|
||||
/// Get copy-on-write pointer to storage settings.
|
||||
/// Copy this pointer into your scope and you will
|
||||
/// get consistent settings.
|
||||
const MergeTreeSettingsPtr getCOWSettings() const
|
||||
{
|
||||
std::shared_lock lock(settings_mutex);
|
||||
return guarded_settings.copyPtr();
|
||||
}
|
||||
|
||||
protected:
|
||||
|
||||
friend struct MergeTreeDataPart;
|
||||
friend class MergeTreeDataMergerMutator;
|
||||
friend class ReplicatedMergeTreeAlterThread;
|
||||
@ -698,6 +721,26 @@ protected:
|
||||
String log_name;
|
||||
Logger * log;
|
||||
|
||||
/// Just hides settings pointer from direct usage
|
||||
class MergeTreeSettingsGuard
|
||||
{
|
||||
private:
|
||||
/// Settings COW pointer. Data maybe changed at any point of time.
|
||||
/// If you need consistent settings, just copy pointer to your scope.
|
||||
MergeTreeSettingsPtr settings_ptr;
|
||||
public:
|
||||
MergeTreeSettingsGuard(MergeTreeSettingsPtr settings_ptr_)
|
||||
: settings_ptr(settings_ptr_)
|
||||
{}
|
||||
|
||||
const MergeTreeSettingsPtr copyPtr() const { return settings_ptr; }
|
||||
MergeTreeSettingsPtr getPtr() { return settings_ptr; }
|
||||
void setPtr(MergeTreeSettingsPtr ptr) { settings_ptr = ptr; }
|
||||
};
|
||||
|
||||
/// Storage settings. Don't use this field directly, if you
|
||||
/// want readonly settings. Prefer getCOWSettings() method.
|
||||
MergeTreeSettingsGuard guarded_settings;
|
||||
|
||||
/// Work with data parts
|
||||
|
||||
@ -785,6 +828,8 @@ protected:
|
||||
std::mutex grab_old_parts_mutex;
|
||||
/// The same for clearOldTemporaryDirectories.
|
||||
std::mutex clear_old_temporary_directories_mutex;
|
||||
/// Mutex for settings usage
|
||||
mutable std::shared_mutex settings_mutex;
|
||||
|
||||
void setProperties(const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast,
|
||||
const ColumnsDescription & new_columns,
|
||||
|
||||
@ -141,15 +141,16 @@ UInt64 MergeTreeDataMergerMutator::getMaxSourcePartsSizeForMerge(size_t pool_siz
|
||||
throw Exception("Logical error: invalid arguments passed to getMaxSourcePartsSize: pool_used > pool_size", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
size_t free_entries = pool_size - pool_used;
|
||||
const auto data_settings = data.getCOWSettings();
|
||||
|
||||
UInt64 max_size = 0;
|
||||
if (free_entries >= data.settings.number_of_free_entries_in_pool_to_lower_max_size_of_merge)
|
||||
max_size = data.settings.max_bytes_to_merge_at_max_space_in_pool;
|
||||
if (free_entries >= data_settings->number_of_free_entries_in_pool_to_lower_max_size_of_merge)
|
||||
max_size = data_settings->max_bytes_to_merge_at_max_space_in_pool;
|
||||
else
|
||||
max_size = interpolateExponential(
|
||||
data.settings.max_bytes_to_merge_at_min_space_in_pool,
|
||||
data.settings.max_bytes_to_merge_at_max_space_in_pool,
|
||||
static_cast<double>(free_entries) / data.settings.number_of_free_entries_in_pool_to_lower_max_size_of_merge);
|
||||
data_settings->max_bytes_to_merge_at_min_space_in_pool,
|
||||
data_settings->max_bytes_to_merge_at_max_space_in_pool,
|
||||
static_cast<double>(free_entries) / data_settings->number_of_free_entries_in_pool_to_lower_max_size_of_merge);
|
||||
|
||||
return std::min(max_size, static_cast<UInt64>(DiskSpaceMonitor::getUnreservedFreeSpace(data.full_path) / DISK_USAGE_COEFFICIENT_TO_SELECT));
|
||||
}
|
||||
@ -157,11 +158,13 @@ UInt64 MergeTreeDataMergerMutator::getMaxSourcePartsSizeForMerge(size_t pool_siz
|
||||
|
||||
UInt64 MergeTreeDataMergerMutator::getMaxSourcePartSizeForMutation()
|
||||
{
|
||||
|
||||
const auto data_settings = data.getCOWSettings();
|
||||
size_t total_threads_in_pool = pool.getNumberOfThreads();
|
||||
size_t busy_threads_in_pool = CurrentMetrics::values[CurrentMetrics::BackgroundPoolTask].load(std::memory_order_relaxed);
|
||||
|
||||
/// Allow mutations only if there are enough threads, leave free threads for merges else
|
||||
if (total_threads_in_pool - busy_threads_in_pool >= data.settings.number_of_free_entries_in_pool_to_execute_mutation)
|
||||
if (total_threads_in_pool - busy_threads_in_pool >= data_settings->number_of_free_entries_in_pool_to_execute_mutation)
|
||||
return static_cast<UInt64>(DiskSpaceMonitor::getUnreservedFreeSpace(data.full_path) / DISK_USAGE_COEFFICIENT_TO_RESERVE);
|
||||
|
||||
return 0;
|
||||
@ -176,6 +179,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge(
|
||||
String * out_disable_reason)
|
||||
{
|
||||
MergeTreeData::DataPartsVector data_parts = data.getDataPartsVector();
|
||||
const auto data_settings = data.getCOWSettings();
|
||||
|
||||
if (data_parts.empty())
|
||||
{
|
||||
@ -230,7 +234,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge(
|
||||
merge_settings.base = 1;
|
||||
|
||||
bool can_merge_with_ttl =
|
||||
(current_time - last_merge_with_ttl > data.settings.merge_with_ttl_timeout);
|
||||
(current_time - last_merge_with_ttl > data_settings->merge_with_ttl_timeout);
|
||||
|
||||
/// NOTE Could allow selection of different merge strategy.
|
||||
if (can_merge_with_ttl && has_part_with_expired_ttl && !ttl_merges_blocker.isCancelled())
|
||||
@ -552,6 +556,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
|
||||
|
||||
Names all_column_names = data.getColumns().getNamesOfPhysical();
|
||||
NamesAndTypesList all_columns = data.getColumns().getAllPhysical();
|
||||
const auto data_settings = data.getCOWSettings();
|
||||
|
||||
NamesAndTypesList gathering_columns, merging_columns;
|
||||
Names gathering_column_names, merging_column_names;
|
||||
@ -624,13 +629,13 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
|
||||
/// We count total amount of bytes in parts
|
||||
/// and use direct_io + aio if there is more than min_merge_bytes_to_use_direct_io
|
||||
bool read_with_direct_io = false;
|
||||
if (data.settings.min_merge_bytes_to_use_direct_io != 0)
|
||||
if (data_settings->min_merge_bytes_to_use_direct_io != 0)
|
||||
{
|
||||
size_t total_size = 0;
|
||||
for (const auto & part : parts)
|
||||
{
|
||||
total_size += part->bytes_on_disk;
|
||||
if (total_size >= data.settings.min_merge_bytes_to_use_direct_io)
|
||||
if (total_size >= data_settings->min_merge_bytes_to_use_direct_io)
|
||||
{
|
||||
LOG_DEBUG(log, "Will merge parts reading files in O_DIRECT");
|
||||
read_with_direct_io = true;
|
||||
@ -727,7 +732,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
|
||||
merging_columns,
|
||||
compression_codec,
|
||||
merged_column_to_size,
|
||||
data.settings.min_merge_bytes_to_use_direct_io,
|
||||
data_settings->min_merge_bytes_to_use_direct_io,
|
||||
blocks_are_granules_size};
|
||||
|
||||
merged_stream->readPrefix();
|
||||
@ -960,6 +965,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor
|
||||
const auto & updated_header = mutations_interpreter.getUpdatedHeader();
|
||||
|
||||
NamesAndTypesList all_columns = data.getColumns().getAllPhysical();
|
||||
const auto data_settings = data.getCOWSettings();
|
||||
|
||||
Block in_header = in->getHeader();
|
||||
|
||||
@ -1034,7 +1040,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor
|
||||
}
|
||||
|
||||
NameSet files_to_skip = {"checksums.txt", "columns.txt"};
|
||||
auto mrk_extension = data.settings.index_granularity_bytes ? getAdaptiveMrkExtension() : getNonAdaptiveMrkExtension();
|
||||
|
||||
auto mrk_extension = data_settings->index_granularity_bytes ? getAdaptiveMrkExtension() : getNonAdaptiveMrkExtension();
|
||||
for (const auto & entry : updated_header)
|
||||
{
|
||||
IDataType::StreamCallback callback = [&](const IDataType::SubstreamPath & substream_path)
|
||||
@ -1138,9 +1145,11 @@ MergeTreeDataMergerMutator::MergeAlgorithm MergeTreeDataMergerMutator::chooseMer
|
||||
const MergeTreeData::DataPartsVector & parts, size_t sum_rows_upper_bound,
|
||||
const NamesAndTypesList & gathering_columns, bool deduplicate, bool need_remove_expired_values) const
|
||||
{
|
||||
const auto data_settings = data.getCOWSettings();
|
||||
|
||||
if (deduplicate)
|
||||
return MergeAlgorithm::Horizontal;
|
||||
if (data.settings.enable_vertical_merge_algorithm == 0)
|
||||
if (data_settings->enable_vertical_merge_algorithm == 0)
|
||||
return MergeAlgorithm::Horizontal;
|
||||
if (need_remove_expired_values)
|
||||
return MergeAlgorithm::Horizontal;
|
||||
@ -1151,9 +1160,9 @@ MergeTreeDataMergerMutator::MergeAlgorithm MergeTreeDataMergerMutator::chooseMer
|
||||
data.merging_params.mode == MergeTreeData::MergingParams::Replacing ||
|
||||
data.merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing;
|
||||
|
||||
bool enough_ordinary_cols = gathering_columns.size() >= data.settings.vertical_merge_algorithm_min_columns_to_activate;
|
||||
bool enough_ordinary_cols = gathering_columns.size() >= data_settings->vertical_merge_algorithm_min_columns_to_activate;
|
||||
|
||||
bool enough_total_rows = sum_rows_upper_bound >= data.settings.vertical_merge_algorithm_min_rows_to_activate;
|
||||
bool enough_total_rows = sum_rows_upper_bound >= data_settings->vertical_merge_algorithm_min_rows_to_activate;
|
||||
|
||||
bool no_parts_overflow = parts.size() <= RowSourcePart::MAX_PARTS;
|
||||
|
||||
|
||||
@ -673,6 +673,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams(
|
||||
size_t sum_marks = 0;
|
||||
size_t total_rows = 0;
|
||||
|
||||
const auto data_settings = data.getCOWSettings();
|
||||
size_t adaptive_parts = 0;
|
||||
for (size_t i = 0; i < parts.size(); ++i)
|
||||
{
|
||||
@ -689,18 +690,18 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams(
|
||||
|
||||
size_t index_granularity_bytes = 0;
|
||||
if (adaptive_parts > parts.size() / 2)
|
||||
index_granularity_bytes = data.settings.index_granularity_bytes;
|
||||
index_granularity_bytes = data_settings->index_granularity_bytes;
|
||||
|
||||
const size_t max_marks_to_use_cache = roundRowsOrBytesToMarks(
|
||||
settings.merge_tree_max_rows_to_use_cache,
|
||||
settings.merge_tree_max_bytes_to_use_cache,
|
||||
data.settings.index_granularity,
|
||||
data_settings->index_granularity,
|
||||
index_granularity_bytes);
|
||||
|
||||
const size_t min_marks_for_concurrent_read = roundRowsOrBytesToMarks(
|
||||
settings.merge_tree_min_rows_for_concurrent_read,
|
||||
settings.merge_tree_min_bytes_for_concurrent_read,
|
||||
data.settings.index_granularity,
|
||||
data_settings->index_granularity,
|
||||
index_granularity_bytes);
|
||||
|
||||
if (sum_marks > max_marks_to_use_cache)
|
||||
@ -831,6 +832,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithO
|
||||
SortingInfoPtr sorting_info = query_info.sorting_info;
|
||||
size_t adaptive_parts = 0;
|
||||
std::vector<size_t> sum_marks_in_parts(parts.size());
|
||||
const auto data_settings = data.getCOWSettings();
|
||||
|
||||
for (size_t i = 0; i < parts.size(); ++i)
|
||||
{
|
||||
@ -846,18 +848,18 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithO
|
||||
|
||||
size_t index_granularity_bytes = 0;
|
||||
if (adaptive_parts > parts.size() / 2)
|
||||
index_granularity_bytes = data.settings.index_granularity_bytes;
|
||||
index_granularity_bytes = data_settings->index_granularity_bytes;
|
||||
|
||||
const size_t max_marks_to_use_cache = roundRowsOrBytesToMarks(
|
||||
settings.merge_tree_max_rows_to_use_cache,
|
||||
settings.merge_tree_max_bytes_to_use_cache,
|
||||
data.settings.index_granularity,
|
||||
data_settings->index_granularity,
|
||||
index_granularity_bytes);
|
||||
|
||||
const size_t min_marks_for_concurrent_read = roundRowsOrBytesToMarks(
|
||||
settings.merge_tree_min_rows_for_concurrent_read,
|
||||
settings.merge_tree_min_bytes_for_concurrent_read,
|
||||
data.settings.index_granularity,
|
||||
data_settings->index_granularity,
|
||||
index_granularity_bytes);
|
||||
|
||||
if (sum_marks > max_marks_to_use_cache)
|
||||
@ -869,7 +871,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithO
|
||||
return streams;
|
||||
|
||||
/// Let's split ranges to avoid reading much data.
|
||||
auto split_ranges = [rows_granularity = data.settings.index_granularity, max_block_size](const auto & ranges, int direction)
|
||||
auto split_ranges = [rows_granularity = data_settings->index_granularity, max_block_size](const auto & ranges, int direction)
|
||||
{
|
||||
MarkRanges new_ranges;
|
||||
const size_t max_marks_in_range = (max_block_size + rows_granularity - 1) / rows_granularity;
|
||||
@ -1033,6 +1035,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal
|
||||
const Names & virt_columns,
|
||||
const Settings & settings) const
|
||||
{
|
||||
const auto data_settings = data.getCOWSettings();
|
||||
size_t sum_marks = 0;
|
||||
size_t adaptive_parts = 0;
|
||||
for (size_t i = 0; i < parts.size(); ++i)
|
||||
@ -1046,12 +1049,12 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal
|
||||
|
||||
size_t index_granularity_bytes = 0;
|
||||
if (adaptive_parts >= parts.size() / 2)
|
||||
index_granularity_bytes = data.settings.index_granularity_bytes;
|
||||
index_granularity_bytes = data_settings->index_granularity_bytes;
|
||||
|
||||
const size_t max_marks_to_use_cache = roundRowsOrBytesToMarks(
|
||||
settings.merge_tree_max_rows_to_use_cache,
|
||||
settings.merge_tree_max_bytes_to_use_cache,
|
||||
data.settings.index_granularity,
|
||||
data_settings->index_granularity,
|
||||
index_granularity_bytes);
|
||||
|
||||
if (sum_marks > max_marks_to_use_cache)
|
||||
|
||||
@ -168,6 +168,19 @@ const MergeTreeConditionFullText::AtomMap MergeTreeConditionFullText::atom_map
|
||||
return true;
|
||||
}
|
||||
},
|
||||
{
|
||||
"hasToken",
|
||||
[] (RPNElement & out, const Field & value, const MergeTreeIndexFullText & idx)
|
||||
{
|
||||
out.function = RPNElement::FUNCTION_EQUALS;
|
||||
out.bloom_filter = std::make_unique<BloomFilter>(
|
||||
idx.bloom_filter_size, idx.bloom_filter_hashes, idx.seed);
|
||||
|
||||
const auto & str = value.get<String>();
|
||||
stringToBloomFilter(str.c_str(), str.size(), idx.token_extractor_func, *out.bloom_filter);
|
||||
return true;
|
||||
}
|
||||
},
|
||||
{
|
||||
"startsWith",
|
||||
[] (RPNElement & out, const Field & value, const MergeTreeIndexFullText & idx)
|
||||
|
||||
@ -25,12 +25,13 @@ std::optional<std::string> MergeTreeIndexGranularityInfo::getMrkExtensionFromFS(
|
||||
MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo(
|
||||
const MergeTreeData & storage)
|
||||
{
|
||||
fixed_index_granularity = storage.settings.index_granularity;
|
||||
const auto storage_settings = storage.getCOWSettings();
|
||||
fixed_index_granularity = storage_settings->index_granularity;
|
||||
/// Granularity is fixed
|
||||
if (!storage.canUseAdaptiveGranularity())
|
||||
setNonAdaptive();
|
||||
else
|
||||
setAdaptive(storage.settings.index_granularity_bytes);
|
||||
setAdaptive(storage_settings->index_granularity_bytes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -405,25 +405,6 @@ bool MergeTreeIndexConditionSet::operatorFromAST(ASTPtr & node) const
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool checkAtomName(const String & name)
|
||||
{
|
||||
static std::set<String> atoms = {
|
||||
"notEquals",
|
||||
"equals",
|
||||
"less",
|
||||
"greater",
|
||||
"lessOrEquals",
|
||||
"greaterOrEquals",
|
||||
"in",
|
||||
"notIn",
|
||||
"like",
|
||||
"startsWith",
|
||||
"endsWith",
|
||||
"multiSearchAny"
|
||||
};
|
||||
return atoms.find(name) != atoms.end();
|
||||
}
|
||||
|
||||
bool MergeTreeIndexConditionSet::checkASTUseless(const ASTPtr &node, bool atomic) const
|
||||
{
|
||||
if (const auto * func = node->as<ASTFunction>())
|
||||
@ -439,16 +420,14 @@ bool MergeTreeIndexConditionSet::checkASTUseless(const ASTPtr &node, bool atomic
|
||||
return checkASTUseless(args[0], atomic) || checkASTUseless(args[1], atomic);
|
||||
else if (func->name == "not")
|
||||
return checkASTUseless(args[0], atomic);
|
||||
else if (!atomic && checkAtomName(func->name))
|
||||
return checkASTUseless(node, true);
|
||||
else
|
||||
return std::any_of(args.begin(), args.end(),
|
||||
[this, &atomic](const auto & arg) { return checkASTUseless(arg, atomic); });
|
||||
[this](const auto & arg) { return checkASTUseless(arg, true); });
|
||||
}
|
||||
else if (const auto * literal = node->as<ASTLiteral>())
|
||||
return !atomic && literal->value.get<bool>();
|
||||
else if (const auto * identifier = node->as<ASTIdentifier>())
|
||||
return key_columns.find(identifier->getColumnName()) == key_columns.end();
|
||||
return key_columns.find(identifier->getColumnName()) == std::end(key_columns);
|
||||
else
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -108,7 +108,7 @@ size_t MergeTreeRangeReader::DelayedStream::finalize(Block & block)
|
||||
|
||||
|
||||
MergeTreeRangeReader::Stream::Stream(
|
||||
size_t from_mark, size_t to_mark, MergeTreeReader * merge_tree_reader_)
|
||||
size_t from_mark, size_t to_mark, MergeTreeReader * merge_tree_reader_)
|
||||
: current_mark(from_mark), offset_after_current_mark(0)
|
||||
, last_mark(to_mark)
|
||||
, merge_tree_reader(merge_tree_reader_)
|
||||
|
||||
@ -41,7 +41,7 @@ public:
|
||||
{
|
||||
public:
|
||||
DelayedStream() = default;
|
||||
DelayedStream(size_t from_mark, MergeTreeReader * merge_tree_reader);
|
||||
DelayedStream(size_t from_mark, MergeTreeReader * merge_tree_reader);
|
||||
|
||||
/// Read @num_rows rows from @from_mark starting from @offset row
|
||||
/// Returns the number of rows added to block.
|
||||
|
||||
@ -46,7 +46,7 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def)
|
||||
{
|
||||
try
|
||||
{
|
||||
applyChanges(storage_def.settings->changes);
|
||||
loadFromChanges(storage_def.settings->changes);
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
@ -67,7 +67,7 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def)
|
||||
|
||||
#define ADD_IF_ABSENT(NAME) \
|
||||
if (std::find_if(changes.begin(), changes.end(), \
|
||||
[](const SettingChange & c) { return c.name == #NAME; }) \
|
||||
[](const SettingChange & c) { return c.name == #NAME; }) \
|
||||
== changes.end()) \
|
||||
changes.push_back(SettingChange{#NAME, NAME.value});
|
||||
|
||||
@ -75,4 +75,9 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def)
|
||||
#undef ADD_IF_ABSENT
|
||||
}
|
||||
|
||||
|
||||
MergeTreeSettings::MutablePtr MergeTreeSettings::clone() const
|
||||
{
|
||||
return COW::create(*this);
|
||||
}
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
#include <Core/Defines.h>
|
||||
#include <Core/SettingsCommon.h>
|
||||
#include <Common/COW.h>
|
||||
|
||||
|
||||
namespace Poco
|
||||
@ -21,11 +22,14 @@ class ASTStorage;
|
||||
/** Settings for the MergeTree family of engines.
|
||||
* Could be loaded from config or from a CREATE TABLE query (SETTINGS clause).
|
||||
*/
|
||||
struct MergeTreeSettings : public SettingsCollection<MergeTreeSettings>
|
||||
struct MergeTreeSettings : public SettingsCollection<MergeTreeSettings>, public COW<MergeTreeSettings>
|
||||
{
|
||||
|
||||
#define LIST_OF_MERGE_TREE_SETTINGS(M) \
|
||||
M(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.") \
|
||||
friend class COW<MergeTreeSettings>;
|
||||
|
||||
/// M (mutable) for normal settings, IM (immutable) for not updateable settings.
|
||||
#define LIST_OF_MERGE_TREE_SETTINGS(M, IM) \
|
||||
IM(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.") \
|
||||
\
|
||||
/** Merge settings. */ \
|
||||
M(SettingUInt64, max_bytes_to_merge_at_max_space_in_pool, 150ULL * 1024 * 1024 * 1024, "Maximum in total size of parts to merge, when there are maximum free threads in background pool (or entries in replication queue).") \
|
||||
@ -79,7 +83,7 @@ struct MergeTreeSettings : public SettingsCollection<MergeTreeSettings>
|
||||
M(SettingBool, use_minimalistic_part_header_in_zookeeper, false, "Store part header (checksums and columns) in a compact format and a single part znode instead of separate znodes (<part>/columns and <part>/checksums). This can dramatically reduce snapshot size in ZooKeeper. Before enabling check that all replicas support new format.") \
|
||||
M(SettingUInt64, finished_mutations_to_keep, 100, "How many records about mutations that are done to keep. If zero, then keep all of them.") \
|
||||
M(SettingUInt64, min_merge_bytes_to_use_direct_io, 10ULL * 1024 * 1024 * 1024, "Minimal amount of bytes to enable O_DIRECT in merge (0 - disabled).") \
|
||||
M(SettingUInt64, index_granularity_bytes, 10 * 1024 * 1024, "Approximate amount of bytes in single granule (0 - disabled).") \
|
||||
IM(SettingUInt64, index_granularity_bytes, 10 * 1024 * 1024, "Approximate amount of bytes in single granule (0 - disabled).") \
|
||||
M(SettingInt64, merge_with_ttl_timeout, 3600 * 24, "Minimal time in seconds, when merge with TTL can be repeated.") \
|
||||
M(SettingBool, write_final_mark, 1, "Write final mark after end of column (0 - disabled, do nothing if index_granularity_bytes=0)") \
|
||||
M(SettingBool, enable_mixed_granularity_parts, 0, "Enable parts with adaptive and non adaptive granularity") \
|
||||
@ -97,6 +101,14 @@ struct MergeTreeSettings : public SettingsCollection<MergeTreeSettings>
|
||||
|
||||
/// NOTE: will rewrite the AST to add immutable settings.
|
||||
void loadFromQuery(ASTStorage & storage_def);
|
||||
|
||||
MutablePtr clone() const;
|
||||
private:
|
||||
MergeTreeSettings() = default;
|
||||
MergeTreeSettings(const MergeTreeSettings & o) = default;
|
||||
};
|
||||
|
||||
using MergeTreeSettingsPtr = MergeTreeSettings::Ptr;
|
||||
using MutableMergeTreeSettingsPtr = MergeTreeSettings::MutablePtr;
|
||||
|
||||
}
|
||||
|
||||
@ -31,7 +31,7 @@ MergeTreeThreadSelectBlockInputStream::MergeTreeThreadSelectBlockInputStream(
|
||||
/// Maybe it will make sence to add settings `max_block_size_bytes`
|
||||
if (max_block_size_rows && !storage.canUseAdaptiveGranularity())
|
||||
{
|
||||
size_t fixed_index_granularity = storage.settings.index_granularity;
|
||||
size_t fixed_index_granularity = storage.getCOWSettings()->index_granularity;
|
||||
min_marks_to_read = (min_marks_to_read_ * fixed_index_granularity + max_block_size_rows - 1)
|
||||
/ max_block_size_rows * max_block_size_rows / fixed_index_granularity;
|
||||
}
|
||||
|
||||
@ -27,8 +27,9 @@ ReplicatedMergeTreeCleanupThread::ReplicatedMergeTreeCleanupThread(StorageReplic
|
||||
|
||||
void ReplicatedMergeTreeCleanupThread::run()
|
||||
{
|
||||
const auto CLEANUP_SLEEP_MS = storage.settings.cleanup_delay_period * 1000
|
||||
+ std::uniform_int_distribution<UInt64>(0, storage.settings.cleanup_delay_period_random_add * 1000)(rng);
|
||||
auto storage_settings = storage.getCOWSettings();
|
||||
const auto CLEANUP_SLEEP_MS = storage_settings->cleanup_delay_period * 1000
|
||||
+ std::uniform_int_distribution<UInt64>(0, storage_settings->cleanup_delay_period_random_add * 1000)(rng);
|
||||
|
||||
try
|
||||
{
|
||||
@ -74,6 +75,7 @@ void ReplicatedMergeTreeCleanupThread::iterate()
|
||||
void ReplicatedMergeTreeCleanupThread::clearOldLogs()
|
||||
{
|
||||
auto zookeeper = storage.getZooKeeper();
|
||||
auto storage_settings = storage.getCOWSettings();
|
||||
|
||||
Coordination::Stat stat;
|
||||
if (!zookeeper->exists(storage.zookeeper_path + "/log", &stat))
|
||||
@ -82,7 +84,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs()
|
||||
int children_count = stat.numChildren;
|
||||
|
||||
/// We will wait for 1.1 times more records to accumulate than necessary.
|
||||
if (static_cast<double>(children_count) < storage.settings.min_replicated_logs_to_keep * 1.1)
|
||||
if (static_cast<double>(children_count) < storage_settings->min_replicated_logs_to_keep * 1.1)
|
||||
return;
|
||||
|
||||
Strings replicas = zookeeper->getChildren(storage.zookeeper_path + "/replicas", &stat);
|
||||
@ -100,8 +102,8 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs()
|
||||
std::sort(entries.begin(), entries.end());
|
||||
|
||||
String min_saved_record_log_str = entries[
|
||||
entries.size() > storage.settings.max_replicated_logs_to_keep.value
|
||||
? entries.size() - storage.settings.max_replicated_logs_to_keep.value
|
||||
entries.size() > storage_settings->max_replicated_logs_to_keep
|
||||
? entries.size() - storage_settings->max_replicated_logs_to_keep
|
||||
: 0];
|
||||
|
||||
/// Replicas that were marked is_lost but are active.
|
||||
@ -203,7 +205,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs()
|
||||
min_saved_log_pointer = std::min(min_saved_log_pointer, min_log_pointer_lost_candidate);
|
||||
|
||||
/// We will not touch the last `min_replicated_logs_to_keep` records.
|
||||
entries.erase(entries.end() - std::min<UInt64>(entries.size(), storage.settings.min_replicated_logs_to_keep.value), entries.end());
|
||||
entries.erase(entries.end() - std::min<UInt64>(entries.size(), storage_settings->min_replicated_logs_to_keep), entries.end());
|
||||
/// We will not touch records that are no less than `min_saved_log_pointer`.
|
||||
entries.erase(std::lower_bound(entries.begin(), entries.end(), "log-" + padIndex(min_saved_log_pointer)), entries.end());
|
||||
|
||||
@ -285,6 +287,7 @@ struct ReplicatedMergeTreeCleanupThread::NodeWithStat
|
||||
void ReplicatedMergeTreeCleanupThread::clearOldBlocks()
|
||||
{
|
||||
auto zookeeper = storage.getZooKeeper();
|
||||
auto storage_settings = storage.getCOWSettings();
|
||||
|
||||
std::vector<NodeWithStat> timed_blocks;
|
||||
getBlocksSortedByTime(*zookeeper, timed_blocks);
|
||||
@ -294,12 +297,12 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks()
|
||||
|
||||
/// Use ZooKeeper's first node (last according to time) timestamp as "current" time.
|
||||
Int64 current_time = timed_blocks.front().ctime;
|
||||
Int64 time_threshold = std::max(static_cast<Int64>(0), current_time - static_cast<Int64>(1000 * storage.settings.replicated_deduplication_window_seconds));
|
||||
Int64 time_threshold = std::max(static_cast<Int64>(0), current_time - static_cast<Int64>(1000 * storage_settings->replicated_deduplication_window_seconds));
|
||||
|
||||
/// Virtual node, all nodes that are "greater" than this one will be deleted
|
||||
NodeWithStat block_threshold{{}, time_threshold};
|
||||
|
||||
size_t current_deduplication_window = std::min<size_t>(timed_blocks.size(), storage.settings.replicated_deduplication_window.value);
|
||||
size_t current_deduplication_window = std::min<size_t>(timed_blocks.size(), storage_settings->replicated_deduplication_window);
|
||||
auto first_outdated_block_fixed_threshold = timed_blocks.begin() + current_deduplication_window;
|
||||
auto first_outdated_block_time_threshold = std::upper_bound(timed_blocks.begin(), timed_blocks.end(), block_threshold, NodeWithStat::greaterByTime);
|
||||
auto first_outdated_block = std::min(first_outdated_block_fixed_threshold, first_outdated_block_time_threshold);
|
||||
@ -401,10 +404,11 @@ void ReplicatedMergeTreeCleanupThread::getBlocksSortedByTime(zkutil::ZooKeeper &
|
||||
|
||||
void ReplicatedMergeTreeCleanupThread::clearOldMutations()
|
||||
{
|
||||
if (!storage.settings.finished_mutations_to_keep)
|
||||
auto storage_settings = storage.getCOWSettings();
|
||||
if (!storage_settings->finished_mutations_to_keep)
|
||||
return;
|
||||
|
||||
if (storage.queue.countFinishedMutations() <= storage.settings.finished_mutations_to_keep)
|
||||
if (storage.queue.countFinishedMutations() <= storage_settings->finished_mutations_to_keep)
|
||||
{
|
||||
/// Not strictly necessary, but helps to avoid unnecessary ZooKeeper requests.
|
||||
/// If even this replica hasn't finished enough mutations yet, then we don't need to clean anything.
|
||||
@ -431,10 +435,10 @@ void ReplicatedMergeTreeCleanupThread::clearOldMutations()
|
||||
|
||||
/// Do not remove entries that are greater than `min_pointer` (they are not done yet).
|
||||
entries.erase(std::upper_bound(entries.begin(), entries.end(), padIndex(min_pointer)), entries.end());
|
||||
/// Do not remove last `storage.settings.finished_mutations_to_keep` entries.
|
||||
if (entries.size() <= storage.settings.finished_mutations_to_keep)
|
||||
/// Do not remove last `storage_settings->finished_mutations_to_keep` entries.
|
||||
if (entries.size() <= storage_settings->finished_mutations_to_keep)
|
||||
return;
|
||||
entries.erase(entries.end() - storage.settings.finished_mutations_to_keep, entries.end());
|
||||
entries.erase(entries.end() - storage_settings->finished_mutations_to_keep, entries.end());
|
||||
|
||||
if (entries.empty())
|
||||
return;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user