Improved performance of dynamic compilation [#CLICKHOUSE-2].

copy_headers.sh

@@ -16,7 +16,8 @@
 # sudo ./copy_headers.sh . /usr/share/clickhouse/headers/
 
 SOURCE_PATH=${1:-.}
-DST=${2:-$SOURCE_PATH/../headers};
+DST=${2:-$SOURCE_PATH/../headers}
+BUILD_PATH=${3:-$SOURCE_PATH/build}
 
 PATH="/usr/local/bin:/usr/local/sbin:/usr/bin:$PATH"
 
@@ -24,14 +25,19 @@ if [[ -z $CLANG ]]; then
     CLANG="clang"
 fi
 
+START_HEADERS=$(echo \
+    $BUILD_PATH/dbms/src/Common/config_version.h \
+    $SOURCE_PATH/dbms/src/Interpreters/SpecializedAggregator.h \
+    $SOURCE_PATH/dbms/src/AggregateFunctions/AggregateFunction*.h)
+
 # Опция -mcx16 для того, чтобы выбиралось больше заголовочных файлов (с запасом).
 
-for src_file in $($CLANG -M -xc++ -std=gnu++1z -Wall -Werror -msse4 -mcx16 -mpopcnt -O3 -g -fPIC \
-    $(cat "$SOURCE_PATH/build/include_directories.txt") \
-    "$SOURCE_PATH/build/dbms/src/Common/config_version.h" \
-    "$SOURCE_PATH/dbms/src/Interpreters/SpecializedAggregator.h" |
+for src_file in $(echo | $CLANG -M -xc++ -std=gnu++1z -Wall -Werror -msse4 -mcx16 -mpopcnt -O3 -g -fPIC \
+    $(cat "$BUILD_PATH/include_directories.txt") \
+    $(echo $START_HEADERS | sed -r -e 's/[^ ]+/-include \0/g') \
+    - |
     tr -d '\\' |
-    sed -r -e 's/^\w+\.o://');
+    sed -r -e 's/^-\.o://');
 do
     dst_file=$src_file;
     dst_file=$(echo $dst_file | sed -r -e 's/build\///')    # for simplicity reasons, will put generated headers near the rest.

dbms/src/AggregateFunctions/AggregateFunctionArgMinMax.h

@@ -69,6 +69,8 @@ public:
     {
         this->data(place).result.insertResultInto(to);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 }

dbms/src/AggregateFunctions/AggregateFunctionArray.h

@@ -132,6 +132,8 @@ public:
     }
 
     IAggregateFunction::AddFunc getAddressOfAddFunction() const override final { return &addFree; }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 }

dbms/src/AggregateFunctions/AggregateFunctionAvg.h

@@ -70,6 +70,8 @@ public:
         static_cast<ColumnFloat64 &>(to).getData().push_back(
             static_cast<Float64>(this->data(place).sum) / this->data(place).count);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 

dbms/src/AggregateFunctions/AggregateFunctionBitwise.h

@@ -81,6 +81,8 @@ public:
     {
         static_cast<ColumnVector<T> &>(to).getData().push_back(this->data(place).value);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 

dbms/src/AggregateFunctions/AggregateFunctionCount.h

@@ -70,6 +70,8 @@ public:
     {
         data(place).count += x;
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -114,6 +116,8 @@ public:
     {
         static_cast<ColumnUInt64 &>(to).getData().push_back(data(place).count);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -183,6 +187,8 @@ public:
         static_cast<ColumnUInt64 &>(to).getData().push_back(data(place).count);
     }
 
+    const char * getHeaderFilePath() const override { return __FILE__; }
+
 private:
     enum { MAX_ARGS = 8 };
     size_t number_of_arguments = 0;

dbms/src/AggregateFunctions/AggregateFunctionForEach.h

@@ -245,6 +245,8 @@ public:
     }
 
     IAggregateFunction::AddFunc getAddressOfAddFunction() const override final { return &addFree; }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 

dbms/src/AggregateFunctions/AggregateFunctionGroupArray.h

@@ -137,6 +137,8 @@ public:
     {
         return true;
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -408,6 +410,8 @@ public:
     {
         return true;
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 #undef AGGREGATE_FUNCTION_GROUP_ARRAY_MAX_ARRAY_SIZE

dbms/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h

@@ -202,6 +202,8 @@ public:
 
         to_offsets.push_back((to_offsets.empty() ? 0 : to_offsets.back()) + result_array_size);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 

dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h

@@ -98,6 +98,8 @@ public:
         for (auto it = set.begin(); it != set.end(); ++it, ++i)
             data_to[old_size + i] = *it;
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -219,6 +221,8 @@ public:
             deserializeAndInsert(elem, data_to);
         }
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 

dbms/src/AggregateFunctions/AggregateFunctionIf.h

@@ -121,6 +121,8 @@ public:
     }
 
     IAggregateFunction::AddFunc getAddressOfAddFunction() const override final { return &addFree; }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 }

dbms/src/AggregateFunctions/AggregateFunctionMerge.h

@@ -121,6 +121,8 @@ public:
     }
 
     IAggregateFunction::AddFunc getAddressOfAddFunction() const override final { return &addFree; }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 }

dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h

@@ -726,6 +726,8 @@ public:
     {
         this->data(place).insertResultInto(to);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 }

dbms/src/AggregateFunctions/AggregateFunctionNull.h

@@ -157,6 +157,8 @@ public:
     {
         return nested_function->isState();
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 

dbms/src/AggregateFunctions/AggregateFunctionQuantile.h

@@ -99,6 +99,8 @@ public:
         else
             static_cast<ColumnVector<ArgumentFieldType> &>(to).getData().push_back(sample.quantileInterpolated(level));
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -192,6 +194,8 @@ public:
                 data_to.push_back(sample.quantileInterpolated(levels[i]));
         }
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 }

dbms/src/AggregateFunctions/AggregateFunctionQuantileDeterministic.h

@@ -105,6 +105,8 @@ public:
         else
             static_cast<ColumnVector<ArgumentFieldType> &>(to).getData().push_back(sample.quantileInterpolated(level));
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -204,6 +206,8 @@ public:
                 data_to.push_back(sample.quantileInterpolated(levels[i]));
         }
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 }

dbms/src/AggregateFunctions/AggregateFunctionQuantileExact.h

@@ -117,6 +117,8 @@ public:
 
         static_cast<ColumnVector<T> &>(to).getData().push_back(quantile);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -217,6 +219,8 @@ public:
                 data_to[old_size + i] = T();
         }
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 }

dbms/src/AggregateFunctions/AggregateFunctionQuantileExactWeighted.h

@@ -154,6 +154,8 @@ public:
 
         static_cast<ColumnVector<ValueType> &>(to).getData().push_back(it->first);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -296,6 +298,8 @@ public:
             ++level_index;
         }
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 }

dbms/src/AggregateFunctions/AggregateFunctionQuantileTDigest.h

@@ -411,6 +411,8 @@ public:
         else
             static_cast<ColumnVector<T> &>(to).getData().push_back(quantile);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -480,6 +482,8 @@ public:
         else
             static_cast<ColumnVector<T> &>(to).getData().push_back(quantile);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -563,6 +567,8 @@ public:
                 params, &levels.levels[0], &levels.permutation[0], size, &data_to[old_size]);
         }
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -648,6 +654,8 @@ public:
                 params, &levels.levels[0], &levels.permutation[0], size, &data_to[old_size]);
         }
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 }

dbms/src/AggregateFunctions/AggregateFunctionQuantileTiming.h

@@ -843,6 +843,8 @@ public:
     {
         static_cast<ColumnFloat32 &>(to).getData().push_back(this->data(place).getFloat(level));
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -903,6 +905,8 @@ public:
     {
         static_cast<ColumnFloat32 &>(to).getData().push_back(this->data(place).getFloat(level));
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -971,6 +975,8 @@ public:
 
         this->data(place).getManyFloat(&levels.levels[0], &levels.permutation[0], size, &data_to[old_size]);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -1037,6 +1043,8 @@ public:
 
         this->data(place).getManyFloat(&levels.levels[0], &levels.permutation[0], size, &data_to[old_size]);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 

dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h

@@ -238,6 +238,8 @@ public:
 
     IAggregateFunction::AddFunc getAddressOfAddFunction() const override final { return &addFree; }
 
+    const char * getHeaderFilePath() const override { return __FILE__; }
+
 private:
     enum class PatternActionType
     {

dbms/src/AggregateFunctions/AggregateFunctionState.h

@@ -111,6 +111,8 @@ public:
     }
 
     IAggregateFunction::AddFunc getAddressOfAddFunction() const override final { return &addFree; }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 }

dbms/src/AggregateFunctions/AggregateFunctionStatistics.h

@@ -153,6 +153,8 @@ public:
     {
         this->data(place).publish(to);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 /** Implementing the varSamp function.
@@ -416,6 +418,8 @@ public:
     {
         this->data(place).publish(to);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 /** Implementing the covarSamp function.

dbms/src/AggregateFunctions/AggregateFunctionSum.h

@@ -63,6 +63,8 @@ public:
     {
         static_cast<ColumnVector<typename NearestFieldType<T>::Type> &>(to).getData().push_back(this->data(place).sum);
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 

dbms/src/AggregateFunctions/AggregateFunctionTopK.h

@@ -120,6 +120,8 @@ public:
         for (auto it = resultVec.begin(); it != resultVec.end(); ++it, ++i)
             data_to[old_size + i] = it->key;
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -236,6 +238,8 @@ public:
             deserializeAndInsert(elem.key, data_to);
         }
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 

dbms/src/AggregateFunctions/AggregateFunctionUniq.h

@@ -255,7 +255,7 @@ struct OneAdder<T, Data, typename std::enable_if<
     std::is_same<Data, AggregateFunctionUniqHLL12Data<T>>::value>::type>
 {
     template <typename T2 = T>
-    static void addImpl(Data & data, const IColumn & column, size_t row_num,
+    static void ALWAYS_INLINE addImpl(Data & data, const IColumn & column, size_t row_num,
         typename std::enable_if<!std::is_same<T2, String>::value>::type * = nullptr)
     {
         const auto & value = static_cast<const ColumnVector<T2> &>(column).getData()[row_num];
@@ -263,7 +263,7 @@ struct OneAdder<T, Data, typename std::enable_if<
     }
 
     template <typename T2 = T>
-    static void addImpl(Data & data, const IColumn & column,    size_t row_num,
+    static void ALWAYS_INLINE addImpl(Data & data, const IColumn & column,    size_t row_num,
         typename std::enable_if<std::is_same<T2, String>::value>::type * = nullptr)
     {
         StringRef value = column.getDataAt(row_num);
@@ -279,7 +279,7 @@ struct OneAdder<T, Data, typename std::enable_if<
     std::is_same<Data, AggregateFunctionUniqCombinedData<T>>::value>::type>
 {
     template <typename T2 = T>
-    static void addImpl(Data & data, const IColumn & column, size_t row_num,
+    static void ALWAYS_INLINE addImpl(Data & data, const IColumn & column, size_t row_num,
         typename std::enable_if<!std::is_same<T2, String>::value>::type * = nullptr)
     {
         const auto & value = static_cast<const ColumnVector<T2> &>(column).getData()[row_num];
@@ -287,7 +287,7 @@ struct OneAdder<T, Data, typename std::enable_if<
     }
 
     template <typename T2 = T>
-    static void addImpl(Data & data, const IColumn & column,    size_t row_num,
+    static void ALWAYS_INLINE addImpl(Data & data, const IColumn & column,    size_t row_num,
         typename std::enable_if<std::is_same<T2, String>::value>::type * = nullptr)
     {
         StringRef value = column.getDataAt(row_num);
@@ -300,14 +300,14 @@ struct OneAdder<T, Data, typename std::enable_if<
     std::is_same<Data, AggregateFunctionUniqExactData<T>>::value>::type>
 {
     template <typename T2 = T>
-    static void addImpl(Data & data, const IColumn & column, size_t row_num,
+    static void ALWAYS_INLINE addImpl(Data & data, const IColumn & column, size_t row_num,
         typename std::enable_if<!std::is_same<T2, String>::value>::type * = nullptr)
     {
         data.set.insert(static_cast<const ColumnVector<T2> &>(column).getData()[row_num]);
     }
 
     template <typename T2 = T>
-    static void addImpl(Data & data, const IColumn & column, size_t row_num,
+    static void ALWAYS_INLINE addImpl(Data & data, const IColumn & column, size_t row_num,
         typename std::enable_if<std::is_same<T2, String>::value>::type * = nullptr)
     {
         StringRef value = column.getDataAt(row_num);
@@ -364,6 +364,8 @@ public:
     {
         static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -426,6 +428,8 @@ public:
     }
 
     IAggregateFunction::AddFunc getAddressOfAddFunction() const override final { return &addFree; }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 

dbms/src/AggregateFunctions/AggregateFunctionUniqUpTo.h

@@ -176,6 +176,8 @@ public:
     {
         static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).size());
     }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 
@@ -256,6 +258,8 @@ public:
     }
 
     IAggregateFunction::AddFunc getAddressOfAddFunction() const override final { return &addFree; }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
 

dbms/src/AggregateFunctions/IAggregateFunction.h

@@ -121,7 +121,6 @@ public:
       */
     virtual bool isState() const { return false; }
 
-
     /** The inner loop that uses the function pointer is better than using the virtual function.
       * The reason is that in the case of virtual functions GCC 5.1.2 generates code,
       *  which, at each iteration of the loop, reloads the function address (the offset value in the virtual function table) from memory to the register.
@@ -130,6 +129,12 @@ public:
       */
     using AddFunc = void (*)(const IAggregateFunction *, AggregateDataPtr, const IColumn **, size_t, Arena *);
     virtual AddFunc getAddressOfAddFunction() const = 0;
+
+    /** This is used for runtime code generation to determine, which header files to include in generated source.
+      * Always implement it as
+      * const char * getHeaderFilePath() const override { return __FILE__; }
+      */
+    virtual const char * getHeaderFilePath() const = 0;
 };
 
 

dbms/src/Interpreters/Aggregator.cpp

@@ -223,6 +223,7 @@ void Aggregator::compileIfPossible(AggregatedDataVariants::Type type)
 
     /// List of types of aggregate functions.
     std::stringstream aggregate_functions_typenames_str;
+    std::stringstream aggregate_functions_headers_args;
     for (size_t i = 0; i < params.aggregates_size; ++i)
     {
         IAggregateFunction & func = *aggregate_functions[i];
@@ -237,8 +238,21 @@ void Aggregator::compileIfPossible(AggregatedDataVariants::Type type)
                 + ", status: " + toString(status), ErrorCodes::CANNOT_COMPILE_CODE);
 
         aggregate_functions_typenames_str << ((i != 0) ? ", " : "") << type_name;
+
+        std::string header_path = func.getHeaderFilePath();
+        auto pos = header_path.find("/AggregateFunctions/");
+
+        if (pos == std::string::npos)
+            throw Exception("Cannot compile code: unusual path of header file for aggregate function: " + header_path,
+                ErrorCodes::CANNOT_COMPILE_CODE);
+
+        aggregate_functions_headers_args << "-include '" INTERNAL_COMPILER_HEADERS "/dbms/src";
+        aggregate_functions_headers_args.write(&header_path[pos], header_path.size() - pos);
+        aggregate_functions_headers_args << "' ";
     }
 
+    aggregate_functions_headers_args << "-include '" INTERNAL_COMPILER_HEADERS "/dbms/src/Interpreters/SpecializedAggregator.h'";
+
     std::string aggregate_functions_typenames = aggregate_functions_typenames_str.str();
 
     std::stringstream key_str;
@@ -355,9 +369,9 @@ void Aggregator::compileIfPossible(AggregatedDataVariants::Type type)
       * If the counter has reached the value min_count_to_compile, then the compilation starts asynchronously (in a separate thread)
       *  at the end of which `on_ready` callback is called.
       */
+    aggregate_functions_headers_args << " -Wno-unused-function";
     SharedLibraryPtr lib = params.compiler->getOrCount(key, params.min_count_to_compile,
-        "-include " INTERNAL_COMPILER_HEADERS "/dbms/src/Interpreters/SpecializedAggregator.h "
-        "-Wno-unused-function",
+        aggregate_functions_headers_args.str(),
         get_code, on_ready);
 
     /// If the result is already ready.

dbms/src/Interpreters/Compiler.cpp

@@ -215,7 +215,7 @@ void Compiler::compile(
 
     std::stringstream command;
 
-    /// Slightly uncomfortable.
+    /// Slightly unconvenient.
     command <<
         "LD_LIBRARY_PATH=" PATH_SHARE "/clickhouse/bin/"
         " " INTERNAL_COMPILER_EXECUTABLE
@@ -231,6 +231,7 @@ void Compiler::compile(
 #endif
         " -I " INTERNAL_COMPILER_HEADERS "/dbms/src/"
         " -I " INTERNAL_COMPILER_HEADERS "/contrib/libcityhash/include/"
+        " -I " INTERNAL_COMPILER_HEADERS "/contrib/libpcg-random/include/"
         " -I " INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR
         " -I " INTERNAL_Poco_Foundation_INCLUDE_DIR
         " -I " INTERNAL_Boost_INCLUDE_DIRS

dbms/src/Interpreters/SpecializedAggregator.h

@@ -1,32 +1,7 @@
 #include <Common/TypeList.h>
+#include <Common/typeid_cast.h>
 #include <Interpreters/Aggregator.h>
 
-#include <AggregateFunctions/AggregateFunctionArgMinMax.h>
-#include <AggregateFunctions/AggregateFunctionArray.h>
-#include <AggregateFunctions/AggregateFunctionAvg.h>
-#include <AggregateFunctions/AggregateFunctionCount.h>
-#include <AggregateFunctions/AggregateFunctionForEach.h>
-#include <AggregateFunctions/AggregateFunctionGroupArray.h>
-#include <AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h>
-#include <AggregateFunctions/AggregateFunctionGroupUniqArray.h>
-#include <AggregateFunctions/AggregateFunctionIf.h>
-#include <AggregateFunctions/AggregateFunctionMerge.h>
-#include <AggregateFunctions/AggregateFunctionMinMaxAny.h>
-#include <AggregateFunctions/AggregateFunctionNull.h>
-#include <AggregateFunctions/AggregateFunctionQuantileDeterministic.h>
-#include <AggregateFunctions/AggregateFunctionQuantileExact.h>
-#include <AggregateFunctions/AggregateFunctionQuantileExactWeighted.h>
-#include <AggregateFunctions/AggregateFunctionQuantile.h>
-#include <AggregateFunctions/AggregateFunctionQuantileTDigest.h>
-#include <AggregateFunctions/AggregateFunctionQuantileTiming.h>
-#include <AggregateFunctions/AggregateFunctionSequenceMatch.h>
-#include <AggregateFunctions/AggregateFunctionState.h>
-#include <AggregateFunctions/AggregateFunctionStatistics.h>
-#include <AggregateFunctions/AggregateFunctionSum.h>
-#include <AggregateFunctions/AggregateFunctionTopK.h>
-#include <AggregateFunctions/AggregateFunctionUniq.h>
-#include <AggregateFunctions/AggregateFunctionUniqUpTo.h>
-
 
 namespace DB
 {

dbms/src/Parsers/ASTSystemQuery.h

@@ -1,6 +1,6 @@
 #pragma once
+
 #include <Parsers/IAST.h>
-#include <AggregateFunctions/AggregateFunctionCount.h>
 
 
 namespace DB