Merge pull request #10795 from ClickHouse/list-licenses

Added "system.licenses" table

src/Storages/System/CMakeLists.txt

@@ -11,13 +11,30 @@ configure_file (StorageSystemBuildOptions.generated.cpp.in ${CONFIG_BUILD})
 include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 add_headers_and_sources(storages_system .)
 list (APPEND storages_system_sources ${CONFIG_BUILD})
-add_library(clickhouse_storages_system ${storages_system_headers} ${storages_system_sources})
-target_link_libraries(clickhouse_storages_system PRIVATE dbms common string_utils clickhouse_common_zookeeper clickhouse_parsers)
 
-add_custom_target(generate-contributors ./StorageSystemContributors.sh SOURCES StorageSystemContributors.sh WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+add_custom_target(generate-contributors
+    ./StorageSystemContributors.sh
+    SOURCES StorageSystemContributors.sh
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
     # BYPRODUCTS StorageSystemContributors.generated.cpp
 )
+
 if(NOT TARGET generate-source)
     add_custom_target(generate-source)
 endif()
+
 add_dependencies(generate-source generate-contributors)
+
+set(GENERATED_LICENSES_SRC ${CMAKE_CURRENT_BINARY_DIR}/StorageSystemLicenses.generated.cpp)
+
+add_custom_command(
+    OUTPUT StorageSystemLicenses.generated.cpp
+    COMMAND ./StorageSystemLicenses.sh > ${GENERATED_LICENSES_SRC}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+
+list (APPEND storages_system_sources ${GENERATED_LICENSES_SRC})
+# Overlength strings
+set_source_files_properties(${GENERATED_LICENSES_SRC} PROPERTIES COMPILE_FLAGS -w)
+
+add_library(clickhouse_storages_system ${storages_system_headers} ${storages_system_sources})
+target_link_libraries(clickhouse_storages_system PRIVATE dbms common string_utils clickhouse_common_zookeeper clickhouse_parsers)

src/Storages/System/StorageSystemLicenses.cpp

@@ -0,0 +1,31 @@
+#include "StorageSystemLicenses.h"
+
+#include <algorithm>
+#include <DataTypes/DataTypeString.h>
+
+
+extern const char * library_licenses[];
+
+namespace DB
+{
+NamesAndTypesList StorageSystemLicenses::getNamesAndTypes()
+{
+    return {
+        {"library_name", std::make_shared<DataTypeString>()},
+        {"license_type", std::make_shared<DataTypeString>()},
+        {"license_path", std::make_shared<DataTypeString>()},
+        {"license_text", std::make_shared<DataTypeString>()},
+    };
+}
+
+void StorageSystemLicenses::fillData(MutableColumns & res_columns, const Context &, const SelectQueryInfo &) const
+{
+    for (const auto * it = library_licenses; *it; it += 4)
+    {
+        res_columns[0]->insert(String(it[0]));
+        res_columns[1]->insert(String(it[1]));
+        res_columns[2]->insert(String(it[2]));
+        res_columns[3]->insert(String(it[3]));
+    }
+}
+}

src/Storages/System/StorageSystemLicenses.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <Storages/System/IStorageSystemOneBlock.h>
+#include <ext/shared_ptr_helper.h>
+
+
+namespace DB
+{
+class Context;
+
+
+/** System table "licenses" with list of licenses of 3rd party libraries
+  */
+class StorageSystemLicenses final :
+    public ext::shared_ptr_helper<StorageSystemLicenses>,
+    public IStorageSystemOneBlock<StorageSystemLicenses>
+{
+    friend struct ext::shared_ptr_helper<StorageSystemLicenses>;
+protected:
+    void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override;
+
+    using IStorageSystemOneBlock::IStorageSystemOneBlock;
+
+public:
+    std::string getName() const override
+    {
+        return "SystemLicenses";
+    }
+
+    static NamesAndTypesList getNamesAndTypes();
+};
+}

src/Storages/System/StorageSystemLicenses.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+ROOT_PATH="$(git rev-parse --show-toplevel)"
+IFS=$'\t'
+
+echo "// autogenerated by $0"
+echo "const char * library_licenses[] = {"
+${ROOT_PATH}/utils/list-licenses/list-licenses.sh | while read row; do
+    arr=($row)
+
+    echo "\"${arr[0]}\", \"${arr[1]}\", \"${arr[2]}\", R\"heredoc($(cat "${ROOT_PATH}/${arr[2]}"))heredoc\","
+    echo
+done
+echo "nullptr"
+echo "};"

src/Storages/System/attachSystemTables.cpp

@@ -37,6 +37,9 @@
 #include <Storages/System/StorageSystemTables.h>
 #include <Storages/System/StorageSystemZooKeeper.h>
 #include <Storages/System/StorageSystemContributors.h>
+#if !defined(ARCADIA_BUILD)
+    #include <Storages/System/StorageSystemLicenses.h>
+#endif
 #include <Storages/System/StorageSystemDisks.h>
 #include <Storages/System/StorageSystemStoragePolicies.h>
 #include <Storages/System/StorageSystemZeros.h>
@@ -75,6 +78,9 @@ void attachSystemTablesLocal(IDatabase & system_database)
     system_database.attachTable("collations", StorageSystemCollations::create("collations"));
     system_database.attachTable("table_engines", StorageSystemTableEngines::create("table_engines"));
     system_database.attachTable("contributors", StorageSystemContributors::create("contributors"));
+#if !defined(ARCADIA_BUILD)
+    system_database.attachTable("licenses", StorageSystemLicenses::create("licenses"));
+#endif
 #ifdef OS_LINUX
     system_database.attachTable("stack_trace", StorageSystemStackTrace::create("stack_trace"));
 #endif

tests/queries/0_stateless/01276_system_licenses.reference

@@ -0,0 +1,2 @@
+1
+zstd	BSD	/contrib/zstd/LICENSE

tests/queries/0_stateless/01276_system_licenses.sql

@@ -0,0 +1,2 @@
+SELECT count() > 10 FROM system.licenses;
+SELECT library_name, license_type, license_path FROM system.licenses WHERE library_name LIKE '%zstd%';

utils/list-licenses/list-licenses.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+ROOT_PATH="$(git rev-parse --show-toplevel)"
+LIBS_PATH="${ROOT_PATH}/contrib"
+
+ls -1 -d ${LIBS_PATH}/*/ | grep -F -v -- '-cmake' | while read LIB; do
+    LIB_NAME=$(basename $LIB)
+
+    LIB_LICENSE=$(
+        LC_ALL=C find "$LIB" -type f -and '(' -iname 'LICENSE*' -or -iname 'COPYING*' -or -iname 'COPYRIGHT*' ')' -and -not -iname '*.html' -printf "%d\t%p\n" |
+            awk '
+                BEGIN { IGNORECASE=1; min_depth = 0 }
+                /LICENSE/ { if (!min_depth || $1 <= min_depth) { min_depth = $1; license = $2 } }
+                /COPY/    { if (!min_depth || $1 <= min_depth) { min_depth = $1; copying = $2 } }
+                END { if (license) { print license } else { print copying } }')
+
+    if [ -n "$LIB_LICENSE" ]; then
+
+        LICENSE_TYPE=$(
+        (grep -q -F 'Apache' "$LIB_LICENSE" &&
+         echo "Apache") ||
+        (grep -q -F 'Boost' "$LIB_LICENSE" &&
+         echo "Boost") ||
+        (grep -q -i -P 'public\s*domain' "$LIB_LICENSE" &&
+         echo "Public Domain") ||
+        (grep -q -F 'BSD' "$LIB_LICENSE" &&
+         echo "BSD") ||
+        (grep -q -F 'Lesser General Public License' "$LIB_LICENSE" &&
+         echo "LGPL") ||
+        (grep -q -i -F 'The origin of this software must not be misrepresented' "$LIB_LICENSE" &&
+         grep -q -i -F 'Altered source versions must be plainly marked as such' "$LIB_LICENSE" &&
+         grep -q -i -F 'This notice may not be removed or altered' "$LIB_LICENSE" &&
+         echo "zLib") ||
+        (grep -q -i -F 'Permission is hereby granted, free of charge, to any person' "$LIB_LICENSE" &&
+         grep -q -i -F 'The above copyright notice and this permission notice shall be included' "$LIB_LICENSE" &&
+         grep -q -i -F 'THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND' "$LIB_LICENSE" &&
+         echo "MIT") ||
+        (grep -q -i -F 'Permission to use, copy, modify, and distribute this software for any purpose' "$LIB_LICENSE" &&
+         grep -q -i -F 'the name of a copyright holder shall not' "$LIB_LICENSE" &&
+         grep -q -i -F 'THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND' "$LIB_LICENSE" &&
+         echo "MIT/curl") ||
+        (grep -q -i -F 'Redistributions of source code must retain the above copyright' "$LIB_LICENSE" &&
+         grep -q -i -F 'Redistributions in binary form must reproduce' "$LIB_LICENSE" &&
+         grep -q -i -F 'Neither the name' "$LIB_LICENSE" &&
+         echo "BSD 3-clause") ||
+        (grep -q -i -F 'Redistributions of source code must retain the above copyright' "$LIB_LICENSE" &&
+         grep -q -i -F 'Redistributions in binary form must reproduce' "$LIB_LICENSE" &&
+         echo "BSD 2-clause") ||
+        echo "Unknown")
+
+        RELATIVE_PATH=$(echo "$LIB_LICENSE" | sed -r -e 's!^.+/contrib/!/contrib/!')
+
+        echo -e "$LIB_NAME\t$LICENSE_TYPE\t$RELATIVE_PATH"
+    fi
+done