Merge pull request #19390 from ClickHouse/http_referer

Add http_referer to client info

cmake/autogenerated_versions.txt

@@ -1,5 +1,5 @@
 # This strings autochanged from release_lib.sh:
-SET(VERSION_REVISION 54446)
+SET(VERSION_REVISION 54447)
 SET(VERSION_MAJOR 21)
 SET(VERSION_MINOR 2)
 SET(VERSION_PATCH 1)

src/Core/Defines.h

@@ -74,9 +74,10 @@
 #define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET 54441
 
 #define DBMS_MIN_REVISION_WITH_X_FORWARDED_FOR_IN_CLIENT_INFO 54443
+#define DBMS_MIN_REVISION_WITH_REFERER_IN_CLIENT_INFO 54447
 
 /// Version of ClickHouse TCP protocol. Increment it manually when you change the protocol.
-#define DBMS_TCP_PROTOCOL_VERSION 54443
+#define DBMS_TCP_PROTOCOL_VERSION 54447
 
 /// The boundary on which the blocks for asynchronous file operations should be aligned.
 #define DEFAULT_AIO_FILE_BLOCK_SIZE 4096

src/Interpreters/ClientInfo.cpp

@@ -52,6 +52,9 @@ void ClientInfo::write(WriteBuffer & out, const UInt64 server_protocol_revision)
 
         if (server_protocol_revision >= DBMS_MIN_REVISION_WITH_X_FORWARDED_FOR_IN_CLIENT_INFO)
             writeBinary(forwarded_for, out);
+
+        if (server_protocol_revision >= DBMS_MIN_REVISION_WITH_REFERER_IN_CLIENT_INFO)
+            writeBinary(http_referer, out);
     }
 
     if (server_protocol_revision >= DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO)
@@ -126,6 +129,9 @@ void ClientInfo::read(ReadBuffer & in, const UInt64 client_protocol_revision)
 
         if (client_protocol_revision >= DBMS_MIN_REVISION_WITH_X_FORWARDED_FOR_IN_CLIENT_INFO)
             readBinary(forwarded_for, in);
+
+        if (client_protocol_revision >= DBMS_MIN_REVISION_WITH_REFERER_IN_CLIENT_INFO)
+            readBinary(http_referer, in);
     }
 
     if (client_protocol_revision >= DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO)

src/Interpreters/ClientInfo.h

@@ -82,6 +82,7 @@ public:
     /// For http
     HTTPMethod http_method = HTTPMethod::UNKNOWN;
     String http_user_agent;
+    String http_referer;
 
     /// Comma separated list of forwarded IP addresses (from X-Forwarded-For for HTTP interface).
     /// It's expected that proxy appends the forwarded address to the end of the list.

src/Interpreters/QueryLog.cpp

@@ -86,6 +86,7 @@ Block QueryLogElement::createBlock()
         {std::make_shared<DataTypeUInt32>(),                                  "client_version_patch"},
         {std::make_shared<DataTypeUInt8>(),                                   "http_method"},
         {std::make_shared<DataTypeString>(),                                  "http_user_agent"},
+        {std::make_shared<DataTypeString>(),                                  "http_referer"},
         {std::make_shared<DataTypeString>(),                                  "forwarded_for"},
         {std::make_shared<DataTypeString>(),                                  "quota_key"},
 
@@ -214,6 +215,7 @@ void QueryLogElement::appendClientInfo(const ClientInfo & client_info, MutableCo
 
     columns[i++]->insert(UInt64(client_info.http_method));
     columns[i++]->insert(client_info.http_user_agent);
+    columns[i++]->insert(client_info.http_referer);
     columns[i++]->insert(client_info.forwarded_for);
 
     columns[i++]->insert(client_info.quota_key);

src/Interpreters/QueryThreadLog.cpp

@@ -61,6 +61,7 @@ Block QueryThreadLogElement::createBlock()
         {std::make_shared<DataTypeUInt32>(),        "client_version_patch"},
         {std::make_shared<DataTypeUInt8>(),         "http_method"},
         {std::make_shared<DataTypeString>(),        "http_user_agent"},
+        {std::make_shared<DataTypeString>(),        "http_referer"},
         {std::make_shared<DataTypeString>(),        "forwarded_for"},
         {std::make_shared<DataTypeString>(),        "quota_key"},
 

src/Server/HTTPHandler.cpp

@@ -295,6 +295,7 @@ void HTTPHandler::processQuery(
 
     client_info.http_method = http_method;
     client_info.http_user_agent = request.get("User-Agent", "");
+    client_info.http_referer = request.get("Referer", "");
     client_info.forwarded_for = request.get("X-Forwarded-For", "");
 
     /// This will also set client_info.current_user and current_address

src/Storages/System/StorageSystemProcesses.cpp

@@ -43,6 +43,7 @@ NamesAndTypesList StorageSystemProcesses::getNamesAndTypes()
 
         {"http_method", std::make_shared<DataTypeUInt8>()},
         {"http_user_agent", std::make_shared<DataTypeString>()},
+        {"http_referer", std::make_shared<DataTypeString>()},
         {"forwarded_for", std::make_shared<DataTypeString>()},
 
         {"quota_key", std::make_shared<DataTypeString>()},
@@ -99,6 +100,7 @@ void StorageSystemProcesses::fillData(MutableColumns & res_columns, const Contex
 
         res_columns[i++]->insert(UInt64(process.client_info.http_method));
         res_columns[i++]->insert(process.client_info.http_user_agent);
+        res_columns[i++]->insert(process.client_info.http_referer);
         res_columns[i++]->insert(process.client_info.forwarded_for);
 
         res_columns[i++]->insert(process.client_info.quota_key);

tests/queries/0_stateless/01661_referer.reference

@@ -0,0 +1,2 @@
+1
+https://yandex.ru/

tests/queries/0_stateless/01661_referer.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'SELECT 1' --referer 'https://yandex.ru/'
+${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS"
+${CLICKHOUSE_CLIENT} --query "SELECT http_referer FROM system.query_log WHERE http_referer LIKE '%yandex%' LIMIT 1"