Add tests

cmake/find_poco.cmake

@@ -132,7 +132,7 @@ if (Poco_SQLODBC_LIBRARY AND ODBC_FOUND)
     set (USE_POCO_SQLODBC 1)
 endif ()
 
-message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_Crypto_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_SQL_LIBRARY},${Poco_SQLODBC_LIBRARY},${Poco_MongoDB_LIBRARY},${Poco_Redis_INCLUDE_DIR}; MongoDB=${USE_POCO_MONGODB}, Redis=${USE_POCO_REDIS}, DataODBC=${USE_POCO_DATAODBC}, NetSSL=${USE_POCO_NETSSL}")
+message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_Crypto_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_SQL_LIBRARY},${Poco_SQLODBC_LIBRARY},${Poco_MongoDB_LIBRARY},${Poco_Redis_LIBRARY}; MongoDB=${USE_POCO_MONGODB}, Redis=${USE_POCO_REDIS}, DataODBC=${USE_POCO_DATAODBC}, NetSSL=${USE_POCO_NETSSL}")
 
 # How to make sutable poco:
 # use branch:

dbms/src/Dictionaries/CMakeLists.txt

@@ -37,7 +37,11 @@ if(USE_POCO_MONGODB)
 endif()
 
 if(USE_POCO_REDIS)
-    target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${Poco_Redis_INCLUDE_DIR})
+    # for code highlighting in CLion
+    # target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${Poco_Redis_INCLUDE_DIR})
+
+    # for build
+    target_link_libraries(clickhouse_dictionaries PRIVATE ${Poco_Redis_LIBRARY})
 endif()
 
 add_subdirectory(Embedded)

dbms/src/Dictionaries/registerDictionaries.cpp

@@ -7,6 +7,7 @@ void registerDictionarySourceFile(DictionarySourceFactory & source_factory);
 void registerDictionarySourceMysql(DictionarySourceFactory & source_factory);
 void registerDictionarySourceClickHouse(DictionarySourceFactory & source_factory);
 void registerDictionarySourceMongoDB(DictionarySourceFactory & source_factory);
+void registerDictionarySourceRedis(DictionarySourceFactory & source_factory);
 void registerDictionarySourceXDBC(DictionarySourceFactory & source_factory);
 void registerDictionarySourceJDBC(DictionarySourceFactory & source_factory);
 void registerDictionarySourceExecutable(DictionarySourceFactory & source_factory);
@@ -30,6 +31,7 @@ void registerDictionaries()
         registerDictionarySourceMysql(source_factory);
         registerDictionarySourceClickHouse(source_factory);
         registerDictionarySourceMongoDB(source_factory);
+        registerDictionarySourceRedis(source_factory);
         registerDictionarySourceXDBC(source_factory);
         registerDictionarySourceJDBC(source_factory);
         registerDictionarySourceExecutable(source_factory);

dbms/tests/external_dictionaries/generate_and_test.py

@@ -119,6 +119,17 @@ def generate_structure(args):
             [ 'mongodb_user_flat', 0, True ],
         ])
 
+    if not args.no_redis:
+        dictionaries.extend([
+            [ 'redis_flat', 0, True ],
+            [ 'redis_hashed', 0, True ],
+            [ 'redis_cache', 0, True ],
+            [ 'redis_complex_integers_key_hashed', 1, False ],
+            [ 'redis_complex_integers_key_cache', 1, False ],
+            [ 'redis_complex_mixed_key_hashed', 2, False ],
+            [ 'redis_complex_mixed_key_cache', 2, False ],
+        ])
+
     if args.use_lib:
         dictionaries.extend([
             # [ 'library_flat', 0, True ],
@@ -382,6 +393,51 @@ def generate_data(args):
             print 'Could not create MongoDB collection'
             exit(-1)
 
+    # create Redis storage from complete_query via JSON file
+    if not args.no_redis:
+        print 'Creating Redis storage'
+        table_rows = json.loads(subprocess.check_output([
+            args.client,
+            '--port',
+            args.port,
+            '--output_format_json_quote_64bit_integers',
+            '0',
+            '--query',
+            "select * from test.dictionary_source where not ignore(" \
+            "concat('new Date(\\'', toString(Date_), '\\')') as Date_, " \
+            "concat('new ISODate(\\'', replaceOne(toString(DateTime_, 'UTC'), ' ', 'T'), 'Z\\')') as DateTime_" \
+            ") format JSON"
+        ]))['data']
+
+        # print json.dumps(table_rows)
+
+        # For Integers the first byte of the reply is ":"
+        # For Bulk Strings the first byte of the reply is "$"
+
+        proto_for_redis = ""
+        for counter, collection in enumerate(table_rows):
+            proto_for_redis += "SELECT " + str(counter) + "\r\n"
+            proto_for_redis += "FLUSHDB\r\n"
+            for key, value in collection.iteritems():
+                value_type = "$"
+                if isinstance(value, int):
+                    value_type = ":"
+                else:
+                    value = str(value)
+                    if "Date" in value:
+                        value = value[value.find("'") + 1:-2]
+
+                proto_for_redis += "SET " + "$" + key + " " + value_type + str(value) + "\r\n"
+
+        # with open("clickhouse_redis.log", "w") as f:
+        #     f.write(json.dumps(table_rows) + "\n" + proto_for_redis + "\n")
+
+        open('generated/full.json', 'w').write(proto_for_redis)
+        result = system('cat {0}/full.json | redis-cli > \\dev\\null'.format(args.generated))
+        if result != 0:
+            print 'Could not create Redis storage'
+            exit(-1)
+
 
 def generate_dictionaries(args):
     dictionary_skeleton = '''
@@ -482,6 +538,13 @@ def generate_dictionaries(args):
     </mongodb>
     '''.format(mongo_host=args.mongo_host)
 
+    source_redis = '''
+    <redis>
+        <host>{redis_host}</host>
+        <port>6379</port>
+    </redis>
+    '''.format(redis_host=args.redis_host)
+
     source_executable = '''
     <executable>
         <command>cat %s</command>
@@ -668,6 +731,17 @@ def generate_dictionaries(args):
         [ source_mongodb_user, layout_flat ],
     ])
 
+    if not args.no_redis:
+        sources_and_layouts.extend([
+        [ source_redis, layout_flat ],
+        [ source_redis, layout_hashed ],
+        [ source_redis, layout_cache ],
+        [ source_redis, layout_complex_key_cache ],
+        [ source_redis, layout_complex_key_hashed ],
+        [ source_redis, layout_complex_key_hashed ],
+        [ source_redis, layout_complex_key_cache ],
+    ])
+
     if args.use_lib:
         sources_and_layouts.extend([
         #[ source_library, layout_flat ],
@@ -947,6 +1021,8 @@ if __name__ == '__main__':
     parser.add_argument('--no_mongo', action='store_true', help = 'Dont use mongodb dictionaries')
     parser.add_argument('--mongo_host', default = 'localhost', help = 'mongo server host')
     parser.add_argument('--use_mongo_user', action='store_true', help = 'Test mongodb with user-pass')
+    parser.add_argument('--no_redis', action='store_true', help = 'Dont use redis dictionaries')
+    parser.add_argument('--redis_host', default = 'localhost', help = 'redis server host')
 
     parser.add_argument('--no_http', action='store_true', help = 'Dont use http dictionaries')
     parser.add_argument('--http_port', default = 58000, help = 'http server port')

dbms/tests/external_dictionaries/run.sh

@@ -9,6 +9,7 @@ fi
 
 NO_MYSQL=0
 NO_MONGO=0
+NO_REDIS=0
 
 for arg in "$@"; do
     if [ "$arg" = "--no_mysql" ]; then
@@ -17,6 +18,9 @@ for arg in "$@"; do
     if [ "$arg" == "--no_mongo" ]; then
         NO_MONGO=1
     fi
+    if [ "$arg" == "--no_redis" ]; then
+        NO_REDIS=1
+    fi
 done
 
 # MySQL
@@ -101,6 +105,31 @@ else
     fi
 fi
 
+# Redis
+if [ $NO_REDIS -eq 1 ]; then
+    echo "Not using Redis"
+else
+    if [ -z $(which redis-cli) ]; then
+        echo 'Installing Redis'
+
+        sudo apt-get update &>/dev/null
+        sudo apt-get install redis-server
+
+        which redis-server >/dev/null
+        if [ $? -ne 0 ]; then
+            echo 'Failed installing redis-server'
+            exit -1
+        fi
+    fi
+
+    echo | redis-cli &>/dev/null
+    if [ $? -ne 0 ]; then
+        sudo systemctl start redis.service
+    else
+        echo 'Redis already started'
+    fi
+fi
+
 # ClickHouse
 clickhouse-server &> clickhouse.log &
 sleep 3

docs/en/query_language/dicts/external_dicts_dict_sources.md

@@ -30,6 +30,7 @@ Types of sources (`source_type`):
     - [MySQL](#dicts-external_dicts_dict_sources-mysql)
     - [ClickHouse](#dicts-external_dicts_dict_sources-clickhouse)
     - [MongoDB](#dicts-external_dicts_dict_sources-mongodb)
+    - [Redis](#dicts-external_dicts_dict_sources-redis)
     - [ODBC](#dicts-external_dicts_dict_sources-odbc)
 
 
@@ -421,4 +422,23 @@ Setting fields:
 - `db` – Name of the database.
 - `collection` – Name of the collection.
 
+
+### Redis {#dicts-external_dicts_dict_sources-redis}
+
+Example of settings:
+
+```xml
+<source>
+    <redis>
+        <host>localhost</host>
+        <port>6379</port>
+    </redis>
+</source>
+```
+
+Setting fields:
+
+- `host` – The Redis host.
+- `port` – The port on the Redis server.
+
 [Original article](https://clickhouse.yandex/docs/en/query_language/dicts/external_dicts_dict_sources/) <!--hide-->