Add tests

2024-11-22 07:31:57 +00:00 · 2019-02-12 12:23:22 +03:00 · 2019-02-12 12:23:22 +03:00 · 6e28c22876
commit 6e28c22876
parent 933906403a
6 changed files with 133 additions and 2 deletions
--- a/cmake/find_poco.cmake
+++ b/cmake/find_poco.cmake
@ -132,7 +132,7 @@ if (Poco_SQLODBC_LIBRARY AND ODBC_FOUND)
    set (USE_POCO_SQLODBC 1)
 endif ()

-message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_Crypto_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_SQL_LIBRARY},${Poco_SQLODBC_LIBRARY},${Poco_MongoDB_LIBRARY},${Poco_Redis_INCLUDE_DIR}; MongoDB=${USE_POCO_MONGODB}, Redis=${USE_POCO_REDIS}, DataODBC=${USE_POCO_DATAODBC}, NetSSL=${USE_POCO_NETSSL}")
+message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_Crypto_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_SQL_LIBRARY},${Poco_SQLODBC_LIBRARY},${Poco_MongoDB_LIBRARY},${Poco_Redis_LIBRARY}; MongoDB=${USE_POCO_MONGODB}, Redis=${USE_POCO_REDIS}, DataODBC=${USE_POCO_DATAODBC}, NetSSL=${USE_POCO_NETSSL}")

 # How to make sutable poco:
 # use branch:
--- a/dbms/src/Dictionaries/CMakeLists.txt
+++ b/dbms/src/Dictionaries/CMakeLists.txt
@ -37,7 +37,11 @@ if(USE_POCO_MONGODB)
 endif()

 if(USE_POCO_REDIS)
-    target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${Poco_Redis_INCLUDE_DIR})
+    # for code highlighting in CLion
+    # target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${Poco_Redis_INCLUDE_DIR})
+
+    # for build
+    target_link_libraries(clickhouse_dictionaries PRIVATE ${Poco_Redis_LIBRARY})
 endif()

 add_subdirectory(Embedded)
--- a/dbms/src/Dictionaries/registerDictionaries.cpp
+++ b/dbms/src/Dictionaries/registerDictionaries.cpp
@ -7,6 +7,7 @@ void registerDictionarySourceFile(DictionarySourceFactory & source_factory);
 void registerDictionarySourceMysql(DictionarySourceFactory & source_factory);
 void registerDictionarySourceClickHouse(DictionarySourceFactory & source_factory);
 void registerDictionarySourceMongoDB(DictionarySourceFactory & source_factory);
+void registerDictionarySourceRedis(DictionarySourceFactory & source_factory);
 void registerDictionarySourceXDBC(DictionarySourceFactory & source_factory);
 void registerDictionarySourceJDBC(DictionarySourceFactory & source_factory);
 void registerDictionarySourceExecutable(DictionarySourceFactory & source_factory);
@ -30,6 +31,7 @@ void registerDictionaries()
        registerDictionarySourceMysql(source_factory);
        registerDictionarySourceClickHouse(source_factory);
        registerDictionarySourceMongoDB(source_factory);
+        registerDictionarySourceRedis(source_factory);
        registerDictionarySourceXDBC(source_factory);
        registerDictionarySourceJDBC(source_factory);
        registerDictionarySourceExecutable(source_factory);
--- a/dbms/tests/external_dictionaries/generate_and_test.py
+++ b/dbms/tests/external_dictionaries/generate_and_test.py
@ -119,6 +119,17 @@ def generate_structure(args):
            [ 'mongodb_user_flat', 0, True ],
        ])

+    if not args.no_redis:
+        dictionaries.extend([
+            [ 'redis_flat', 0, True ],
+            [ 'redis_hashed', 0, True ],
+            [ 'redis_cache', 0, True ],
+            [ 'redis_complex_integers_key_hashed', 1, False ],
+            [ 'redis_complex_integers_key_cache', 1, False ],
+            [ 'redis_complex_mixed_key_hashed', 2, False ],
+            [ 'redis_complex_mixed_key_cache', 2, False ],
+        ])
+
    if args.use_lib:
        dictionaries.extend([
            # [ 'library_flat', 0, True ],
@ -382,6 +393,51 @@ def generate_data(args):
            print 'Could not create MongoDB collection'
            exit(-1)

+    # create Redis storage from complete_query via JSON file
+    if not args.no_redis:
+        print 'Creating Redis storage'
+        table_rows = json.loads(subprocess.check_output([
+            args.client,
+            '--port',
+            args.port,
+            '--output_format_json_quote_64bit_integers',
+            '0',
+            '--query',
+            "select * from test.dictionary_source where not ignore(" \
+            "concat('new Date(\\'', toString(Date_), '\\')') as Date_, " \
+            "concat('new ISODate(\\'', replaceOne(toString(DateTime_, 'UTC'), ' ', 'T'), 'Z\\')') as DateTime_" \
+            ") format JSON"
+        ]))['data']
+
+        # print json.dumps(table_rows)
+
+        # For Integers the first byte of the reply is ":"
+        # For Bulk Strings the first byte of the reply is "$"
+
+        proto_for_redis = ""
+        for counter, collection in enumerate(table_rows):
+            proto_for_redis += "SELECT " + str(counter) + "\r\n"
+            proto_for_redis += "FLUSHDB\r\n"
+            for key, value in collection.iteritems():
+                value_type = "$"
+                if isinstance(value, int):
+                    value_type = ":"
+                else:
+                    value = str(value)
+                    if "Date" in value:
+                        value = value[value.find("'") + 1:-2]
+
+                proto_for_redis += "SET " + "$" + key + " " + value_type + str(value) + "\r\n"
+
+        # with open("clickhouse_redis.log", "w") as f:
+        #     f.write(json.dumps(table_rows) + "\n" + proto_for_redis + "\n")
+
+        open('generated/full.json', 'w').write(proto_for_redis)
+        result = system('cat {0}/full.json | redis-cli > \\dev\\null'.format(args.generated))
+        if result != 0:
+            print 'Could not create Redis storage'
+            exit(-1)
+

 def generate_dictionaries(args):
    dictionary_skeleton = '''
@ -482,6 +538,13 @@ def generate_dictionaries(args):
    </mongodb>
    '''.format(mongo_host=args.mongo_host)

+    source_redis = '''
+    <redis>
+        <host>{redis_host}</host>
+        <port>6379</port>
+    </redis>
+    '''.format(redis_host=args.redis_host)
+
    source_executable = '''
    <executable>
        <command>cat %s</command>
@ -668,6 +731,17 @@ def generate_dictionaries(args):
        [ source_mongodb_user, layout_flat ],
    ])

+    if not args.no_redis:
+        sources_and_layouts.extend([
+        [ source_redis, layout_flat ],
+        [ source_redis, layout_hashed ],
+        [ source_redis, layout_cache ],
+        [ source_redis, layout_complex_key_cache ],
+        [ source_redis, layout_complex_key_hashed ],
+        [ source_redis, layout_complex_key_hashed ],
+        [ source_redis, layout_complex_key_cache ],
+    ])
+
    if args.use_lib:
        sources_and_layouts.extend([
        #[ source_library, layout_flat ],
@ -947,6 +1021,8 @@ if __name__ == '__main__':
    parser.add_argument('--no_mongo', action='store_true', help = 'Dont use mongodb dictionaries')
    parser.add_argument('--mongo_host', default = 'localhost', help = 'mongo server host')
    parser.add_argument('--use_mongo_user', action='store_true', help = 'Test mongodb with user-pass')
+    parser.add_argument('--no_redis', action='store_true', help = 'Dont use redis dictionaries')
+    parser.add_argument('--redis_host', default = 'localhost', help = 'redis server host')

    parser.add_argument('--no_http', action='store_true', help = 'Dont use http dictionaries')
    parser.add_argument('--http_port', default = 58000, help = 'http server port')
--- a/dbms/tests/external_dictionaries/run.sh
+++ b/dbms/tests/external_dictionaries/run.sh
@ -9,6 +9,7 @@ fi

 NO_MYSQL=0
 NO_MONGO=0
+NO_REDIS=0

 for arg in "$@"; do
    if [ "$arg" = "--no_mysql" ]; then
@ -17,6 +18,9 @@ for arg in "$@"; do
    if [ "$arg" == "--no_mongo" ]; then
        NO_MONGO=1
    fi
+    if [ "$arg" == "--no_redis" ]; then
+        NO_REDIS=1
+    fi
 done

 # MySQL
@ -101,6 +105,31 @@ else
    fi
 fi

+# Redis
+if [ $NO_REDIS -eq 1 ]; then
+    echo "Not using Redis"
+else
+    if [ -z $(which redis-cli) ]; then
+        echo 'Installing Redis'
+
+        sudo apt-get update &>/dev/null
+        sudo apt-get install redis-server
+
+        which redis-server >/dev/null
+        if [ $? -ne 0 ]; then
+            echo 'Failed installing redis-server'
+            exit -1
+        fi
+    fi
+
+    echo | redis-cli &>/dev/null
+    if [ $? -ne 0 ]; then
+        sudo systemctl start redis.service
+    else
+        echo 'Redis already started'
+    fi
+fi
+
 # ClickHouse
 clickhouse-server &> clickhouse.log &
 sleep 3
--- a/docs/en/query_language/dicts/external_dicts_dict_sources.md
+++ b/docs/en/query_language/dicts/external_dicts_dict_sources.md
@ -30,6 +30,7 @@ Types of sources (`source_type`):
    - [MySQL](#dicts-external_dicts_dict_sources-mysql)
    - [ClickHouse](#dicts-external_dicts_dict_sources-clickhouse)
    - [MongoDB](#dicts-external_dicts_dict_sources-mongodb)
+    - [Redis](#dicts-external_dicts_dict_sources-redis)
    - [ODBC](#dicts-external_dicts_dict_sources-odbc)


@ -421,4 +422,23 @@ Setting fields:
 - `db` – Name of the database.
 - `collection` – Name of the collection.

+
+### Redis {#dicts-external_dicts_dict_sources-redis}
+
+Example of settings:
+
+```xml
+<source>
+    <redis>
+        <host>localhost</host>
+        <port>6379</port>
+    </redis>
+</source>
+```
+
+Setting fields:
+
+- `host` – The Redis host.
+- `port` – The port on the Redis server.
+
 [Original article](https://clickhouse.yandex/docs/en/query_language/dicts/external_dicts_dict_sources/) <!--hide-->