ClickHouse/tests/integration/test_graphite_merge_tree/test.py

import datetime
import os.path as p
import time

import pytest
from helpers.client import QueryRuntimeException
from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV
from helpers.test_tools import csv_compare

cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance(
    "instance",
    main_configs=["configs/graphite_rollup.xml"],
    user_configs=["configs/users.xml"],
)
q = instance.query


@pytest.fixture(scope="module")
def started_cluster():
    try:
        cluster.start()
        q("CREATE DATABASE test")

        yield cluster

    finally:
        cluster.shutdown()


@pytest.fixture
def graphite_table(started_cluster):
    q(
        """
DROP TABLE IF EXISTS test.graphite;
CREATE TABLE test.graphite
    (metric String, value Float64, timestamp UInt32, date Date, updated UInt32)
    ENGINE = GraphiteMergeTree('graphite_rollup')
    PARTITION BY toYYYYMM(date)
    ORDER BY (metric, timestamp)
    SETTINGS index_granularity=8192;
"""
    )

    yield

    q("DROP TABLE test.graphite")


def test_rollup_versions(graphite_table):
    timestamp = int(time.time())
    rounded_timestamp = timestamp - timestamp % 60
    date = datetime.date.today().isoformat()

    # Insert rows with timestamps relative to the current time so that the
    # first retention clause is active.
    # Two parts are created.
    q(
        """
INSERT INTO test.graphite (metric, value, timestamp, date, updated)
      VALUES ('one_min.x1', 100, {timestamp}, '{date}', 1);
INSERT INTO test.graphite (metric, value, timestamp, date, updated)
      VALUES ('one_min.x1', 200, {timestamp}, '{date}', 2);
""".format(
            timestamp=timestamp, date=date
        )
    )

    expected1 = """\
one_min.x1	100	{timestamp}	{date}	1
one_min.x1	200	{timestamp}	{date}	2
""".format(
        timestamp=timestamp, date=date
    )

    assert TSV(q("SELECT * FROM test.graphite ORDER BY updated")) == TSV(expected1)

    q("OPTIMIZE TABLE test.graphite")

    # After rollup only the row with max version is retained.
    expected2 = """\
one_min.x1	200	{timestamp}	{date}	2
""".format(
        timestamp=rounded_timestamp, date=date
    )

    assert TSV(q("SELECT * FROM test.graphite")) == TSV(expected2)


def test_rollup_aggregation(graphite_table):
    # This query essentially emulates what rollup does.
    result1 = q(
        """
SELECT avg(v), max(upd)
FROM (SELECT timestamp,
            argMax(value, (updated, number)) AS v,
            max(updated) AS upd
      FROM (SELECT 'one_min.x5' AS metric,
                   toFloat64(number) AS value,
                   toUInt32(1111111111 + intDiv(number, 3)) AS timestamp,
                   toDate('2017-02-02') AS date,
                   toUInt32(intDiv(number, 2)) AS updated,
                   number
            FROM system.numbers LIMIT 1000000)
      WHERE intDiv(timestamp, 600) * 600 = 1111444200
      GROUP BY timestamp)
"""
    )

    expected1 = """\
999634.9918367347	499999
"""
    assert TSV(result1) == TSV(expected1)

    # Timestamp 1111111111 is in sufficiently distant past
    # so that the last retention clause is active.
    result2 = q(
        """
INSERT INTO test.graphite
    SELECT 'one_min.x' AS metric,
           toFloat64(number) AS value,
           toUInt32(1111111111 + intDiv(number, 3)) AS timestamp,
           toDate('2017-02-02') AS date, toUInt32(intDiv(number, 2)) AS updated
    FROM (SELECT * FROM system.numbers LIMIT 1000000)
    WHERE intDiv(timestamp, 600) * 600 = 1111444200;

OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL;

SELECT * FROM test.graphite;
"""
    )

    expected2 = """\
one_min.x	999634.9918367347	1111444200	2017-02-02	499999
"""

    assert TSV(result2) == TSV(expected2)


def test_rollup_aggregation_2(graphite_table):
    result = q(
        """
INSERT INTO test.graphite
    SELECT 'one_min.x' AS metric,
           toFloat64(number) AS value,
           toUInt32(1111111111 - intDiv(number, 3)) AS timestamp,
           toDate('2017-02-02') AS date,
           toUInt32(100 - number) AS updated
    FROM (SELECT * FROM system.numbers LIMIT 50);

OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL;

SELECT * FROM test.graphite;
"""
    )

    expected = """\
one_min.x	24	1111110600	2017-02-02	100
"""

    assert TSV(result) == TSV(expected)


def test_multiple_paths_and_versions(graphite_table):
    result = q(
        """
INSERT INTO test.graphite
    SELECT 'one_min.x' AS metric,
           toFloat64(number) AS value,
           toUInt32(1111111111 + intDiv(number, 3) * 600) AS timestamp,
           toDate('2017-02-02') AS date,
           toUInt32(100 - number) AS updated
    FROM (SELECT * FROM system.numbers LIMIT 50);

OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL;

SELECT * FROM test.graphite;


INSERT INTO test.graphite
    SELECT 'one_min.y' AS metric,
           toFloat64(number) AS value,
           toUInt32(1111111111 + number * 600) AS timestamp,
           toDate('2017-02-02') AS date,
           toUInt32(100 - number) AS updated
    FROM (SELECT * FROM system.numbers LIMIT 50);

OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL;

SELECT * FROM test.graphite;
"""
    )

    with open(
        p.join(p.dirname(__file__), "test_multiple_paths_and_versions.reference")
    ) as reference:
        assert TSV(result) == TSV(reference)


def test_multiple_output_blocks(graphite_table):
    MERGED_BLOCK_SIZE = 8192

    to_insert = ""
    expected = ""
    for i in range(2 * MERGED_BLOCK_SIZE + 1):
        rolled_up_time = 1000000200 + 600 * i

        for j in range(3):
            cur_time = rolled_up_time + 100 * j
            to_insert += "one_min.x1	{}	{}	2001-09-09	1\n".format(10 * j, cur_time)
            to_insert += "one_min.x1	{}	{}	2001-09-09	2\n".format(
                10 * (j + 1), cur_time
            )

        expected += "one_min.x1	20	{}	2001-09-09	2\n".format(rolled_up_time)

    q("INSERT INTO test.graphite FORMAT TSV", to_insert)

    result = q(
        """
OPTIMIZE TABLE test.graphite PARTITION 200109 FINAL;

SELECT * FROM test.graphite;
"""
    )

    assert TSV(result) == TSV(expected)


def test_paths_not_matching_any_pattern(graphite_table):
    to_insert = """\
one_min.x1	100	1000000000	2001-09-09	1
zzzzzzzz	100	1000000001	2001-09-09	1
zzzzzzzz	200	1000000001	2001-09-09	2
"""

    q("INSERT INTO test.graphite FORMAT TSV", to_insert)

    expected = """\
one_min.x1	100	999999600	2001-09-09	1
zzzzzzzz	200	1000000001	2001-09-09	2
"""

    result = q(
        """
OPTIMIZE TABLE test.graphite PARTITION 200109 FINAL;

SELECT * FROM test.graphite;
"""
    )

    assert TSV(result) == TSV(expected)


def test_system_graphite_retentions(graphite_table):
    expected = """
graphite_rollup	all	\\\\.count$	sum	0	0	1	0	['test']	['graphite']
graphite_rollup	all	\\\\.max$	max	0	0	2	0	['test']	['graphite']
graphite_rollup	all	^five_min\\\\.		31536000	14400	3	0	['test']	['graphite']
graphite_rollup	all	^five_min\\\\.		5184000	3600	3	0	['test']	['graphite']
graphite_rollup	all	^five_min\\\\.		0	300	3	0	['test']	['graphite']
graphite_rollup	all	^one_min	avg	31536000	600	4	0	['test']	['graphite']
graphite_rollup	all	^one_min	avg	7776000	300	4	0	['test']	['graphite']
graphite_rollup	all	^one_min	avg	0	60	4	0	['test']	['graphite']
    """
    result = q("SELECT * from system.graphite_retentions")

    mismatch = csv_compare(result, expected)
    assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected}\ndiff\n{mismatch}\n"

    q(
        """
DROP TABLE IF EXISTS test.graphite2;
CREATE TABLE test.graphite2
    (metric String, value Float64, timestamp UInt32, date Date, updated UInt32)
    ENGINE = GraphiteMergeTree('graphite_rollup')
    PARTITION BY toYYYYMM(date)
    ORDER BY (metric, timestamp)
    SETTINGS index_granularity=8192;
    """
    )
    expected = """
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
    """
    result = q(
        """
    SELECT
        config_name,
        Tables.database,
        Tables.table
    FROM system.graphite_retentions
    """
    )
    assert TSV(result) == TSV(expected)


def test_path_dangling_pointer(graphite_table):
    q(
        """
DROP TABLE IF EXISTS test.graphite2;
CREATE TABLE test.graphite2
    (metric String, value Float64, timestamp UInt32, date Date, updated UInt32)
    ENGINE = GraphiteMergeTree('graphite_rollup')
    PARTITION BY toYYYYMM(date)
    ORDER BY (metric, timestamp)
    SETTINGS index_granularity=1;
    """
    )

    path = "abcd" * 4000000  # 16MB
    q(
        "INSERT INTO test.graphite2 FORMAT TSV",
        "{}\t0.0\t0\t2018-01-01\t100\n".format(path),
    )
    q(
        "INSERT INTO test.graphite2 FORMAT TSV",
        "{}\t0.0\t0\t2018-01-01\t101\n".format(path),
    )
    for version in range(10):
        q(
            "INSERT INTO test.graphite2 FORMAT TSV",
            "{}\t0.0\t0\t2018-01-01\t{}\n".format(path, version),
        )

    while True:
        q("OPTIMIZE TABLE test.graphite2 PARTITION 201801 FINAL")
        parts = int(
            q(
                "SELECT count() FROM system.parts "
                "WHERE active AND database='test' "
                "AND table='graphite2'"
            )
        )
        if parts == 1:
            break
        print(("Parts", parts))

    assert TSV(q("SELECT value, timestamp, date, updated FROM test.graphite2")) == TSV(
        "0\t0\t2018-01-01\t101\n"
    )

    q("DROP TABLE test.graphite2")


def test_combined_rules(graphite_table):
    # 1487970000 ~ Sat 25 Feb 00:00:00 MSK 2017
    to_insert = "INSERT INTO test.graphite VALUES "
    expected_unmerged = ""
    for i in range(384):
        to_insert += "('five_min.count', {v}, {t}, toDate({t}), 1), ".format(
            v=1, t=1487970000 + (i * 300)
        )
        to_insert += "('five_min.max', {v}, {t}, toDate({t}), 1), ".format(
            v=i, t=1487970000 + (i * 300)
        )
        expected_unmerged += (
            "five_min.count\t{v1}\t{t}\n" "five_min.max\t{v2}\t{t}\n"
        ).format(v1=1, v2=i, t=1487970000 + (i * 300))

    q(to_insert)
    assert TSV(
        q(
            "SELECT metric, value, timestamp FROM test.graphite"
            " ORDER BY (timestamp, metric)"
        )
    ) == TSV(expected_unmerged)

    q("OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL")
    expected_merged = """
        five_min.count	48	1487970000	2017-02-25	1
        five_min.count	48	1487984400	2017-02-25	1
        five_min.count	48	1487998800	2017-02-25	1
        five_min.count	48	1488013200	2017-02-25	1
        five_min.count	48	1488027600	2017-02-25	1
        five_min.count	48	1488042000	2017-02-25	1
        five_min.count	48	1488056400	2017-02-26	1
        five_min.count	48	1488070800	2017-02-26	1
        five_min.max	47	1487970000	2017-02-25	1
        five_min.max	95	1487984400	2017-02-25	1
        five_min.max	143	1487998800	2017-02-25	1
        five_min.max	191	1488013200	2017-02-25	1
        five_min.max	239	1488027600	2017-02-25	1
        five_min.max	287	1488042000	2017-02-25	1
        five_min.max	335	1488056400	2017-02-26	1
        five_min.max	383	1488070800	2017-02-26	1
    """
    assert TSV(q("SELECT * FROM test.graphite" " ORDER BY (metric, timestamp)")) == TSV(
        expected_merged
    )


def test_combined_rules_with_default(graphite_table):
    q(
        """
DROP TABLE IF EXISTS test.graphite;
CREATE TABLE test.graphite
    (metric String, value Float64, timestamp UInt32, date Date, updated UInt32)
    ENGINE = GraphiteMergeTree('graphite_rollup_with_default')
    PARTITION BY toYYYYMM(date)
    ORDER BY (metric, timestamp)
    SETTINGS index_granularity=1;
      """
    )
    # 1487970000 ~ Sat 25 Feb 00:00:00 MSK 2017
    to_insert = "INSERT INTO test.graphite VALUES "
    expected_unmerged = ""
    for i in range(100):
        to_insert += "('top_level.count', {v}, {t}, toDate({t}), 1), ".format(
            v=1, t=1487970000 + (i * 60)
        )
        to_insert += "('top_level.max', {v}, {t}, toDate({t}), 1), ".format(
            v=i, t=1487970000 + (i * 60)
        )
        expected_unmerged += (
            "top_level.count\t{v1}\t{t}\n" "top_level.max\t{v2}\t{t}\n"
        ).format(v1=1, v2=i, t=1487970000 + (i * 60))

    q(to_insert)
    assert TSV(
        q(
            "SELECT metric, value, timestamp FROM test.graphite"
            " ORDER BY (timestamp, metric)"
        )
    ) == TSV(expected_unmerged)

    q("OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL")
    expected_merged = """
        top_level.count	10	1487970000	2017-02-25	1
        top_level.count	10	1487970600	2017-02-25	1
        top_level.count	10	1487971200	2017-02-25	1
        top_level.count	10	1487971800	2017-02-25	1
        top_level.count	10	1487972400	2017-02-25	1
        top_level.count	10	1487973000	2017-02-25	1
        top_level.count	10	1487973600	2017-02-25	1
        top_level.count	10	1487974200	2017-02-25	1
        top_level.count	10	1487974800	2017-02-25	1
        top_level.count	10	1487975400	2017-02-25	1
        top_level.max	9	1487970000	2017-02-25	1
        top_level.max	19	1487970600	2017-02-25	1
        top_level.max	29	1487971200	2017-02-25	1
        top_level.max	39	1487971800	2017-02-25	1
        top_level.max	49	1487972400	2017-02-25	1
        top_level.max	59	1487973000	2017-02-25	1
        top_level.max	69	1487973600	2017-02-25	1
        top_level.max	79	1487974200	2017-02-25	1
        top_level.max	89	1487974800	2017-02-25	1
        top_level.max	99	1487975400	2017-02-25	1
    """
    assert TSV(q("SELECT * FROM test.graphite" " ORDER BY (metric, timestamp)")) == TSV(
        expected_merged
    )


def test_broken_partial_rollup(graphite_table):
    q(
        """
DROP TABLE IF EXISTS test.graphite;
CREATE TABLE test.graphite
    (metric String, value Float64, timestamp UInt32, date Date, updated UInt32)
    ENGINE = GraphiteMergeTree('graphite_rollup_broken')
    PARTITION BY toYYYYMM(date)
    ORDER BY (metric, timestamp)
    SETTINGS index_granularity=1;
      """
    )
    to_insert = """\
one_min.x1	100	1000000000	2001-09-09	1
zzzzzzzz	100	1000000001	2001-09-09	1
zzzzzzzz	200	1000000001	2001-09-09	2
"""

    q("INSERT INTO test.graphite FORMAT TSV", to_insert)

    expected = """\
one_min.x1	100	1000000000	2001-09-09	1
zzzzzzzz	200	1000000001	2001-09-09	2
"""

    result = q(
        """
OPTIMIZE TABLE test.graphite PARTITION 200109 FINAL;

SELECT * FROM test.graphite;
"""
    )

    assert TSV(result) == TSV(expected)


def test_wrong_rollup_config(graphite_table):
    with pytest.raises(QueryRuntimeException) as exc:
        q(
            """
CREATE TABLE test.graphite_not_created
    (metric String, value Float64, timestamp UInt32, date Date, updated UInt32)
    ENGINE = GraphiteMergeTree('graphite_rollup_wrong_age_precision')
    PARTITION BY toYYYYMM(date)
    ORDER BY (metric, timestamp)
    SETTINGS index_granularity=1;
        """
        )

    # The order of retentions is not guaranteed
    assert "age and precision should only grow up: " in str(exc.value)
    assert "36000:600" in str(exc.value)
    assert "72000:300" in str(exc.value)