Add settings changes check to CI

.github/workflows/pull_request.yml

@@ -128,6 +128,14 @@ jobs:
       data: ${{ needs.RunConfig.outputs.data }}
       run_command: |
         python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
+  SettingsChangesCheck:
+    needs: [ RunConfig, BuilderDebRelease ]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Settings changes check
+      runner_type: style-checker
+      data: ${{ needs.RunConfig.outputs.data }}
 #########################################################################################
 #################################### ORDINARY BUILDS ####################################
 #########################################################################################
@@ -928,6 +936,7 @@ jobs:
       - CompatibilityCheckAarch64
       - SQLancerTestRelease
       - SQLancerTestDebug
+      - SettingsChangesCheck
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Check out repository code

docker/images.json

@@ -61,6 +61,10 @@
         "name": "clickhouse/upgrade-check",
         "dependent": []
     },
+    "docker/test/settings_changes": {
+        "name": "clickhouse/settings-changes-check",
+        "dependent": []
+    },
     "docker/test/integration/runner": {
         "only_amd64": true,
         "name": "clickhouse/integration-tests-runner",

docker/test/settings_changes/Dockerfile

@@ -0,0 +1,19 @@
+# rebuild in #33610
+# docker build -t clickhouse/settings-changes-check .
+ARG FROM_TAG=latest
+FROM clickhouse/test-base:$FROM_TAG
+
+RUN apt-get update -y \
+    && env DEBIAN_FRONTEND=noninteractive \
+        apt-get install --yes --no-install-recommends \
+            bash \
+            python3 \
+            python3-lxml \
+            python3-termcolor \
+            python3-requests \
+            curl \
+            sudo \
+    && apt-get clean
+
+COPY run.sh /
+CMD ["/bin/bash", "/run.sh"]

docker/test/settings_changes/run.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+set -x
+
+# we mount tests folder from repo to /usr/share
+ln -s /usr/share/clickhouse-test/ci/download_release_packages.py /usr/bin/download_release_packages
+ln -s /usr/share/clickhouse-test/ci/get_previous_release_tag.py /usr/bin/get_previous_release_tag
+
+echo "Get previous release tag"
+previous_release_tag=$(dpkg --info package_folder/clickhouse-client*.deb | grep "Version: " | awk '{print $2}' | cut -f1 -d'+' | get_previous_release_tag)
+echo $previous_release_tag
+
+echo "Download clickhouse packages from the previous release"
+mkdir previous_release_package_folder
+
+echo $previous_release_tag | download_release_packages && echo -e "Download script exit code$OK" >> /test_output/test_results.tsv \
+    || echo -e "Download script failed$FAIL" >> /test_output/test_results.tsv
+
+# Check if we downloaded previous release packages successfully
+if ! [ "$(ls -A previous_release_package_folder/clickhouse-common-static_*.deb)" ]
+then
+    echo -e 'failure\tFailed to download previous release packages' > /test_output/check_status.tsv
+    exit
+fi
+
+echo -e "Successfully downloaded previous release packages\tOK" >> /test_output/test_results.tsv
+
+dpkg-deb -xv package_folder/clickhouse-common-static_*.deb package_folder/
+dpkg-deb -xv previous_release_package_folder/clickhouse-common-static_*.deb previous_release_package_folder/
+
+if ! [ "$(ls -A package_folder/usr/bin/clickhouse && ls -A previous_release_package_folder/usr/bin/clickhouse)" ]
+then
+    echo -e 'failure\tFailed extract clickhouse binary from deb packages' > /test_output/check_status.tsv
+    exit
+fi
+
+package_folder/usr/bin/clickhouse local -q "select * from system.settings format Native" > new_settings.native
+previous_release_package_folder/usr/bin/clickhouse local -q "select * from system.settings format Native" > old_settings.native
+
+package_folder/usr/bin/clickhouse local -nmq "
+CREATE TABLE old_settings AS file('old_settings.native');
+CREATE TABLE new_settings AS file('new_settings.native');
+
+SELECT
+    name,
+    new_settings.value AS new_value,
+    old_settings.value AS old_value
+FROM new_settings
+LEFT JOIN old_settings ON new_settings.name = old_settings.name
+WHERE (new_settings.value != old_settings.value) AND (name NOT IN (
+    SELECT arrayJoin(tupleElement(changes, 'name'))
+    FROM system.settings_changes
+    WHERE version = extract(version(), '^(?:\\d+\\.\\d+)')
+))
+SETTINGS join_use_nulls = 1
+INTO OUTFILE 'changed_settings.txt'
+FORMAT Pretty;
+
+SELECT name
+FROM new_settings
+WHERE (name NOT IN (
+    SELECT name
+    FROM old_settings
+)) AND (name NOT IN (
+    SELECT arrayJoin(tupleElement(changes, 'name'))
+    FROM system.settings_changes
+    WHERE version = '23.9'
+))
+INTO OUTFILE 'new_settings.txt'
+FORMAT Pretty;
+"
+
+description="OK"
+status="success"
+
+if [ -s changed_settings.txt ]
+then
+    mv changed_settings.txt /test_output/
+    echo -e "Changed settings are not reflected in settings changes history (see changed_settings.txt)\tFAIL" >> /test_output/test_results.tsv
+    description="New or changed settings are not reflected in settings changes history"
+    status="failure"
+else
+    echo -e "There are no changed settings or they are reflected in settings changes history\tOK" >> /test_output/test_results.tsv
+fi
+
+if [ -s new_settings.txt ]
+then
+    mv new_settings.txt /test_output/
+    echo -e "New settings are not reflected in settings changes history (see new_settings.txt)\tFAIL" >> /test_output/test_results.tsv
+    description="New of changed settings are not reflected in settings changes history"
+    status="failure"
+else
+    echo -e "There are no new settings or they are reflected in settings changes history\tOK" >> /test_output/test_results.tsv
+fi
+
+echo -e "$status\t$description" > /test_output/check_status.tsv

tests/ci/ci_config.py

@@ -195,6 +195,11 @@ bugfix_validate_check = DigestConfig(
         "clickhouse/stateless-test",
     ],
 )
+settings_changes_check_digest = DigestConfig(
+    include_paths=["./tests/ci/settings_changes_check.py"],
+    exclude_files=[".md"],
+    docker=["clickhouse/settings-changes-check"],
+)
 # common test params
 statless_test_common_params = {
     "digest": statless_check_digest,
@@ -246,6 +251,11 @@ sql_test_params = {
     "run_command": "sqltest.py",
     "timeout": 10800,
 }
+settings_changes_test_params = {
+    "digest": settings_changes_check_digest,
+    "run_command": "settings_changes_check.py",
+    "timeout": 1800,
+}
 
 
 @dataclass
@@ -750,6 +760,9 @@ CI_CONFIG = CiConfig(
         "Upgrade check (debug)": TestConfig(
             "package_debug", job_config=JobConfig(**upgrade_test_common_params)  # type: ignore
         ),
+        "Settings changes check": TestConfig(
+            "package_release", job_config=JobConfig(**settings_changes_test_params)  # type: ignore
+        ),
         "Integration tests (asan)": TestConfig(
             "package_asan",
             job_config=JobConfig(num_batches=4, **integration_test_common_params),  # type: ignore
@@ -1019,6 +1032,12 @@ CHECK_DESCRIPTIONS = [
         "successfully startup without any errors, crashes or sanitizer asserts",
         lambda x: x.startswith("Upgrade check ("),
     ),
+    CheckDescription(
+        "Settings changes check",
+        "Compares the list of settings from last release and from the PR and "
+        "checks that all changes are reflected in settings changes history",
+        lambda x: x == "Settings changes check",
+    ),
     CheckDescription(
         "ClickBench",
         "Runs [ClickBench](https://github.com/ClickHouse/ClickBench/) with instant-attach table",

tests/ci/settings_changes_check.py

@@ -0,0 +1,5 @@
+import stress_check
+
+# We can reuse logic of running stress test as it's the same
+if __name__ == "__main__":
+    stress_check.run_stress_test("clickhouse/settings-changes-check")