Merge pull request #44848 from qoega/green-ci-2

Pre-pulling images for integration tests to detect broken infrastructure earlier

docker/test/integration/runner/compose/docker_compose_clickhouse.yml

@@ -0,0 +1,5 @@
+version: '2.3'
+# Used to pre-pull images with docker-compose
+services:
+  clickhouse1:
+    image: clickhouse/integration-test

docker/test/integration/runner/compose/docker_compose_hdfs.yml

@@ -5,8 +5,8 @@ services:
         hostname: hdfs1
         restart: always
         expose:
-            - ${HDFS_NAME_PORT}
+            - ${HDFS_NAME_PORT:-50070}
-            - ${HDFS_DATA_PORT}
+            - ${HDFS_DATA_PORT:-50075}
         entrypoint: /etc/bootstrap.sh -d
         volumes:
             - type: ${HDFS_FS:-tmpfs}

docker/test/integration/runner/compose/docker_compose_kafka.yml

@@ -15,7 +15,7 @@ services:
     image: confluentinc/cp-kafka:5.2.0
     hostname: kafka1
     ports:
-      - ${KAFKA_EXTERNAL_PORT}:${KAFKA_EXTERNAL_PORT}
+      - ${KAFKA_EXTERNAL_PORT:-8081}:${KAFKA_EXTERNAL_PORT:-8081}
     environment:
       KAFKA_ADVERTISED_LISTENERS: INSIDE://localhost:${KAFKA_EXTERNAL_PORT},OUTSIDE://kafka1:19092
       KAFKA_ADVERTISED_HOST_NAME: kafka1
@@ -35,7 +35,7 @@ services:
     image: confluentinc/cp-schema-registry:5.2.0
     hostname: schema-registry
     ports:
-      - ${SCHEMA_REGISTRY_EXTERNAL_PORT}:${SCHEMA_REGISTRY_INTERNAL_PORT}
+      - ${SCHEMA_REGISTRY_EXTERNAL_PORT:-12313}:${SCHEMA_REGISTRY_INTERNAL_PORT:-12313}
     environment:
       SCHEMA_REGISTRY_HOST_NAME: schema-registry
       SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT

docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml

@@ -15,8 +15,8 @@ services:
         source: ${KERBERIZED_HDFS_LOGS:-}
         target: /var/log/hadoop-hdfs
     expose:
-      - ${KERBERIZED_HDFS_NAME_PORT}
+      - ${KERBERIZED_HDFS_NAME_PORT:-50070}
-      - ${KERBERIZED_HDFS_DATA_PORT}
+      - ${KERBERIZED_HDFS_DATA_PORT:-1006}
     depends_on:
       - hdfskerberos
     entrypoint: /etc/bootstrap.sh -d

docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml

@@ -23,7 +23,7 @@ services:
     # restart: always
     hostname: kerberized_kafka1
     ports:
-      - ${KERBERIZED_KAFKA_EXTERNAL_PORT}:${KERBERIZED_KAFKA_EXTERNAL_PORT}
+      - ${KERBERIZED_KAFKA_EXTERNAL_PORT:-19092}:${KERBERIZED_KAFKA_EXTERNAL_PORT:-19092}
     environment:
       KAFKA_LISTENERS: OUTSIDE://:19092,UNSECURED_OUTSIDE://:19093,UNSECURED_INSIDE://0.0.0.0:${KERBERIZED_KAFKA_EXTERNAL_PORT}
       KAFKA_ADVERTISED_LISTENERS: OUTSIDE://kerberized_kafka1:19092,UNSECURED_OUTSIDE://kerberized_kafka1:19093,UNSECURED_INSIDE://localhost:${KERBERIZED_KAFKA_EXTERNAL_PORT}
@@ -41,7 +41,7 @@ services:
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/secrets/broker_jaas.conf -Djava.security.krb5.conf=/etc/kafka/secrets/krb.conf -Dsun.security.krb5.debug=true"
     volumes:
-      - ${KERBERIZED_KAFKA_DIR}/secrets:/etc/kafka/secrets
+      - ${KERBERIZED_KAFKA_DIR:-}/secrets:/etc/kafka/secrets
       - /dev/urandom:/dev/random
     depends_on:
       - kafka_kerberized_zookeeper

docker/test/integration/runner/compose/docker_compose_meili.yml

@@ -4,13 +4,13 @@ services:
         image: getmeili/meilisearch:v0.27.0 
         restart: always
         ports:
-            - ${MEILI_EXTERNAL_PORT}:${MEILI_INTERNAL_PORT}
+            - ${MEILI_EXTERNAL_PORT:-7700}:${MEILI_INTERNAL_PORT:-7700}
 
     meili_secure:
         image: getmeili/meilisearch:v0.27.0 
         restart: always
         ports:
-            - ${MEILI_SECURE_EXTERNAL_PORT}:${MEILI_SECURE_INTERNAL_PORT}
+            - ${MEILI_SECURE_EXTERNAL_PORT:-7700}:${MEILI_SECURE_INTERNAL_PORT:-7700}
         environment:
             MEILI_MASTER_KEY: "password"
 

docker/test/integration/runner/compose/docker_compose_minio.yml

@@ -9,7 +9,7 @@ services:
       - data1-1:/data1
       - ${MINIO_CERTS_DIR:-}:/certs
     expose:
-      - ${MINIO_PORT}
+      - ${MINIO_PORT:-9001}
     environment:
       MINIO_ACCESS_KEY: minio
       MINIO_SECRET_KEY: minio123

docker/test/integration/runner/compose/docker_compose_mongo.yml

@@ -7,11 +7,11 @@ services:
             MONGO_INITDB_ROOT_USERNAME: root
             MONGO_INITDB_ROOT_PASSWORD: clickhouse
         ports:
-            - ${MONGO_EXTERNAL_PORT}:${MONGO_INTERNAL_PORT}
+            - ${MONGO_EXTERNAL_PORT:-27017}:${MONGO_INTERNAL_PORT:-27017}
         command: --profile=2 --verbose
 
     mongo2:
         image: mongo:5.0
         restart: always
         ports:
-            - ${MONGO_NO_CRED_EXTERNAL_PORT}:${MONGO_NO_CRED_INTERNAL_PORT}
+            - ${MONGO_NO_CRED_EXTERNAL_PORT:-27017}:${MONGO_NO_CRED_INTERNAL_PORT:-27017}

docker/test/integration/runner/compose/docker_compose_mongo_secure.yml

@@ -7,7 +7,7 @@ services:
             MONGO_INITDB_ROOT_USERNAME: root
             MONGO_INITDB_ROOT_PASSWORD: clickhouse
         volumes:
-            - ${MONGO_CONFIG_PATH}:/mongo/
+            - ${MONGO_CONFIG_PATH:-}:/mongo/
         ports:
-            - ${MONGO_EXTERNAL_PORT}:${MONGO_INTERNAL_PORT}
+            - ${MONGO_EXTERNAL_PORT:-27017}:${MONGO_INTERNAL_PORT:-27017}
         command: --config /mongo/mongo_secure.conf --profile=2 --verbose

docker/test/integration/runner/compose/docker_compose_mysql.yml

@@ -8,7 +8,7 @@ services:
             MYSQL_ROOT_HOST: ${MYSQL_ROOT_HOST}
             DATADIR: /mysql/
         expose:
-            - ${MYSQL_PORT}
+            - ${MYSQL_PORT:-3306}
         command: --server_id=100 
             --log-bin='mysql-bin-1.log' 
             --default-time-zone='+3:00' 

docker/test/integration/runner/compose/docker_compose_mysql_5_7_for_materialized_mysql.yml

@@ -1,21 +0,0 @@
-version: '2.3'
-services:
-    mysql1:
-        image: mysql:5.7
-        restart: 'no'
-        environment:
-            MYSQL_ROOT_PASSWORD: clickhouse
-        ports:
-            - 3308:3306
-        command: --server_id=100 --log-bin='mysql-bin-1.log'
-            --default-time-zone='+3:00'
-            --gtid-mode="ON"
-            --enforce-gtid-consistency
-            --log-error-verbosity=3
-            --log-error=/var/log/mysqld/error.log
-            --general-log=ON
-            --general-log-file=/var/log/mysqld/general.log
-        volumes:
-            - type: ${MYSQL_LOGS_FS:-tmpfs}
-              source: ${MYSQL_LOGS:-}
-              target: /var/log/mysqld/

docker/test/integration/runner/compose/docker_compose_mysql_8_0.yml

@@ -8,7 +8,7 @@ services:
             MYSQL_ROOT_HOST: ${MYSQL_ROOT_HOST}
             DATADIR: /mysql/
         expose:
-            - ${MYSQL8_PORT}
+            - ${MYSQL8_PORT:-3306}
         command: --server_id=100 --log-bin='mysql-bin-1.log' 
             --default_authentication_plugin='mysql_native_password' 
             --default-time-zone='+3:00' --gtid-mode="ON" 

docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml

@@ -8,7 +8,7 @@ services:
             MYSQL_ROOT_HOST: ${MYSQL_CLUSTER_ROOT_HOST}
             DATADIR: /mysql/
         expose:
-            - ${MYSQL_CLUSTER_PORT}
+            - ${MYSQL_CLUSTER_PORT:-3306}
         command: --server_id=100 
             --log-bin='mysql-bin-2.log' 
             --default-time-zone='+3:00' 
@@ -30,7 +30,7 @@ services:
             MYSQL_ROOT_HOST: ${MYSQL_CLUSTER_ROOT_HOST}
             DATADIR: /mysql/
         expose:
-            - ${MYSQL_CLUSTER_PORT}
+            - ${MYSQL_CLUSTER_PORT:-3306}
         command: --server_id=100 
             --log-bin='mysql-bin-3.log' 
             --default-time-zone='+3:00' 
@@ -52,7 +52,7 @@ services:
             MYSQL_ROOT_HOST: ${MYSQL_CLUSTER_ROOT_HOST}
             DATADIR: /mysql/
         expose:
-            - ${MYSQL_CLUSTER_PORT}
+            - ${MYSQL_CLUSTER_PORT:-3306}
         command: --server_id=100 
             --log-bin='mysql-bin-4.log' 
             --default-time-zone='+3:00' 

docker/test/integration/runner/compose/docker_compose_nats.yml

@@ -3,9 +3,9 @@ services:
   nats1:
     image: nats
     ports:
-      - "${NATS_EXTERNAL_PORT}:${NATS_INTERNAL_PORT}"
+      - "${NATS_EXTERNAL_PORT:-4444}:${NATS_INTERNAL_PORT:-4444}"
     command: "-p 4444 --user click --pass house --tls --tlscert=/etc/certs/server-cert.pem --tlskey=/etc/certs/server-key.pem"
     volumes:
       - type: bind
-        source: "${NATS_CERT_DIR}/nats"
+        source: "${NATS_CERT_DIR:-}/nats"
         target: /etc/certs

docker/test/integration/runner/compose/docker_compose_postgres.yml

@@ -5,7 +5,7 @@ services:
         command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=2", "-c", "logging_collector=on", "-c", "log_directory=/postgres/logs", "-c", "log_filename=postgresql.log", "-c", "log_statement=all", "-c", "max_connections=200"]
         restart: always
         expose:
-            - ${POSTGRES_PORT}
+            - ${POSTGRES_PORT:-5432}
         healthcheck:
             test: ["CMD-SHELL", "pg_isready -U postgres"]
             interval: 10s

docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml

@@ -9,7 +9,7 @@ services:
             POSTGRES_PASSWORD: mysecretpassword
             PGDATA: /postgres/data
         expose:
-            - ${POSTGRES_PORT}
+            - ${POSTGRES_PORT:-5432}
         volumes:
             - type: ${POSTGRES_LOGS_FS:-tmpfs}
               source: ${POSTGRES2_DIR:-}
@@ -23,7 +23,7 @@ services:
             POSTGRES_PASSWORD: mysecretpassword
             PGDATA: /postgres/data
         expose:
-            - ${POSTGRES_PORT}
+            - ${POSTGRES_PORT:-5432}
         volumes:
             - type: ${POSTGRES_LOGS_FS:-tmpfs}
               source: ${POSTGRES3_DIR:-}
@@ -37,7 +37,7 @@ services:
             POSTGRES_PASSWORD: mysecretpassword
             PGDATA: /postgres/data
         expose:
-            - ${POSTGRES_PORT}
+            - ${POSTGRES_PORT:-5432}
         volumes:
             - type: ${POSTGRES_LOGS_FS:-tmpfs}
               source: ${POSTGRES4_DIR:-}

docker/test/integration/runner/compose/docker_compose_rabbitmq.yml

@@ -5,7 +5,7 @@ services:
         image: rabbitmq:3.8-management-alpine
         hostname: rabbitmq1
         expose:
-            - ${RABBITMQ_PORT}
+            - ${RABBITMQ_PORT:-5672}
         environment:
             RABBITMQ_DEFAULT_USER: "root"
             RABBITMQ_DEFAULT_PASS: "clickhouse"

docker/test/integration/runner/compose/docker_compose_redis.yml

@@ -4,5 +4,5 @@ services:
         image: redis
         restart: always
         ports:
-            - ${REDIS_EXTERNAL_PORT}:${REDIS_INTERNAL_PORT}
+            - ${REDIS_EXTERNAL_PORT:-6379}:${REDIS_INTERNAL_PORT:-6379}
         command: redis-server --requirepass "clickhouse" --databases 32

tests/ci/integration_test_check.py

@@ -247,6 +247,11 @@ if __name__ == "__main__":
         retcode = process.wait()
         if retcode == 0:
             logging.info("Run tests successfully")
+        elif retcode == 13:
+            logging.warning(
+                "There were issues with infrastructure. Not writing status report to restart job."
+            )
+            sys.exit(1)
         else:
             logging.info("Some tests failed")
 

tests/integration/ci-runner.py

@@ -292,6 +292,34 @@ class ClickhouseIntegrationTestsRunner:
             "clickhouse/postgresql-java-client",
         ]
 
+    def _pre_pull_images(self, repo_path):
+        image_cmd = self._get_runner_image_cmd(repo_path)
+
+        cmd = (
+            "cd {repo_path}/tests/integration && "
+            "timeout -s 9 1h ./runner {runner_opts} {image_cmd} --pre-pull --command '{command}' ".format(
+                repo_path=repo_path,
+                runner_opts=self._get_runner_opts(),
+                image_cmd=image_cmd,
+                command=r""" echo Pre Pull finished """,
+            )
+        )
+
+        for i in range(5):
+            logging.info("Pulling images before running tests. Attempt %s", i)
+            try:
+                subprocess.check_output(
+                    cmd,
+                    shell=True,
+                )
+                return
+            except subprocess.CalledProcessError as err:
+                logging.info("docker-compose pull failed: " + str(err))
+                continue
+        logging.error("Pulling images failed for 5 attempts. Will fail the worker.")
+        # We pass specific retcode to to ci/integration_test_check.py to skip status reporting and restart job
+        exit(13)
+
     def _can_run_with(self, path, opt):
         with open(path, "r") as script:
             for line in script:
@@ -806,6 +834,10 @@ class ClickhouseIntegrationTestsRunner:
             )
 
         self._install_clickhouse(build_path)
+
+        logging.info("Pulling images")
+        runner._pre_pull_images(repo_path)
+
         logging.info(
             "Dump iptables before run %s",
             subprocess.check_output("sudo iptables -nvL", shell=True),

tests/integration/runner

@@ -246,6 +246,10 @@ if __name__ == "__main__":
         "--no-random", action="store", dest="no_random", help="Disable tests order randomization"
     )
 
+    parser.add_argument(
+        "--pre-pull", action="store_true", default=False, dest="pre_pull", help="Pull images for docker_compose before all other actions"
+    )
+
     parser.add_argument(
         "-t",
         "--tests_list",
@@ -383,7 +387,7 @@ if __name__ == "__main__":
     if args.keyword_expression:
         args.pytest_args += ["-k", args.keyword_expression]
 
-    cmd = "docker run {net} {tty} --rm --name {name} --privileged \
+    cmd_base = "docker run {net} {tty} --rm --name {name} --privileged \
         --volume={odbc_bridge_bin}:/clickhouse-odbc-bridge --volume={bin}:/clickhouse \
         --volume={library_bridge_bin}:/clickhouse-library-bridge \
         --volume={base_cfg}:/clickhouse-config --volume={cases_dir}:/ClickHouse/tests/integration \
@@ -392,7 +396,7 @@ if __name__ == "__main__":
         {dockerd_internal_volume} -e DOCKER_CLIENT_TIMEOUT=300 -e COMPOSE_HTTP_TIMEOUT=600 \
         -e XTABLES_LOCKFILE=/run/host/xtables.lock \
         -e PYTHONUNBUFFERED=1 \
-        {env_tags} {env_cleanup} -e PYTEST_OPTS='{parallel} {opts} {tests_list} {rand} -vvv' {img} {command}".format(
+        {env_tags} {env_cleanup} -e PYTEST_OPTS='{parallel} {opts} {tests_list} {rand} -vvv' {img}".format(
         net=net,
         tty=tty,
         bin=args.binary,
@@ -410,9 +414,11 @@ if __name__ == "__main__":
         dockerd_internal_volume=dockerd_internal_volume,
         img=DIND_INTEGRATION_TESTS_IMAGE_NAME + ":" + args.docker_image_version,
         name=CONTAINER_NAME,
-        command=args.command,
     )
 
+    cmd = cmd_base + " " +  args.command
+    cmd_pre_pull = cmd_base + " find /compose -name docker_compose_*.yml -exec docker-compose -f '{}' pull \;"
+
     containers = subprocess.check_output(
         f"docker ps --all --quiet --filter name={CONTAINER_NAME} --format={{{{.ID}}}}",
         shell=True,
@@ -423,5 +429,9 @@ if __name__ == "__main__":
         subprocess.check_call(f"docker kill {' '.join(containers)}", shell=True)
         print(f"Containers {containers} killed")
 
+    if args.pre_pull:
+        print(("Running pre pull as: '" + cmd_pre_pull + "'."))
+        subprocess.check_call(cmd_pre_pull, shell=True)
+
     print(("Running pytest container as: '" + cmd + "'."))
     subprocess.check_call(cmd, shell=True)