Fix compose files to work without correct environment

2024-09-20 00:30:49 +00:00 · 2023-01-02 16:45:50 +00:00 · 2023-01-02 16:45:50 +00:00 · 631c55e379
commit 631c55e379
parent b40f6c0b10
18 changed files with 40 additions and 57 deletions
--- a/docker/test/integration/runner/compose/docker_compose_clickhouse.yml
+++ b/docker/test/integration/runner/compose/docker_compose_clickhouse.yml
@ -0,0 +1,5 @@
+version: '2.3'
+# Used to pre-pull images with docker-compose
+services:
+  clickhouse1:
+    image: clickhouse/integration-test
--- a/docker/test/integration/runner/compose/docker_compose_hdfs.yml
+++ b/docker/test/integration/runner/compose/docker_compose_hdfs.yml
@ -5,10 +5,10 @@ services:
        hostname: hdfs1
        restart: always
        expose:
-            - ${HDFS_NAME_PORT}
-            - ${HDFS_DATA_PORT}
+            - ${HDFS_NAME_PORT:-50070}
+            - ${HDFS_DATA_PORT:-50075}
        entrypoint: /etc/bootstrap.sh -d
        volumes:
            - type: ${HDFS_FS:-tmpfs}
              source: ${HDFS_LOGS:-}
-              target: /usr/local/hadoop/logs
+              target: /usr/local/hadoop/logs
--- a/docker/test/integration/runner/compose/docker_compose_kafka.yml
+++ b/docker/test/integration/runner/compose/docker_compose_kafka.yml
@ -15,7 +15,7 @@ services:
    image: confluentinc/cp-kafka:5.2.0
    hostname: kafka1
    ports:
-      - ${KAFKA_EXTERNAL_PORT}:${KAFKA_EXTERNAL_PORT}
+      - ${KAFKA_EXTERNAL_PORT:-8081}:${KAFKA_EXTERNAL_PORT:-8081}
    environment:
      KAFKA_ADVERTISED_LISTENERS: INSIDE://localhost:${KAFKA_EXTERNAL_PORT},OUTSIDE://kafka1:19092
      KAFKA_ADVERTISED_HOST_NAME: kafka1
@ -35,7 +35,7 @@ services:
    image: confluentinc/cp-schema-registry:5.2.0
    hostname: schema-registry
    ports:
-      - ${SCHEMA_REGISTRY_EXTERNAL_PORT}:${SCHEMA_REGISTRY_INTERNAL_PORT}
+      - ${SCHEMA_REGISTRY_EXTERNAL_PORT:-12313}:${SCHEMA_REGISTRY_INTERNAL_PORT:-12313}
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT
--- a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml
+++ b/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml
@ -15,8 +15,8 @@ services:
        source: ${KERBERIZED_HDFS_LOGS:-}
        target: /var/log/hadoop-hdfs
    expose:
-      - ${KERBERIZED_HDFS_NAME_PORT}
-      - ${KERBERIZED_HDFS_DATA_PORT}
+      - ${KERBERIZED_HDFS_NAME_PORT:-50070}
+      - ${KERBERIZED_HDFS_DATA_PORT:-1006}
    depends_on:
      - hdfskerberos
    entrypoint: /etc/bootstrap.sh -d
--- a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml
+++ b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml
@ -23,7 +23,7 @@ services:
    # restart: always
    hostname: kerberized_kafka1
    ports:
-      - ${KERBERIZED_KAFKA_EXTERNAL_PORT}:${KERBERIZED_KAFKA_EXTERNAL_PORT}
+      - ${KERBERIZED_KAFKA_EXTERNAL_PORT:-19092}:${KERBERIZED_KAFKA_EXTERNAL_PORT:-19092}
    environment:
      KAFKA_LISTENERS: OUTSIDE://:19092,UNSECURED_OUTSIDE://:19093,UNSECURED_INSIDE://0.0.0.0:${KERBERIZED_KAFKA_EXTERNAL_PORT}
      KAFKA_ADVERTISED_LISTENERS: OUTSIDE://kerberized_kafka1:19092,UNSECURED_OUTSIDE://kerberized_kafka1:19093,UNSECURED_INSIDE://localhost:${KERBERIZED_KAFKA_EXTERNAL_PORT}
@ -41,7 +41,7 @@ services:
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/secrets/broker_jaas.conf -Djava.security.krb5.conf=/etc/kafka/secrets/krb.conf -Dsun.security.krb5.debug=true"
    volumes:
-      - ${KERBERIZED_KAFKA_DIR}/secrets:/etc/kafka/secrets
+      - ${KERBERIZED_KAFKA_DIR:-}/secrets:/etc/kafka/secrets
      - /dev/urandom:/dev/random
    depends_on:
      - kafka_kerberized_zookeeper
--- a/docker/test/integration/runner/compose/docker_compose_meili.yml
+++ b/docker/test/integration/runner/compose/docker_compose_meili.yml
@ -4,13 +4,13 @@ services:
        image: getmeili/meilisearch:v0.27.0 
        restart: always
        ports:
-            - ${MEILI_EXTERNAL_PORT}:${MEILI_INTERNAL_PORT}
+            - ${MEILI_EXTERNAL_PORT:-7700}:${MEILI_INTERNAL_PORT:-7700}

    meili_secure:
        image: getmeili/meilisearch:v0.27.0 
        restart: always
        ports:
-            - ${MEILI_SECURE_EXTERNAL_PORT}:${MEILI_SECURE_INTERNAL_PORT}
+            - ${MEILI_SECURE_EXTERNAL_PORT:-7700}:${MEILI_SECURE_INTERNAL_PORT:-7700}
        environment:
            MEILI_MASTER_KEY: "password"

--- a/docker/test/integration/runner/compose/docker_compose_minio.yml
+++ b/docker/test/integration/runner/compose/docker_compose_minio.yml
@ -2,19 +2,18 @@ version: '2.3'

 services:
  minio1:
-    # Newer version of minio results in such errors:
-    # "AWSErrorMarshaller: Encountered AWSError 'InternalError': We encountered an internal error, please try again"
-    image: minio/minio:RELEASE.2021-09-23T04-46-24Z
+    image: minio/minio:RELEASE.2022-12-12T19-27-27Z
    volumes:
-      - data1-1:/data1
+      - ${MINIO_DATA_DIR}:/data1
      - ${MINIO_CERTS_DIR:-}:/certs
+      - ${MINIO_LOGS_DIR:-}:/logs
    expose:
-      - ${MINIO_PORT}
+      - ${MINIO_PORT:-9001}
    environment:
-      MINIO_ACCESS_KEY: minio
-      MINIO_SECRET_KEY: minio123
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: minio123
      MINIO_PROMETHEUS_AUTH_TYPE: public
-    command: server --address :9001 --certs-dir /certs /data1-1
+    command: server --address :9001 --certs-dir /certs /data1
    depends_on:
      - proxy1
      - proxy2
--- a/docker/test/integration/runner/compose/docker_compose_mongo.yml
+++ b/docker/test/integration/runner/compose/docker_compose_mongo.yml
@ -7,11 +7,11 @@ services:
            MONGO_INITDB_ROOT_USERNAME: root
            MONGO_INITDB_ROOT_PASSWORD: clickhouse
        ports:
-            - ${MONGO_EXTERNAL_PORT}:${MONGO_INTERNAL_PORT}
+            - ${MONGO_EXTERNAL_PORT:-27017}:${MONGO_INTERNAL_PORT:-27017}
        command: --profile=2 --verbose

    mongo2:
        image: mongo:5.0
        restart: always
        ports:
-            - ${MONGO_NO_CRED_EXTERNAL_PORT}:${MONGO_NO_CRED_INTERNAL_PORT}
+            - ${MONGO_NO_CRED_EXTERNAL_PORT:-27017}:${MONGO_NO_CRED_INTERNAL_PORT:-27017}
--- a/docker/test/integration/runner/compose/docker_compose_mongo_secure.yml
+++ b/docker/test/integration/runner/compose/docker_compose_mongo_secure.yml
@ -7,7 +7,7 @@ services:
            MONGO_INITDB_ROOT_USERNAME: root
            MONGO_INITDB_ROOT_PASSWORD: clickhouse
        volumes:
-            - ${MONGO_CONFIG_PATH}:/mongo/
+            - ${MONGO_CONFIG_PATH:-}:/mongo/
        ports:
-            - ${MONGO_EXTERNAL_PORT}:${MONGO_INTERNAL_PORT}
+            - ${MONGO_EXTERNAL_PORT:-27017}:${MONGO_INTERNAL_PORT:-27017}
        command: --config /mongo/mongo_secure.conf --profile=2 --verbose
--- a/docker/test/integration/runner/compose/docker_compose_mysql.yml
+++ b/docker/test/integration/runner/compose/docker_compose_mysql.yml
@ -8,7 +8,7 @@ services:
            MYSQL_ROOT_HOST: ${MYSQL_ROOT_HOST}
            DATADIR: /mysql/
        expose:
-            - ${MYSQL_PORT}
+            - ${MYSQL_PORT:-3306}
        command: --server_id=100 
            --log-bin='mysql-bin-1.log' 
            --default-time-zone='+3:00' 
--- a/docker/test/integration/runner/compose/docker_compose_mysql_5_7_for_materialized_mysql.yml
+++ b/docker/test/integration/runner/compose/docker_compose_mysql_5_7_for_materialized_mysql.yml
@ -1,21 +0,0 @@
-version: '2.3'
-services:
-    mysql1:
-        image: mysql:5.7
-        restart: 'no'
-        environment:
-            MYSQL_ROOT_PASSWORD: clickhouse
-        ports:
-            - 3308:3306
-        command: --server_id=100 --log-bin='mysql-bin-1.log'
-            --default-time-zone='+3:00'
-            --gtid-mode="ON"
-            --enforce-gtid-consistency
-            --log-error-verbosity=3
-            --log-error=/var/log/mysqld/error.log
-            --general-log=ON
-            --general-log-file=/var/log/mysqld/general.log
-        volumes:
-            - type: ${MYSQL_LOGS_FS:-tmpfs}
-              source: ${MYSQL_LOGS:-}
-              target: /var/log/mysqld/
--- a/docker/test/integration/runner/compose/docker_compose_mysql_8_0.yml
+++ b/docker/test/integration/runner/compose/docker_compose_mysql_8_0.yml
@ -8,7 +8,7 @@ services:
            MYSQL_ROOT_HOST: ${MYSQL_ROOT_HOST}
            DATADIR: /mysql/
        expose:
-            - ${MYSQL8_PORT}
+            - ${MYSQL8_PORT:-3306}
        command: --server_id=100 --log-bin='mysql-bin-1.log' 
            --default_authentication_plugin='mysql_native_password' 
            --default-time-zone='+3:00' --gtid-mode="ON" 
--- a/docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml
+++ b/docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml
@ -8,7 +8,7 @@ services:
            MYSQL_ROOT_HOST: ${MYSQL_CLUSTER_ROOT_HOST}
            DATADIR: /mysql/
        expose:
-            - ${MYSQL_CLUSTER_PORT}
+            - ${MYSQL_CLUSTER_PORT:-3306}
        command: --server_id=100 
            --log-bin='mysql-bin-2.log' 
            --default-time-zone='+3:00' 
@ -30,7 +30,7 @@ services:
            MYSQL_ROOT_HOST: ${MYSQL_CLUSTER_ROOT_HOST}
            DATADIR: /mysql/
        expose:
-            - ${MYSQL_CLUSTER_PORT}
+            - ${MYSQL_CLUSTER_PORT:-3306}
        command: --server_id=100 
            --log-bin='mysql-bin-3.log' 
            --default-time-zone='+3:00' 
@ -52,7 +52,7 @@ services:
            MYSQL_ROOT_HOST: ${MYSQL_CLUSTER_ROOT_HOST}
            DATADIR: /mysql/
        expose:
-            - ${MYSQL_CLUSTER_PORT}
+            - ${MYSQL_CLUSTER_PORT:-3306}
        command: --server_id=100 
            --log-bin='mysql-bin-4.log' 
            --default-time-zone='+3:00' 
--- a/docker/test/integration/runner/compose/docker_compose_nats.yml
+++ b/docker/test/integration/runner/compose/docker_compose_nats.yml
@ -3,9 +3,9 @@ services:
  nats1:
    image: nats
    ports:
-      - "${NATS_EXTERNAL_PORT}:${NATS_INTERNAL_PORT}"
+      - "${NATS_EXTERNAL_PORT:-4444}:${NATS_INTERNAL_PORT:-4444}"
    command: "-p 4444 --user click --pass house --tls --tlscert=/etc/certs/server-cert.pem --tlskey=/etc/certs/server-key.pem"
    volumes:
      - type: bind
-        source: "${NATS_CERT_DIR}/nats"
+        source: "${NATS_CERT_DIR:-}/nats"
        target: /etc/certs
--- a/docker/test/integration/runner/compose/docker_compose_postgres.yml
+++ b/docker/test/integration/runner/compose/docker_compose_postgres.yml
@ -5,7 +5,7 @@ services:
        command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=2", "-c", "logging_collector=on", "-c", "log_directory=/postgres/logs", "-c", "log_filename=postgresql.log", "-c", "log_statement=all", "-c", "max_connections=200"]
        restart: always
        expose:
-            - ${POSTGRES_PORT}
+            - ${POSTGRES_PORT:-5432}
        healthcheck:
            test: ["CMD-SHELL", "pg_isready -U postgres"]
            interval: 10s
--- a/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml
+++ b/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml
@ -9,7 +9,7 @@ services:
            POSTGRES_PASSWORD: mysecretpassword
            PGDATA: /postgres/data
        expose:
-            - ${POSTGRES_PORT}
+            - ${POSTGRES_PORT:-5432}
        volumes:
            - type: ${POSTGRES_LOGS_FS:-tmpfs}
              source: ${POSTGRES2_DIR:-}
@ -23,7 +23,7 @@ services:
            POSTGRES_PASSWORD: mysecretpassword
            PGDATA: /postgres/data
        expose:
-            - ${POSTGRES_PORT}
+            - ${POSTGRES_PORT:-5432}
        volumes:
            - type: ${POSTGRES_LOGS_FS:-tmpfs}
              source: ${POSTGRES3_DIR:-}
@ -37,7 +37,7 @@ services:
            POSTGRES_PASSWORD: mysecretpassword
            PGDATA: /postgres/data
        expose:
-            - ${POSTGRES_PORT}
+            - ${POSTGRES_PORT:-5432}
        volumes:
            - type: ${POSTGRES_LOGS_FS:-tmpfs}
              source: ${POSTGRES4_DIR:-}
--- a/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml
+++ b/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml
@ -5,7 +5,7 @@ services:
        image: rabbitmq:3.8-management-alpine
        hostname: rabbitmq1
        expose:
-            - ${RABBITMQ_PORT}
+            - ${RABBITMQ_PORT:-5672}
        environment:
            RABBITMQ_DEFAULT_USER: "root"
            RABBITMQ_DEFAULT_PASS: "clickhouse"
--- a/docker/test/integration/runner/compose/docker_compose_redis.yml
+++ b/docker/test/integration/runner/compose/docker_compose_redis.yml
@ -4,5 +4,5 @@ services:
        image: redis
        restart: always
        ports:
-            - ${REDIS_EXTERNAL_PORT}:${REDIS_INTERNAL_PORT}
+            - ${REDIS_EXTERNAL_PORT:-6379}:${REDIS_INTERNAL_PORT:-6379}
        command: redis-server --requirepass "clickhouse" --databases 32